@yahyahammad/venastine 0.2.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +233 -94
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +277 -43
  12. package/agents/tui_commands.py +7 -12
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +158 -20
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1238 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +830 -66
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +1516 -440
  38. package/core/memory.py +120 -8
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +158 -26
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1671 -199
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +7 -12
  70. package/package.json +234 -4
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +51 -2
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +353 -17
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +368 -35
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +347 -30
  118. package/tui/app.py +4645 -438
  119. package/tui/app.tcss +185 -2
  120. package/tui/commands.py +10 -154
  121. package/tui/diffs.py +286 -265
  122. package/tui/hosted_driver.py +102 -0
  123. package/tui/markdown.py +60 -10
  124. package/tui/meters.py +103 -7
  125. package/tui/screens.py +346 -92
  126. package/tui/themes.py +59 -19
  127. package/tui/widgets.py +1575 -111
  128. package/SECURITY.md +0 -88
  129. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -31,17 +31,16 @@ from __future__ import annotations
31
31
  import ipaddress
32
32
  import json
33
33
  import logging
34
- import os
35
34
  import re
36
35
  import socket
37
36
  from urllib.parse import urlparse
38
37
 
39
- import config
40
38
  # safety -> security, and never the reverse. `security.posture` imports
41
39
  # `config` and the stdlib only, so this adds no cycle -- which is exactly
42
40
  # why batch 37 kept `logging_setup` OUT of `security/`: that one pulls in
43
41
  # this module, and the edge only stays acyclic while it runs one way.
44
42
  from security import posture
43
+ from security import secrets as harness_secrets
45
44
 
46
45
  logger = logging.getLogger(__name__)
47
46
 
@@ -143,7 +142,9 @@ def is_url_permitted(url: str, resolve: bool = True) -> str | None:
143
142
  <that URL>" to a grounding pass that obliges.
144
143
 
145
144
  `resolve=False` answers from the literal text only -- an IP literal
146
- is still checked, a NAME is taken on trust. That is for `web_search`,
145
+ is still checked, in every spelling the platform's resolver reads as
146
+ a number (#207: `2130706433` and `0x7f.0.0.1` too, not only dotted
147
+ quads), and a NAME is taken on trust. That is for `web_search`,
147
148
  which filters results rather than fetching them: a DNS lookup per
148
149
  search hit is real latency for a check whose failure mode is "the
149
150
  model is shown a URL that fetch_url will refuse anyway". The caller
@@ -174,6 +175,21 @@ def is_url_permitted(url: str, resolve: bool = True) -> str | None:
174
175
  except ValueError:
175
176
  literal = None
176
177
 
178
+ if literal is None and not resolve:
179
+ # #207, batch 213. `ipaddress` reads dotted quads and IPv6 only, so
180
+ # `2130706433`, `0x7f.0.0.1` and `127.1` -- all 127.0.0.1 to the
181
+ # platform's resolver -- were taken on trust as NAMES here, and
182
+ # only fetch_url refused them. AI_NUMERICHOST asks that resolver to
183
+ # read the text as a number and never to look anything up, so it
184
+ # costs no network, which was the whole reason for resolve=False.
185
+ # Measured on Windows and in WSL: the same spellings parse, and a
186
+ # name fails with gaierror.
187
+ try:
188
+ literal = ipaddress.ip_address(socket.getaddrinfo(
189
+ host, None, flags=socket.AI_NUMERICHOST)[0][4][0])
190
+ except (socket.gaierror, ValueError, IndexError):
191
+ literal = None
192
+
177
193
  if literal is not None:
178
194
  addresses = [literal]
179
195
  elif resolve:
@@ -206,6 +222,18 @@ def is_url_permitted(url: str, resolve: bool = True) -> str | None:
206
222
  # ---- Secret redaction -----------------------------------------------------
207
223
  # ---------------------------------------------------------------------------
208
224
 
225
+ # A PEM body's line break: a real one, or the two-character escape a JSON
226
+ # string carries -- a cloud service-account key file is exactly that shape.
227
+ # Unambiguous on purpose: every repetition holds a mandatory break, so the
228
+ # spaces between two breaks belong to exactly one repetition and a body
229
+ # that fails to close backtracks in linear time, not exponential.
230
+ _PEM_BREAK = r"(?:[ \t]*(?:\r?\n|\\r\\n|\\n))"
231
+ _PEM_SEP = _PEM_BREAK + r"+[ \t]*"
232
+ # One body line: base64, at least 16 characters, so a word of prose after a
233
+ # quoted header is not one; or the two RFC 1421 headers a legacy encrypted
234
+ # key carries before its base64.
235
+ _PEM_UNIT = r"(?:[A-Za-z0-9+/=]{16,}|(?:Proc-Type|DEK-Info):[^\r\n\\]*)"
236
+
209
237
  _SECRET_PATTERNS = [
210
238
  # OpenAI-style keys
211
239
  re.compile(r"sk-[a-zA-Z0-9]{20,}"),
@@ -219,8 +247,55 @@ _SECRET_PATTERNS = [
219
247
  re.compile(r"AIza[0-9A-Za-z\-_]{35}"),
220
248
  # Slack tokens
221
249
  re.compile(r"xox[baprs]-[0-9A-Za-z\-]{10,}"),
222
- # PEM private key headers
223
- re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
250
+ # PEM private keys: the header, and the WHOLE BLOCK when it is there
251
+ # (TECHNICAL_DEBT 52, batch 160). This used to be the header line alone,
252
+ # so the base64 body -- the key material -- survived verbatim under a
253
+ # `[REDACTED]` that read as the key having been removed. The body is
254
+ # OPTIONAL in the pattern on purpose: `_input_leaf` uses `.search()`,
255
+ # and anything carrying the header still matches, so input refusals are
256
+ # exactly what they were. The body is LINES, each after a line break:
257
+ # a block with its END line goes whole, and one a cut ended goes as far
258
+ # as its lines do. Lines rather than "anything up to END" because prose
259
+ # quotes both markers -- "starts with -----BEGIN ... and ends with
260
+ # -----END ..." -- and a lazy match took the sentence between them.
261
+ # Measured: that sentence now loses its two markers and keeps its words.
262
+ re.compile(
263
+ r"-----BEGIN [A-Z ]*PRIVATE KEY-----"
264
+ r"(?:(?:" + _PEM_SEP + _PEM_UNIT + r")*"
265
+ r"(?:" + _PEM_SEP + r"[A-Za-z0-9+/=]{1,15})?"
266
+ r"(?:" + _PEM_BREAK + r"*[ \t]*)-----END [A-Z ]*PRIVATE KEY-----"
267
+ r"|(?:" + _PEM_SEP + _PEM_UNIT + r")*)"),
268
+ # --- Batch 160, TECHNICAL_DEBT 52: the CURRENT siblings of the vendor
269
+ # --- formats above, each of which the older pattern beside it misses
270
+ # --- by a character class -- a `_` or a second `-` the 2024 shapes
271
+ # --- never had. Measured through the real `redact_output_text` before
272
+ # --- this: every one survived. They refuse inputs too, as `ghp_` and
273
+ # --- `sk-` always have; one vendor list, one vocabulary.
274
+ # GitHub fine-grained PATs, the format `ghp_` is being retired for
275
+ re.compile(r"github_pat_[A-Za-z0-9_]{22,}"),
276
+ # GitHub OAuth, user-to-server, server-to-server and refresh tokens
277
+ re.compile(r"gh[ousr]_[A-Za-z0-9]{36}"),
278
+ # OpenAI project, service-account and admin keys: `sk-proj-...`
279
+ re.compile(r"sk-(?:proj|svcacct|admin)-[A-Za-z0-9_\-]{20,}"),
280
+ # Stripe live secret and restricted keys
281
+ re.compile(r"[sr]k_live_[A-Za-z0-9]{16,}"),
282
+ # npm access tokens
283
+ re.compile(r"npm_[A-Za-z0-9]{36}"),
284
+ # --- ROADMAP_v3 §79 (HD5): the three credentials `/connect` hands the
285
+ # --- harness that no pattern above can see. A sign-in puts them in a
286
+ # --- session the model shares, so they are redacted and refused like
287
+ # --- the rest of the list.
288
+ # Hugging Face user access tokens and OAuth access tokens. An
289
+ # identifier like `hf_hub_download` breaks the run of word characters
290
+ # at its second underscore and is not matched.
291
+ re.compile(r"(?<![A-Za-z0-9])hf_(?:oauth_)?[A-Za-z0-9]{30,}"),
292
+ # OpenRouter keys, which `sk-` above misses at the second hyphen
293
+ re.compile(r"sk-or-v1-[A-Za-z0-9]{32,}"),
294
+ # ModelScope access tokens in their prefixed form: `ms-` and a UUID. A
295
+ # bare UUID is NOT matched -- thread ids are UUIDs -- and neither is one
296
+ # that merely ENDS a word like `items-<uuid>`.
297
+ re.compile(r"(?<![A-Za-z0-9])ms-[0-9a-f]{8}-[0-9a-f]{4}"
298
+ r"-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"),
224
299
  ]
225
300
 
226
301
 
@@ -246,7 +321,7 @@ def redact_secrets(text: str) -> str:
246
321
  # ---- Credential shapes (#167, batch 20) ------------------------------------
247
322
  # ---------------------------------------------------------------------------
248
323
  #
249
- # The seven patterns above are VENDOR TOKENS -- every one has a prefix some
324
+ # The patterns above are VENDOR TOKENS -- every one has a prefix some
250
325
  # company mints, so a match is high-confidence and a bare `[REDACTED]` loses
251
326
  # nothing structural. A credential that is just a password in a field has no
252
327
  # prefix, and nothing matched it (#167): batch 14 added five build manifests
@@ -306,9 +381,102 @@ _ASSIGNMENT_RE = re.compile(
306
381
  r"(?i)\b(password|passwd)\b(\s*(?:[=:][ \t]*|[ \t]))"
307
382
  r"([\"'])([^\"'\n]{4,200})\3")
308
383
 
384
+ # --- Batch 160, TECHNICAL_DEBT 52: three STRUCTURAL shapes, output only by
385
+ # --- owner ruling. Each carries a credential with no vendor prefix, and
386
+ # --- each anchors on structure a security paper does not produce by
387
+ # --- accident, which is the false-positive argument the password shapes
388
+ # --- above already make.
389
+
390
+ # Authorization: Bearer abc... -> Authorization: Bearer [REDACTED]. Anchored
391
+ # on the HEADER NAME, never on the scheme word: "Bearer tokens expire" is
392
+ # prose and must survive, and that is the keyword-widening the comment above
393
+ # refuses. The separator admits a header line, a JSON or dict key (`"..":`)
394
+ # and `=`, so a curl line, a captured request and a logged dict all match.
395
+ # The value's alphabet has no `$ < > { } [`, so `Bearer $TOKEN`,
396
+ # `Bearer <token>` and an already-redacted value never match, and
397
+ # `Digest username=...`, whose secret is not one value, is left alone. A
398
+ # value with NO scheme word must also be 16 characters and carry a digit,
399
+ # because API documentation writes "Authorization: required" and a bare
400
+ # 8-character floor redacted the word.
401
+ # Groups: 1 header name, 2 separator, 3 scheme (or None).
402
+ _AUTH_HEADER_RE = re.compile(
403
+ r"(?i)\b((?:proxy-)?authorization)([\"']?\s*[:=]\s*[\"']?)"
404
+ r"(?:((?:bearer|basic|token)\s+)[A-Za-z0-9._~+/=\-]{8,}"
405
+ r"|(?=[A-Za-z._~+/=\-]*[0-9])[A-Za-z0-9._~+/=\-]{16,})")
406
+
407
+ # A JSON Web Token, whole: three base64url segments, the first two of
408
+ # which open with `eyJ` because both are JSON objects (`{"`). A bearer
409
+ # credential wherever it appears, and the shape `_AUTH_HEADER_RE` cannot
410
+ # reach when the token is pasted without its header.
411
+ _JWT_RE = re.compile(
412
+ r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")
413
+
414
+ # The END of a PEM key block whose BEGIN is not in this string: the footer
415
+ # and the run of base64 lines above it. The vendor pattern takes a block
416
+ # from its header; this is the other half, for text a cut started in the
417
+ # middle of a block -- a page, a session's tail as its start sweeps through
418
+ # the output. Found as a literal and walked BACKWARDS in code, because a
419
+ # regex that has to start at the first body line is tried at every line
420
+ # start of a long base64 blob and is quadratic on one with no footer.
421
+ _PEM_END_RE = re.compile(r"-----END [A-Z ]*PRIVATE KEY-----")
422
+ _PEM_BODY_CHARS = frozenset(
423
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")
424
+
425
+
426
+ def _before_pem_break(text: str, i: int):
427
+ """The index before the line break(s) ending at *i*, or None.
428
+
429
+ `_PEM_SEP` read backwards: spaces, then one or more breaks (real, or
430
+ the JSON escape), each with its own spaces."""
431
+ k = i
432
+ while k and text[k - 1] in " \t":
433
+ k -= 1
434
+ seen = False
435
+ while True:
436
+ if k and text[k - 1] == "\n":
437
+ k -= 1
438
+ if k and text[k - 1] == "\r":
439
+ k -= 1
440
+ elif k >= 2 and text[k - 2:k] == "\\n":
441
+ k -= 2
442
+ if k >= 2 and text[k - 2:k] == "\\r":
443
+ k -= 2
444
+ else:
445
+ break
446
+ seen = True
447
+ while k and text[k - 1] in " \t":
448
+ k -= 1
449
+ return k if seen else None
450
+
451
+
452
+ def _redact_pem_tails(text: str) -> str:
453
+ """Replace each orphan footer, and the base64 lines above it, whole.
454
+
455
+ Linear: each walk stops at the first character outside the body
456
+ alphabet, and a footer's own `-` is one, so no two walks share text."""
457
+ pieces, last = [], 0
458
+ for m in _PEM_END_RE.finditer(text):
459
+ start = i = m.start()
460
+ while True:
461
+ k = _before_pem_break(text, i)
462
+ if k is None:
463
+ break
464
+ r = k
465
+ while r > last and text[r - 1] in _PEM_BODY_CHARS:
466
+ r -= 1
467
+ if r == k:
468
+ break
469
+ start = i = r
470
+ pieces.append(text[last:start])
471
+ pieces.append(REDACTION_MARKER)
472
+ last = m.end()
473
+ pieces.append(text[last:])
474
+ return "".join(pieces)
475
+
309
476
 
310
477
  def _redact_credential_shapes(text: str) -> str:
311
- """Apply the three context shapes, preserving surrounding structure.
478
+ """Apply the context and structural shapes, preserving surrounding
479
+ structure.
312
480
 
313
481
  Each replacer checks the captured VALUE against the placeholder shapes
314
482
  before substituting -- a template reference is exactly what docs and
@@ -329,7 +497,13 @@ def _redact_credential_shapes(text: str) -> str:
329
497
  return (f"{m.group(1)}{m.group(2)}{m.group(3)}"
330
498
  f"{REDACTION_MARKER}{m.group(3)}")
331
499
 
332
- return _ASSIGNMENT_RE.sub(_assign, text)
500
+ text = _ASSIGNMENT_RE.sub(_assign, text)
501
+
502
+ text = _AUTH_HEADER_RE.sub(
503
+ lambda m: (f"{m.group(1)}{m.group(2)}{m.group(3) or ''}"
504
+ f"{REDACTION_MARKER}"), text)
505
+ text = _JWT_RE.sub(REDACTION_MARKER, text)
506
+ return _redact_pem_tails(text)
333
507
 
334
508
 
335
509
  def redaction_enabled() -> bool:
@@ -353,8 +527,13 @@ def redaction_enabled() -> bool:
353
527
 
354
528
  What this does NOT govern, whichever way it reads: input refusals
355
529
  (_input_leaf -- a denial is legible, not destructive), the depth-cap
356
- substitution (fail-closed structural bound), and logging_setup's
357
- formatter redaction (the second sink guards app.log regardless).
530
+ substitution (fail-closed structural bound), and `redact_diagnostic_text`
531
+ -- the harness's own diagnostics, which guard app.log, the transcript,
532
+ `MessageLog.error` and a pipeline record regardless of this answer. That
533
+ last clause used to name `logging_setup` alone and to be true only about
534
+ the moment of reading: the formatter did run regardless, with a strictly
535
+ SMALLER vocabulary than this function's, which is the gap batch 144
536
+ closed. The two now differ in one respect and it is the one named here.
358
537
  """
359
538
  active = posture.current()
360
539
  if not active.redact_tool_outputs:
@@ -362,6 +541,71 @@ def redaction_enabled() -> bool:
362
541
  return not active.redact_off_env
363
542
 
364
543
 
544
+ def redact_diagnostic_text(text: str) -> str:
545
+ """The one redaction path the HARNESS'S OWN DIAGNOSTICS take.
546
+
547
+ THE SAME VOCABULARY AS TOOL OUTPUT, which it was not until batch 144
548
+ (batch 144). `redact_secrets` is the seven vendor-token patterns and
549
+ nothing else. `_redact_credential_shapes` is the only thing in this
550
+ project that matches a credential with NO vendor prefix -- a URL's
551
+ userinfo, a `<password>` element, a quoted `password =` assignment --
552
+ and it had exactly one caller, `redact_output_text`. So one string took
553
+ two different vocabularies depending on which sink it reached. Driven
554
+ through the real `configure_logging` with nothing monkeypatched, on
555
+ issue #167's own seven cases:
556
+
557
+ case app.log transcript tool output
558
+ anthropic key redacted redacted redacted
559
+ openai key redacted redacted redacted
560
+ github pat redacted redacted redacted
561
+ aws access key id redacted redacted redacted
562
+ gradle password LEAKED LEAKED redacted
563
+ maven server password LEAKED LEAKED redacted
564
+ gem source url LEAKED LEAKED redacted
565
+
566
+ The four vendor rows are the control: they prove the formatter was
567
+ attached and firing, so the three LEAKED rows were the vocabulary and
568
+ not a probe that had stopped working. It needed no contrivance to
569
+ reach: `fetch_url` logs the URL it failed on BY DESIGN and says so in
570
+ a comment naming this redaction as the guard, and `core/loop.py` puts
571
+ the same description in `MessageLog.error`, where it persists in
572
+ `app.db` -- which `docs/PRIVACY.md` says is deleted by wiping the
573
+ database rather than one thread.
574
+
575
+ FIVE CALLERS, and the one-caller shape is what made the gap
576
+ survivable: `logging_setup`'s formatter, `tui/app.py`'s
577
+ `TranscriptLogHandler.emit` and its tool-error line, `core/loop.py`'s
578
+ `mark_turn_failed`, and the orchestrator's pipeline event. Adding a
579
+ pattern or a shape now reaches all five, and
580
+ `tests/test_logging_setup.py`'s parity test is what fails if a sixth
581
+ sink composes its own.
582
+
583
+ UNCONDITIONAL, all three passes, and that is not an extension of the
584
+ switch's reach but the removal of a difference. `redaction_enabled()`
585
+ governs what the MODEL is shown, which a user may reasonably choose to
586
+ see raw for one run; it has never governed what `app.log` keeps across
587
+ runs, and the switch's own docstring already said so.
588
+
589
+ LIVE VALUES FIRST, which is the finding rather than a preference
590
+ (batch 144). `redact_live_values` matches a stored value by exact
591
+ substring, so a pattern pass that has already rewritten part of one
592
+ leaves the remainder standing. Measured, with a stored value that
593
+ properly contains a vendor token: the log's old order returned
594
+ `PREFIX-[REDACTED]-SUFFIX` and this one returns `[REDACTED]`. The repo
595
+ has locked the general rule three times -- `fetch_url` redacts before
596
+ it cuts, `param_digest` redacts before truncating, the diff block runs
597
+ on whole texts -- and this is the same rule where the destroyer is a
598
+ substitution rather than a cut.
599
+
600
+ The cost is measured and is per log RECORD: 6.7 us -> 9.4 us on Linux,
601
+ +40%, for three more regex passes over a line that already takes
602
+ seven. Invisible at INFO, and under a millisecond per thousand records
603
+ at DEBUG.
604
+ """
605
+ return _redact_credential_shapes(
606
+ redact_secrets(harness_secrets.redact_live_values(text)))
607
+
608
+
365
609
  def redact_output_text(text: str) -> str:
366
610
  """The one redaction path tool OUTPUT takes, under one switch.
367
611
 
@@ -379,10 +623,57 @@ def redact_output_text(text: str) -> str:
379
623
  redacted LESS than the line it replaced -- the vendor tokens but not
380
624
  the credential shapes -- which is a regression dressed as a feature.
381
625
  The docstring above already called this "the one path"; it now is one
382
- for both consumers."""
626
+ for both consumers.
627
+
628
+ AND IT IS NOW `redact_diagnostic_text` UNDER THE SWITCH (batch 144),
629
+ rather than its own copy of the same three passes. The two had drifted
630
+ into different vocabularies AND different orders while reading as
631
+ siblings, which is why the difference that is deliberate is now the
632
+ only one there is."""
633
+ # SS61: THE HARNESS'S OWN SECRETS COME OUT FIRST, AND UNCONDITIONALLY.
634
+ # Above the switch on purpose, and still first inside the shared
635
+ # composition below. `redact_tool_outputs` governs pattern GUESSES
636
+ # about other people's credentials, and a user who turns it off has
637
+ # chosen to see those raw. A value this harness put into a subprocess,
638
+ # coming back out of one, is a different thing: passing it to the model
639
+ # is the harness leaking its OWN secret, and no switch asks for that.
640
+ # It sits beside the depth cap, which this module already keeps
641
+ # fail-closed for the same shape of reason -- it is structure and
642
+ # self-inflicted leakage, not content judgment.
643
+ #
644
+ # The VALUES never come here: security/secrets.py does the substitution
645
+ # itself, so exactly one module holds plaintext and the redactor is not
646
+ # it (fix at the producer).
383
647
  if not redaction_enabled():
384
- return text
385
- return _redact_credential_shapes(redact_secrets(text))
648
+ return harness_secrets.redact_live_values(text)
649
+ return redact_diagnostic_text(text)
650
+
651
+
652
+ #: What a credential is spelled with: base64 and base64url, hex, and the
653
+ #: separators vendor tokens use. A run of these that a cut splits is the
654
+ #: fragment the cut drops, because the redactor runs after the cut and
655
+ #: cannot match half a token. Punctuation that no token contains -- quotes,
656
+ #: `:`, `@`, braces, commas -- ends a run, so a cut through minified JSON
657
+ #: costs one value and not the whole line. Two cuts take it: a shell
658
+ #: session's dropped middle (A72) and an MCP result's bound (A79).
659
+ TOKEN_CHARS = frozenset(
660
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-+/=.~")
661
+
662
+
663
+ def fragment_end(head: str) -> int:
664
+ """Where the token-character run that ends *head* begins."""
665
+ i = len(head)
666
+ while i and head[i - 1] in TOKEN_CHARS:
667
+ i -= 1
668
+ return i
669
+
670
+
671
+ def fragment_start(tail: str) -> int:
672
+ """Where the token-character run that starts *tail* ends."""
673
+ i = 0
674
+ while i < len(tail) and tail[i] in TOKEN_CHARS:
675
+ i += 1
676
+ return i
386
677
 
387
678
 
388
679
 
@@ -393,6 +684,28 @@ def redact_output_text(text: str) -> str:
393
684
  _DIGEST_VALUE_CHARS = 60
394
685
  _DIGEST_CHARS = 140
395
686
 
687
+ # A VALUE CARRYING A URL GETS ITS OWN, WIDER CAP, and the digest a wider
688
+ # total to hold one (batch 96).
689
+ #
690
+ # Batch 65 made a truncated URL clickable by carrying the whole one
691
+ # alongside the line, so the click has always opened the right page. What
692
+ # it could not fix is the TERMINAL: `https://host/a/b…` minus the ellipsis
693
+ # is itself a syntactically valid URL, so a terminal's own link detector
694
+ # sees a shorter address than the harness resolved and ctrl+click opens
695
+ # THAT -- measured, and reported as the click going somewhere wrong.
696
+ #
697
+ # The honest fix is to stop producing the mismatch: draw the whole URL, so
698
+ # the visible text is the target again and the two layers cannot disagree.
699
+ # `markdown.link_spans`' elision stays for anything past this cap, where
700
+ # the invariant is still "the visible text tells you the origin".
701
+ #
702
+ # BOTH CAPS MOVE OR NEITHER DOES. A 200-character value cut at a
703
+ # 140-character TOTAL is the same mismatch one line further down, which is
704
+ # why the total is raised only for a digest that actually holds a URL --
705
+ # every other digest keeps the width it has always had.
706
+ _DIGEST_URL_CHARS = 200
707
+ _DIGEST_URL_TOTAL = 240
708
+
396
709
 
397
710
  def param_digest(params, omit=()) -> str:
398
711
  """A short, REDACTED description of what a tool was called with.
@@ -437,6 +750,7 @@ def param_digest(params, omit=()) -> str:
437
750
  if not isinstance(params, dict) or not params:
438
751
  return ""
439
752
  parts = []
753
+ widened = False
440
754
  for key, value in params.items():
441
755
  if key in omit:
442
756
  continue
@@ -448,12 +762,34 @@ def param_digest(params, omit=()) -> str:
448
762
  # like every other pattern substitution; it stays display-only
449
763
  # and never logs, whatever the switch reads.
450
764
  text = redact_output_text(text)
451
- if len(text) > _DIGEST_VALUE_CHARS:
452
- text = text[:_DIGEST_VALUE_CHARS - 1] + "…"
765
+ # ONE LINE, and the collapse happens BEFORE the truncation so that
766
+ # sixty characters means sixty VISIBLE ones (batch 96). A digest is
767
+ # drawn as a single row -- `▸ shell <digest>` -- and nothing
768
+ # downstream re-flows it, so a multi-line command (a heredoc, an
769
+ # inline `python3 - <<EOF`) drew its remaining lines at column 0
770
+ # underneath, outside the indent and outside the tool's line. The
771
+ # transcript, `core/replay.py` and the research pass trace all read
772
+ # this one producer, so all three drew it.
773
+ #
774
+ # AFTER the redaction, which is the ordering this function exists
775
+ # to hold: the redactor works on whole strings, and collapsing
776
+ # first would not change what it matches, but doing it after keeps
777
+ # the rule "redact, THEN shorten" true of every step below it.
778
+ text = " ".join(text.split())
779
+ # `http` is a FILTER, not a grammar: which substrings are URLs is
780
+ # `tui/markdown.py`'s question, asked there against one definition.
781
+ # This only declines to widen a value that cannot contain one --
782
+ # the same test, and the same reasoning, as `redacted_values`.
783
+ has_url = "http" in text
784
+ widened = widened or has_url
785
+ cap = _DIGEST_URL_CHARS if has_url else _DIGEST_VALUE_CHARS
786
+ if len(text) > cap:
787
+ text = text[:cap - 1] + "…"
453
788
  parts.append(text)
454
789
  digest = " ".join(p for p in parts if p)
455
- if len(digest) > _DIGEST_CHARS:
456
- digest = digest[:_DIGEST_CHARS - 1] + "…"
790
+ total = _DIGEST_URL_TOTAL if widened else _DIGEST_CHARS
791
+ if len(digest) > total:
792
+ digest = digest[:total - 1] + "…"
457
793
  return digest
458
794
 
459
795