@yahyahammad/venastine 0.2.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +233 -94
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +277 -43
- package/agents/tui_commands.py +7 -12
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +158 -20
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1238 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +830 -66
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +1516 -440
- package/core/memory.py +120 -8
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +158 -26
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1671 -199
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +7 -12
- package/package.json +234 -4
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +51 -2
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +353 -17
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +368 -35
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +347 -30
- package/tui/app.py +4645 -438
- package/tui/app.tcss +185 -2
- package/tui/commands.py +10 -154
- package/tui/diffs.py +286 -265
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +60 -10
- package/tui/meters.py +103 -7
- package/tui/screens.py +346 -92
- package/tui/themes.py +59 -19
- package/tui/widgets.py +1575 -111
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
|
@@ -31,17 +31,16 @@ from __future__ import annotations
|
|
|
31
31
|
import ipaddress
|
|
32
32
|
import json
|
|
33
33
|
import logging
|
|
34
|
-
import os
|
|
35
34
|
import re
|
|
36
35
|
import socket
|
|
37
36
|
from urllib.parse import urlparse
|
|
38
37
|
|
|
39
|
-
import config
|
|
40
38
|
# safety -> security, and never the reverse. `security.posture` imports
|
|
41
39
|
# `config` and the stdlib only, so this adds no cycle -- which is exactly
|
|
42
40
|
# why batch 37 kept `logging_setup` OUT of `security/`: that one pulls in
|
|
43
41
|
# this module, and the edge only stays acyclic while it runs one way.
|
|
44
42
|
from security import posture
|
|
43
|
+
from security import secrets as harness_secrets
|
|
45
44
|
|
|
46
45
|
logger = logging.getLogger(__name__)
|
|
47
46
|
|
|
@@ -143,7 +142,9 @@ def is_url_permitted(url: str, resolve: bool = True) -> str | None:
|
|
|
143
142
|
<that URL>" to a grounding pass that obliges.
|
|
144
143
|
|
|
145
144
|
`resolve=False` answers from the literal text only -- an IP literal
|
|
146
|
-
is still checked,
|
|
145
|
+
is still checked, in every spelling the platform's resolver reads as
|
|
146
|
+
a number (#207: `2130706433` and `0x7f.0.0.1` too, not only dotted
|
|
147
|
+
quads), and a NAME is taken on trust. That is for `web_search`,
|
|
147
148
|
which filters results rather than fetching them: a DNS lookup per
|
|
148
149
|
search hit is real latency for a check whose failure mode is "the
|
|
149
150
|
model is shown a URL that fetch_url will refuse anyway". The caller
|
|
@@ -174,6 +175,21 @@ def is_url_permitted(url: str, resolve: bool = True) -> str | None:
|
|
|
174
175
|
except ValueError:
|
|
175
176
|
literal = None
|
|
176
177
|
|
|
178
|
+
if literal is None and not resolve:
|
|
179
|
+
# #207, batch 213. `ipaddress` reads dotted quads and IPv6 only, so
|
|
180
|
+
# `2130706433`, `0x7f.0.0.1` and `127.1` -- all 127.0.0.1 to the
|
|
181
|
+
# platform's resolver -- were taken on trust as NAMES here, and
|
|
182
|
+
# only fetch_url refused them. AI_NUMERICHOST asks that resolver to
|
|
183
|
+
# read the text as a number and never to look anything up, so it
|
|
184
|
+
# costs no network, which was the whole reason for resolve=False.
|
|
185
|
+
# Measured on Windows and in WSL: the same spellings parse, and a
|
|
186
|
+
# name fails with gaierror.
|
|
187
|
+
try:
|
|
188
|
+
literal = ipaddress.ip_address(socket.getaddrinfo(
|
|
189
|
+
host, None, flags=socket.AI_NUMERICHOST)[0][4][0])
|
|
190
|
+
except (socket.gaierror, ValueError, IndexError):
|
|
191
|
+
literal = None
|
|
192
|
+
|
|
177
193
|
if literal is not None:
|
|
178
194
|
addresses = [literal]
|
|
179
195
|
elif resolve:
|
|
@@ -206,6 +222,18 @@ def is_url_permitted(url: str, resolve: bool = True) -> str | None:
|
|
|
206
222
|
# ---- Secret redaction -----------------------------------------------------
|
|
207
223
|
# ---------------------------------------------------------------------------
|
|
208
224
|
|
|
225
|
+
# A PEM body's line break: a real one, or the two-character escape a JSON
|
|
226
|
+
# string carries -- a cloud service-account key file is exactly that shape.
|
|
227
|
+
# Unambiguous on purpose: every repetition holds a mandatory break, so the
|
|
228
|
+
# spaces between two breaks belong to exactly one repetition and a body
|
|
229
|
+
# that fails to close backtracks in linear time, not exponential.
|
|
230
|
+
_PEM_BREAK = r"(?:[ \t]*(?:\r?\n|\\r\\n|\\n))"
|
|
231
|
+
_PEM_SEP = _PEM_BREAK + r"+[ \t]*"
|
|
232
|
+
# One body line: base64, at least 16 characters, so a word of prose after a
|
|
233
|
+
# quoted header is not one; or the two RFC 1421 headers a legacy encrypted
|
|
234
|
+
# key carries before its base64.
|
|
235
|
+
_PEM_UNIT = r"(?:[A-Za-z0-9+/=]{16,}|(?:Proc-Type|DEK-Info):[^\r\n\\]*)"
|
|
236
|
+
|
|
209
237
|
_SECRET_PATTERNS = [
|
|
210
238
|
# OpenAI-style keys
|
|
211
239
|
re.compile(r"sk-[a-zA-Z0-9]{20,}"),
|
|
@@ -219,8 +247,55 @@ _SECRET_PATTERNS = [
|
|
|
219
247
|
re.compile(r"AIza[0-9A-Za-z\-_]{35}"),
|
|
220
248
|
# Slack tokens
|
|
221
249
|
re.compile(r"xox[baprs]-[0-9A-Za-z\-]{10,}"),
|
|
222
|
-
# PEM private
|
|
223
|
-
|
|
250
|
+
# PEM private keys: the header, and the WHOLE BLOCK when it is there
|
|
251
|
+
# (TECHNICAL_DEBT 52, batch 160). This used to be the header line alone,
|
|
252
|
+
# so the base64 body -- the key material -- survived verbatim under a
|
|
253
|
+
# `[REDACTED]` that read as the key having been removed. The body is
|
|
254
|
+
# OPTIONAL in the pattern on purpose: `_input_leaf` uses `.search()`,
|
|
255
|
+
# and anything carrying the header still matches, so input refusals are
|
|
256
|
+
# exactly what they were. The body is LINES, each after a line break:
|
|
257
|
+
# a block with its END line goes whole, and one a cut ended goes as far
|
|
258
|
+
# as its lines do. Lines rather than "anything up to END" because prose
|
|
259
|
+
# quotes both markers -- "starts with -----BEGIN ... and ends with
|
|
260
|
+
# -----END ..." -- and a lazy match took the sentence between them.
|
|
261
|
+
# Measured: that sentence now loses its two markers and keeps its words.
|
|
262
|
+
re.compile(
|
|
263
|
+
r"-----BEGIN [A-Z ]*PRIVATE KEY-----"
|
|
264
|
+
r"(?:(?:" + _PEM_SEP + _PEM_UNIT + r")*"
|
|
265
|
+
r"(?:" + _PEM_SEP + r"[A-Za-z0-9+/=]{1,15})?"
|
|
266
|
+
r"(?:" + _PEM_BREAK + r"*[ \t]*)-----END [A-Z ]*PRIVATE KEY-----"
|
|
267
|
+
r"|(?:" + _PEM_SEP + _PEM_UNIT + r")*)"),
|
|
268
|
+
# --- Batch 160, TECHNICAL_DEBT 52: the CURRENT siblings of the vendor
|
|
269
|
+
# --- formats above, each of which the older pattern beside it misses
|
|
270
|
+
# --- by a character class -- a `_` or a second `-` the 2024 shapes
|
|
271
|
+
# --- never had. Measured through the real `redact_output_text` before
|
|
272
|
+
# --- this: every one survived. They refuse inputs too, as `ghp_` and
|
|
273
|
+
# --- `sk-` always have; one vendor list, one vocabulary.
|
|
274
|
+
# GitHub fine-grained PATs, the format `ghp_` is being retired for
|
|
275
|
+
re.compile(r"github_pat_[A-Za-z0-9_]{22,}"),
|
|
276
|
+
# GitHub OAuth, user-to-server, server-to-server and refresh tokens
|
|
277
|
+
re.compile(r"gh[ousr]_[A-Za-z0-9]{36}"),
|
|
278
|
+
# OpenAI project, service-account and admin keys: `sk-proj-...`
|
|
279
|
+
re.compile(r"sk-(?:proj|svcacct|admin)-[A-Za-z0-9_\-]{20,}"),
|
|
280
|
+
# Stripe live secret and restricted keys
|
|
281
|
+
re.compile(r"[sr]k_live_[A-Za-z0-9]{16,}"),
|
|
282
|
+
# npm access tokens
|
|
283
|
+
re.compile(r"npm_[A-Za-z0-9]{36}"),
|
|
284
|
+
# --- ROADMAP_v3 §79 (HD5): the three credentials `/connect` hands the
|
|
285
|
+
# --- harness that no pattern above can see. A sign-in puts them in a
|
|
286
|
+
# --- session the model shares, so they are redacted and refused like
|
|
287
|
+
# --- the rest of the list.
|
|
288
|
+
# Hugging Face user access tokens and OAuth access tokens. An
|
|
289
|
+
# identifier like `hf_hub_download` breaks the run of word characters
|
|
290
|
+
# at its second underscore and is not matched.
|
|
291
|
+
re.compile(r"(?<![A-Za-z0-9])hf_(?:oauth_)?[A-Za-z0-9]{30,}"),
|
|
292
|
+
# OpenRouter keys, which `sk-` above misses at the second hyphen
|
|
293
|
+
re.compile(r"sk-or-v1-[A-Za-z0-9]{32,}"),
|
|
294
|
+
# ModelScope access tokens in their prefixed form: `ms-` and a UUID. A
|
|
295
|
+
# bare UUID is NOT matched -- thread ids are UUIDs -- and neither is one
|
|
296
|
+
# that merely ENDS a word like `items-<uuid>`.
|
|
297
|
+
re.compile(r"(?<![A-Za-z0-9])ms-[0-9a-f]{8}-[0-9a-f]{4}"
|
|
298
|
+
r"-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"),
|
|
224
299
|
]
|
|
225
300
|
|
|
226
301
|
|
|
@@ -246,7 +321,7 @@ def redact_secrets(text: str) -> str:
|
|
|
246
321
|
# ---- Credential shapes (#167, batch 20) ------------------------------------
|
|
247
322
|
# ---------------------------------------------------------------------------
|
|
248
323
|
#
|
|
249
|
-
# The
|
|
324
|
+
# The patterns above are VENDOR TOKENS -- every one has a prefix some
|
|
250
325
|
# company mints, so a match is high-confidence and a bare `[REDACTED]` loses
|
|
251
326
|
# nothing structural. A credential that is just a password in a field has no
|
|
252
327
|
# prefix, and nothing matched it (#167): batch 14 added five build manifests
|
|
@@ -306,9 +381,102 @@ _ASSIGNMENT_RE = re.compile(
|
|
|
306
381
|
r"(?i)\b(password|passwd)\b(\s*(?:[=:][ \t]*|[ \t]))"
|
|
307
382
|
r"([\"'])([^\"'\n]{4,200})\3")
|
|
308
383
|
|
|
384
|
+
# --- Batch 160, TECHNICAL_DEBT 52: three STRUCTURAL shapes, output only by
|
|
385
|
+
# --- owner ruling. Each carries a credential with no vendor prefix, and
|
|
386
|
+
# --- each anchors on structure a security paper does not produce by
|
|
387
|
+
# --- accident, which is the false-positive argument the password shapes
|
|
388
|
+
# --- above already make.
|
|
389
|
+
|
|
390
|
+
# Authorization: Bearer abc... -> Authorization: Bearer [REDACTED]. Anchored
|
|
391
|
+
# on the HEADER NAME, never on the scheme word: "Bearer tokens expire" is
|
|
392
|
+
# prose and must survive, and that is the keyword-widening the comment above
|
|
393
|
+
# refuses. The separator admits a header line, a JSON or dict key (`"..":`)
|
|
394
|
+
# and `=`, so a curl line, a captured request and a logged dict all match.
|
|
395
|
+
# The value's alphabet has no `$ < > { } [`, so `Bearer $TOKEN`,
|
|
396
|
+
# `Bearer <token>` and an already-redacted value never match, and
|
|
397
|
+
# `Digest username=...`, whose secret is not one value, is left alone. A
|
|
398
|
+
# value with NO scheme word must also be 16 characters and carry a digit,
|
|
399
|
+
# because API documentation writes "Authorization: required" and a bare
|
|
400
|
+
# 8-character floor redacted the word.
|
|
401
|
+
# Groups: 1 header name, 2 separator, 3 scheme (or None).
|
|
402
|
+
_AUTH_HEADER_RE = re.compile(
|
|
403
|
+
r"(?i)\b((?:proxy-)?authorization)([\"']?\s*[:=]\s*[\"']?)"
|
|
404
|
+
r"(?:((?:bearer|basic|token)\s+)[A-Za-z0-9._~+/=\-]{8,}"
|
|
405
|
+
r"|(?=[A-Za-z._~+/=\-]*[0-9])[A-Za-z0-9._~+/=\-]{16,})")
|
|
406
|
+
|
|
407
|
+
# A JSON Web Token, whole: three base64url segments, the first two of
|
|
408
|
+
# which open with `eyJ` because both are JSON objects (`{"`). A bearer
|
|
409
|
+
# credential wherever it appears, and the shape `_AUTH_HEADER_RE` cannot
|
|
410
|
+
# reach when the token is pasted without its header.
|
|
411
|
+
_JWT_RE = re.compile(
|
|
412
|
+
r"\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")
|
|
413
|
+
|
|
414
|
+
# The END of a PEM key block whose BEGIN is not in this string: the footer
|
|
415
|
+
# and the run of base64 lines above it. The vendor pattern takes a block
|
|
416
|
+
# from its header; this is the other half, for text a cut started in the
|
|
417
|
+
# middle of a block -- a page, a session's tail as its start sweeps through
|
|
418
|
+
# the output. Found as a literal and walked BACKWARDS in code, because a
|
|
419
|
+
# regex that has to start at the first body line is tried at every line
|
|
420
|
+
# start of a long base64 blob and is quadratic on one with no footer.
|
|
421
|
+
_PEM_END_RE = re.compile(r"-----END [A-Z ]*PRIVATE KEY-----")
|
|
422
|
+
_PEM_BODY_CHARS = frozenset(
|
|
423
|
+
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _before_pem_break(text: str, i: int):
|
|
427
|
+
"""The index before the line break(s) ending at *i*, or None.
|
|
428
|
+
|
|
429
|
+
`_PEM_SEP` read backwards: spaces, then one or more breaks (real, or
|
|
430
|
+
the JSON escape), each with its own spaces."""
|
|
431
|
+
k = i
|
|
432
|
+
while k and text[k - 1] in " \t":
|
|
433
|
+
k -= 1
|
|
434
|
+
seen = False
|
|
435
|
+
while True:
|
|
436
|
+
if k and text[k - 1] == "\n":
|
|
437
|
+
k -= 1
|
|
438
|
+
if k and text[k - 1] == "\r":
|
|
439
|
+
k -= 1
|
|
440
|
+
elif k >= 2 and text[k - 2:k] == "\\n":
|
|
441
|
+
k -= 2
|
|
442
|
+
if k >= 2 and text[k - 2:k] == "\\r":
|
|
443
|
+
k -= 2
|
|
444
|
+
else:
|
|
445
|
+
break
|
|
446
|
+
seen = True
|
|
447
|
+
while k and text[k - 1] in " \t":
|
|
448
|
+
k -= 1
|
|
449
|
+
return k if seen else None
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def _redact_pem_tails(text: str) -> str:
|
|
453
|
+
"""Replace each orphan footer, and the base64 lines above it, whole.
|
|
454
|
+
|
|
455
|
+
Linear: each walk stops at the first character outside the body
|
|
456
|
+
alphabet, and a footer's own `-` is one, so no two walks share text."""
|
|
457
|
+
pieces, last = [], 0
|
|
458
|
+
for m in _PEM_END_RE.finditer(text):
|
|
459
|
+
start = i = m.start()
|
|
460
|
+
while True:
|
|
461
|
+
k = _before_pem_break(text, i)
|
|
462
|
+
if k is None:
|
|
463
|
+
break
|
|
464
|
+
r = k
|
|
465
|
+
while r > last and text[r - 1] in _PEM_BODY_CHARS:
|
|
466
|
+
r -= 1
|
|
467
|
+
if r == k:
|
|
468
|
+
break
|
|
469
|
+
start = i = r
|
|
470
|
+
pieces.append(text[last:start])
|
|
471
|
+
pieces.append(REDACTION_MARKER)
|
|
472
|
+
last = m.end()
|
|
473
|
+
pieces.append(text[last:])
|
|
474
|
+
return "".join(pieces)
|
|
475
|
+
|
|
309
476
|
|
|
310
477
|
def _redact_credential_shapes(text: str) -> str:
|
|
311
|
-
"""Apply the
|
|
478
|
+
"""Apply the context and structural shapes, preserving surrounding
|
|
479
|
+
structure.
|
|
312
480
|
|
|
313
481
|
Each replacer checks the captured VALUE against the placeholder shapes
|
|
314
482
|
before substituting -- a template reference is exactly what docs and
|
|
@@ -329,7 +497,13 @@ def _redact_credential_shapes(text: str) -> str:
|
|
|
329
497
|
return (f"{m.group(1)}{m.group(2)}{m.group(3)}"
|
|
330
498
|
f"{REDACTION_MARKER}{m.group(3)}")
|
|
331
499
|
|
|
332
|
-
|
|
500
|
+
text = _ASSIGNMENT_RE.sub(_assign, text)
|
|
501
|
+
|
|
502
|
+
text = _AUTH_HEADER_RE.sub(
|
|
503
|
+
lambda m: (f"{m.group(1)}{m.group(2)}{m.group(3) or ''}"
|
|
504
|
+
f"{REDACTION_MARKER}"), text)
|
|
505
|
+
text = _JWT_RE.sub(REDACTION_MARKER, text)
|
|
506
|
+
return _redact_pem_tails(text)
|
|
333
507
|
|
|
334
508
|
|
|
335
509
|
def redaction_enabled() -> bool:
|
|
@@ -353,8 +527,13 @@ def redaction_enabled() -> bool:
|
|
|
353
527
|
|
|
354
528
|
What this does NOT govern, whichever way it reads: input refusals
|
|
355
529
|
(_input_leaf -- a denial is legible, not destructive), the depth-cap
|
|
356
|
-
substitution (fail-closed structural bound), and
|
|
357
|
-
|
|
530
|
+
substitution (fail-closed structural bound), and `redact_diagnostic_text`
|
|
531
|
+
-- the harness's own diagnostics, which guard app.log, the transcript,
|
|
532
|
+
`MessageLog.error` and a pipeline record regardless of this answer. That
|
|
533
|
+
last clause used to name `logging_setup` alone and to be true only about
|
|
534
|
+
the moment of reading: the formatter did run regardless, with a strictly
|
|
535
|
+
SMALLER vocabulary than this function's, which is the gap batch 144
|
|
536
|
+
closed. The two now differ in one respect and it is the one named here.
|
|
358
537
|
"""
|
|
359
538
|
active = posture.current()
|
|
360
539
|
if not active.redact_tool_outputs:
|
|
@@ -362,6 +541,71 @@ def redaction_enabled() -> bool:
|
|
|
362
541
|
return not active.redact_off_env
|
|
363
542
|
|
|
364
543
|
|
|
544
|
+
def redact_diagnostic_text(text: str) -> str:
|
|
545
|
+
"""The one redaction path the HARNESS'S OWN DIAGNOSTICS take.
|
|
546
|
+
|
|
547
|
+
THE SAME VOCABULARY AS TOOL OUTPUT, which it was not until batch 144
|
|
548
|
+
(batch 144). `redact_secrets` is the seven vendor-token patterns and
|
|
549
|
+
nothing else. `_redact_credential_shapes` is the only thing in this
|
|
550
|
+
project that matches a credential with NO vendor prefix -- a URL's
|
|
551
|
+
userinfo, a `<password>` element, a quoted `password =` assignment --
|
|
552
|
+
and it had exactly one caller, `redact_output_text`. So one string took
|
|
553
|
+
two different vocabularies depending on which sink it reached. Driven
|
|
554
|
+
through the real `configure_logging` with nothing monkeypatched, on
|
|
555
|
+
issue #167's own seven cases:
|
|
556
|
+
|
|
557
|
+
case app.log transcript tool output
|
|
558
|
+
anthropic key redacted redacted redacted
|
|
559
|
+
openai key redacted redacted redacted
|
|
560
|
+
github pat redacted redacted redacted
|
|
561
|
+
aws access key id redacted redacted redacted
|
|
562
|
+
gradle password LEAKED LEAKED redacted
|
|
563
|
+
maven server password LEAKED LEAKED redacted
|
|
564
|
+
gem source url LEAKED LEAKED redacted
|
|
565
|
+
|
|
566
|
+
The four vendor rows are the control: they prove the formatter was
|
|
567
|
+
attached and firing, so the three LEAKED rows were the vocabulary and
|
|
568
|
+
not a probe that had stopped working. It needed no contrivance to
|
|
569
|
+
reach: `fetch_url` logs the URL it failed on BY DESIGN and says so in
|
|
570
|
+
a comment naming this redaction as the guard, and `core/loop.py` puts
|
|
571
|
+
the same description in `MessageLog.error`, where it persists in
|
|
572
|
+
`app.db` -- which `docs/PRIVACY.md` says is deleted by wiping the
|
|
573
|
+
database rather than one thread.
|
|
574
|
+
|
|
575
|
+
FIVE CALLERS, and the one-caller shape is what made the gap
|
|
576
|
+
survivable: `logging_setup`'s formatter, `tui/app.py`'s
|
|
577
|
+
`TranscriptLogHandler.emit` and its tool-error line, `core/loop.py`'s
|
|
578
|
+
`mark_turn_failed`, and the orchestrator's pipeline event. Adding a
|
|
579
|
+
pattern or a shape now reaches all five, and
|
|
580
|
+
`tests/test_logging_setup.py`'s parity test is what fails if a sixth
|
|
581
|
+
sink composes its own.
|
|
582
|
+
|
|
583
|
+
UNCONDITIONAL, all three passes, and that is not an extension of the
|
|
584
|
+
switch's reach but the removal of a difference. `redaction_enabled()`
|
|
585
|
+
governs what the MODEL is shown, which a user may reasonably choose to
|
|
586
|
+
see raw for one run; it has never governed what `app.log` keeps across
|
|
587
|
+
runs, and the switch's own docstring already said so.
|
|
588
|
+
|
|
589
|
+
LIVE VALUES FIRST, which is the finding rather than a preference
|
|
590
|
+
(batch 144). `redact_live_values` matches a stored value by exact
|
|
591
|
+
substring, so a pattern pass that has already rewritten part of one
|
|
592
|
+
leaves the remainder standing. Measured, with a stored value that
|
|
593
|
+
properly contains a vendor token: the log's old order returned
|
|
594
|
+
`PREFIX-[REDACTED]-SUFFIX` and this one returns `[REDACTED]`. The repo
|
|
595
|
+
has locked the general rule three times -- `fetch_url` redacts before
|
|
596
|
+
it cuts, `param_digest` redacts before truncating, the diff block runs
|
|
597
|
+
on whole texts -- and this is the same rule where the destroyer is a
|
|
598
|
+
substitution rather than a cut.
|
|
599
|
+
|
|
600
|
+
The cost is measured and is per log RECORD: 6.7 us -> 9.4 us on Linux,
|
|
601
|
+
+40%, for three more regex passes over a line that already takes
|
|
602
|
+
seven. Invisible at INFO, and under a millisecond per thousand records
|
|
603
|
+
at DEBUG.
|
|
604
|
+
"""
|
|
605
|
+
return _redact_credential_shapes(
|
|
606
|
+
redact_secrets(harness_secrets.redact_live_values(text)))
|
|
607
|
+
|
|
608
|
+
|
|
365
609
|
def redact_output_text(text: str) -> str:
|
|
366
610
|
"""The one redaction path tool OUTPUT takes, under one switch.
|
|
367
611
|
|
|
@@ -379,10 +623,57 @@ def redact_output_text(text: str) -> str:
|
|
|
379
623
|
redacted LESS than the line it replaced -- the vendor tokens but not
|
|
380
624
|
the credential shapes -- which is a regression dressed as a feature.
|
|
381
625
|
The docstring above already called this "the one path"; it now is one
|
|
382
|
-
for both consumers.
|
|
626
|
+
for both consumers.
|
|
627
|
+
|
|
628
|
+
AND IT IS NOW `redact_diagnostic_text` UNDER THE SWITCH (batch 144),
|
|
629
|
+
rather than its own copy of the same three passes. The two had drifted
|
|
630
|
+
into different vocabularies AND different orders while reading as
|
|
631
|
+
siblings, which is why the difference that is deliberate is now the
|
|
632
|
+
only one there is."""
|
|
633
|
+
# SS61: THE HARNESS'S OWN SECRETS COME OUT FIRST, AND UNCONDITIONALLY.
|
|
634
|
+
# Above the switch on purpose, and still first inside the shared
|
|
635
|
+
# composition below. `redact_tool_outputs` governs pattern GUESSES
|
|
636
|
+
# about other people's credentials, and a user who turns it off has
|
|
637
|
+
# chosen to see those raw. A value this harness put into a subprocess,
|
|
638
|
+
# coming back out of one, is a different thing: passing it to the model
|
|
639
|
+
# is the harness leaking its OWN secret, and no switch asks for that.
|
|
640
|
+
# It sits beside the depth cap, which this module already keeps
|
|
641
|
+
# fail-closed for the same shape of reason -- it is structure and
|
|
642
|
+
# self-inflicted leakage, not content judgment.
|
|
643
|
+
#
|
|
644
|
+
# The VALUES never come here: security/secrets.py does the substitution
|
|
645
|
+
# itself, so exactly one module holds plaintext and the redactor is not
|
|
646
|
+
# it (fix at the producer).
|
|
383
647
|
if not redaction_enabled():
|
|
384
|
-
return text
|
|
385
|
-
return
|
|
648
|
+
return harness_secrets.redact_live_values(text)
|
|
649
|
+
return redact_diagnostic_text(text)
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
#: What a credential is spelled with: base64 and base64url, hex, and the
|
|
653
|
+
#: separators vendor tokens use. A run of these that a cut splits is the
|
|
654
|
+
#: fragment the cut drops, because the redactor runs after the cut and
|
|
655
|
+
#: cannot match half a token. Punctuation that no token contains -- quotes,
|
|
656
|
+
#: `:`, `@`, braces, commas -- ends a run, so a cut through minified JSON
|
|
657
|
+
#: costs one value and not the whole line. Two cuts take it: a shell
|
|
658
|
+
#: session's dropped middle (A72) and an MCP result's bound (A79).
|
|
659
|
+
TOKEN_CHARS = frozenset(
|
|
660
|
+
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-+/=.~")
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
def fragment_end(head: str) -> int:
|
|
664
|
+
"""Where the token-character run that ends *head* begins."""
|
|
665
|
+
i = len(head)
|
|
666
|
+
while i and head[i - 1] in TOKEN_CHARS:
|
|
667
|
+
i -= 1
|
|
668
|
+
return i
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def fragment_start(tail: str) -> int:
|
|
672
|
+
"""Where the token-character run that starts *tail* ends."""
|
|
673
|
+
i = 0
|
|
674
|
+
while i < len(tail) and tail[i] in TOKEN_CHARS:
|
|
675
|
+
i += 1
|
|
676
|
+
return i
|
|
386
677
|
|
|
387
678
|
|
|
388
679
|
|
|
@@ -393,6 +684,28 @@ def redact_output_text(text: str) -> str:
|
|
|
393
684
|
_DIGEST_VALUE_CHARS = 60
|
|
394
685
|
_DIGEST_CHARS = 140
|
|
395
686
|
|
|
687
|
+
# A VALUE CARRYING A URL GETS ITS OWN, WIDER CAP, and the digest a wider
|
|
688
|
+
# total to hold one (batch 96).
|
|
689
|
+
#
|
|
690
|
+
# Batch 65 made a truncated URL clickable by carrying the whole one
|
|
691
|
+
# alongside the line, so the click has always opened the right page. What
|
|
692
|
+
# it could not fix is the TERMINAL: `https://host/a/b…` minus the ellipsis
|
|
693
|
+
# is itself a syntactically valid URL, so a terminal's own link detector
|
|
694
|
+
# sees a shorter address than the harness resolved and ctrl+click opens
|
|
695
|
+
# THAT -- measured, and reported as the click going somewhere wrong.
|
|
696
|
+
#
|
|
697
|
+
# The honest fix is to stop producing the mismatch: draw the whole URL, so
|
|
698
|
+
# the visible text is the target again and the two layers cannot disagree.
|
|
699
|
+
# `markdown.link_spans`' elision stays for anything past this cap, where
|
|
700
|
+
# the invariant is still "the visible text tells you the origin".
|
|
701
|
+
#
|
|
702
|
+
# BOTH CAPS MOVE OR NEITHER DOES. A 200-character value cut at a
|
|
703
|
+
# 140-character TOTAL is the same mismatch one line further down, which is
|
|
704
|
+
# why the total is raised only for a digest that actually holds a URL --
|
|
705
|
+
# every other digest keeps the width it has always had.
|
|
706
|
+
_DIGEST_URL_CHARS = 200
|
|
707
|
+
_DIGEST_URL_TOTAL = 240
|
|
708
|
+
|
|
396
709
|
|
|
397
710
|
def param_digest(params, omit=()) -> str:
|
|
398
711
|
"""A short, REDACTED description of what a tool was called with.
|
|
@@ -437,6 +750,7 @@ def param_digest(params, omit=()) -> str:
|
|
|
437
750
|
if not isinstance(params, dict) or not params:
|
|
438
751
|
return ""
|
|
439
752
|
parts = []
|
|
753
|
+
widened = False
|
|
440
754
|
for key, value in params.items():
|
|
441
755
|
if key in omit:
|
|
442
756
|
continue
|
|
@@ -448,12 +762,34 @@ def param_digest(params, omit=()) -> str:
|
|
|
448
762
|
# like every other pattern substitution; it stays display-only
|
|
449
763
|
# and never logs, whatever the switch reads.
|
|
450
764
|
text = redact_output_text(text)
|
|
451
|
-
|
|
452
|
-
|
|
765
|
+
# ONE LINE, and the collapse happens BEFORE the truncation so that
|
|
766
|
+
# sixty characters means sixty VISIBLE ones (batch 96). A digest is
|
|
767
|
+
# drawn as a single row -- `▸ shell <digest>` -- and nothing
|
|
768
|
+
# downstream re-flows it, so a multi-line command (a heredoc, an
|
|
769
|
+
# inline `python3 - <<EOF`) drew its remaining lines at column 0
|
|
770
|
+
# underneath, outside the indent and outside the tool's line. The
|
|
771
|
+
# transcript, `core/replay.py` and the research pass trace all read
|
|
772
|
+
# this one producer, so all three drew it.
|
|
773
|
+
#
|
|
774
|
+
# AFTER the redaction, which is the ordering this function exists
|
|
775
|
+
# to hold: the redactor works on whole strings, and collapsing
|
|
776
|
+
# first would not change what it matches, but doing it after keeps
|
|
777
|
+
# the rule "redact, THEN shorten" true of every step below it.
|
|
778
|
+
text = " ".join(text.split())
|
|
779
|
+
# `http` is a FILTER, not a grammar: which substrings are URLs is
|
|
780
|
+
# `tui/markdown.py`'s question, asked there against one definition.
|
|
781
|
+
# This only declines to widen a value that cannot contain one --
|
|
782
|
+
# the same test, and the same reasoning, as `redacted_values`.
|
|
783
|
+
has_url = "http" in text
|
|
784
|
+
widened = widened or has_url
|
|
785
|
+
cap = _DIGEST_URL_CHARS if has_url else _DIGEST_VALUE_CHARS
|
|
786
|
+
if len(text) > cap:
|
|
787
|
+
text = text[:cap - 1] + "…"
|
|
453
788
|
parts.append(text)
|
|
454
789
|
digest = " ".join(p for p in parts if p)
|
|
455
|
-
if
|
|
456
|
-
|
|
790
|
+
total = _DIGEST_URL_TOTAL if widened else _DIGEST_CHARS
|
|
791
|
+
if len(digest) > total:
|
|
792
|
+
digest = digest[:total - 1] + "…"
|
|
457
793
|
return digest
|
|
458
794
|
|
|
459
795
|
|