@yahyahammad/venastine 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2030 -2002
- package/LICENSE +201 -201
- package/README.md +1034 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +528 -528
- package/bin/venastine.py +71 -71
- package/cli_commands.py +838 -835
- package/config.yaml +13 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +971 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2715 -2715
- package/core/commands.py +460 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +234 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +458 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +108 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +733 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +295 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +267 -267
- package/tools/builtin/arxiv.py +681 -681
- package/tools/builtin/fetch_url.py +371 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +328 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1242 -1242
- package/tui/app.py +8318 -8307
- package/tui/app.tcss +747 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3813 -3796
package/logging_setup.py
CHANGED
|
@@ -1,295 +1,295 @@
|
|
|
1
|
-
"""logging_setup.py
|
|
2
|
-
|
|
3
|
-
Call configure_logging() ONCE, early, from the entry point (main.py once
|
|
4
|
-
ROADMAP.md §1's CLI lands), before any module that might log. Not called
|
|
5
|
-
on import, so tests and library-style usage don't get surprise handlers.
|
|
6
|
-
|
|
7
|
-
Diverges from ROADMAP.md §2's literal "stream=sys.stderr" decision in two
|
|
8
|
-
narrow ways, by explicit product decision:
|
|
9
|
-
1. Also writes to a rotating file (logs/app.log by default) so tool
|
|
10
|
-
retries survive CLI exit and process close.
|
|
11
|
-
2. Level is env-overridable via AGENT_LOG_LEVEL instead of a hardcoded
|
|
12
|
-
logging.INFO default. AGENT_LOG_FILE overrides the file path.
|
|
13
|
-
|
|
14
|
-
Idempotent on repeated calls so test runs that invoke configure_logging
|
|
15
|
-
more than once don't stack duplicate handlers and double-emit lines.
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
import logging
|
|
19
|
-
import os
|
|
20
|
-
import sys
|
|
21
|
-
from logging.handlers import RotatingFileHandler
|
|
22
|
-
|
|
23
|
-
# Top-level, not inside format(): this runs on every log record. There is
|
|
24
|
-
# no cycle to dodge, and the reason is the direction rather than a leaf:
|
|
25
|
-
# `safety/policy_enforcement.py` imports `security.posture` and
|
|
26
|
-
# `security.secrets` and nothing else from this project, the edge runs
|
|
27
|
-
# safety -> security and never the reverse, and batch 37 kept THIS module
|
|
28
|
-
# out of `security/` precisely so it could sit above both. The sentence
|
|
29
|
-
# here used to say policy_enforcement was "a leaf (stdlib imports only,
|
|
30
|
-
# nothing from this project)", which stopped being true when value
|
|
31
|
-
# redaction landed; batch 144 corrected the sentence, not the import.
|
|
32
|
-
#
|
|
33
|
-
# ONE NAME, not two. The formatter used to compose
|
|
34
|
-
# `redact_live_values(redact_secrets(...))` itself, which was both a
|
|
35
|
-
# smaller vocabulary than tool output's and the opposite order from the
|
|
36
|
-
# other site that composed the same two passes (batch 144).
|
|
37
|
-
from safety.policy_enforcement import redact_diagnostic_text
|
|
38
|
-
|
|
39
|
-
__all__ = ["configure_logging"]
|
|
40
|
-
|
|
41
|
-
_DEFAULT_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
|
|
42
|
-
_DEFAULT_DATEFMT = "%Y-%m-%d %H:%M:%S"
|
|
43
|
-
|
|
44
|
-
_DEFAULT_LOG_FILE = "logs/app.log"
|
|
45
|
-
_DEFAULT_LEVEL = "INFO"
|
|
46
|
-
|
|
47
|
-
# Rotation defaults live here, not in config.py: this file owns the
|
|
48
|
-
# logging concern end-to-end (mirrors how ROADMAP.md §2 keeps the whole
|
|
49
|
-
# logging subsystem in one file). Bump via the function signature if
|
|
50
|
-
# needed; there is no project-wide reason to surface them as config yet.
|
|
51
|
-
_DEFAULT_MAX_BYTES = 1_000_000
|
|
52
|
-
_DEFAULT_BACKUP_COUNT = 3
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
class _RedactingFormatter(logging.Formatter):
|
|
56
|
-
"""Secret redaction on the LOG, which was the fourth sink (#132).
|
|
57
|
-
|
|
58
|
-
README enumerates three: "before it reaches the model, the transcript
|
|
59
|
-
or the database". The log is a fourth, and it was never an
|
|
60
|
-
acknowledged exception -- just an unlisted one.
|
|
61
|
-
|
|
62
|
-
`registry.dispatch()` states this exact threat in the comment above
|
|
63
|
-
its redaction call: "an exception message routinely carries the
|
|
64
|
-
request that produced it, and for an HTTP client that means a URL
|
|
65
|
-
with an API key in the query string. Redacting only the success path
|
|
66
|
-
would make a failing tool the way secrets escape." It then calls
|
|
67
|
-
`logger.exception(...)` THIRTEEN LINES EARLIER, so the unredacted
|
|
68
|
-
message and traceback were already on disk by the time
|
|
69
|
-
check_output_policy saw the result. Same string, two sinks, one
|
|
70
|
-
redacted.
|
|
71
|
-
|
|
72
|
-
A FORMATTER, not a logging.Filter, and that is load-bearing.
|
|
73
|
-
`logger.exception` renders the traceback at format time, from
|
|
74
|
-
`record.exc_info` -- and the traceback is precisely what carries the
|
|
75
|
-
upstream URL. A filter rewriting `record.msg` never sees it.
|
|
76
|
-
Formatting the record first and redacting the finished string covers
|
|
77
|
-
the message, the arguments and the traceback in one pass.
|
|
78
|
-
|
|
79
|
-
Fixed here rather than at dispatch's call site because dispatch is
|
|
80
|
-
not the only site: `fetch_url` logs a warning naming the URL it
|
|
81
|
-
failed on, and any future `logger.*` inherits this for free. Fixing
|
|
82
|
-
the one site the issue measured would have left the class open --
|
|
83
|
-
the same producer-vs-consumer argument #47 makes one file over.
|
|
84
|
-
"""
|
|
85
|
-
|
|
86
|
-
def format(self, record):
|
|
87
|
-
# SS61, on the sink this docstring calls the fourth one. The
|
|
88
|
-
# harness's OWN secrets come out unconditionally and FIRST, which
|
|
89
|
-
# is `redact_diagnostic_text`'s order and no longer this line's
|
|
90
|
-
# choice: a value this process put into a subprocess and then
|
|
91
|
-
# logged is a leak nobody opted into, and app.log is the sink that
|
|
92
|
-
# KEEPS it across runs.
|
|
93
|
-
#
|
|
94
|
-
# THE SAME VOCABULARY AS TOOL OUTPUT since batch 144.
|
|
95
|
-
# This composed `redact_secrets` alone -- the seven vendor tokens
|
|
96
|
-
# -- so a quoted `password =` assignment, a `<password>` element
|
|
97
|
-
# and a URL's userinfo all reached the file in cleartext while the
|
|
98
|
-
# identical string was redacted on its way to the model. Driven,
|
|
99
|
-
# four of issue #167's seven cases were redacted here and three
|
|
100
|
-
# were not; the four are the control that proves this formatter
|
|
101
|
-
# was attached and firing while it happened.
|
|
102
|
-
#
|
|
103
|
-
# Belt and braces rather than redundancy: nothing is supposed to
|
|
104
|
-
# log a secret at all, `Secret` has no `__str__` to print, and
|
|
105
|
-
# `lock()` names its reason rather than quoting anything. This
|
|
106
|
-
# catches the traceback, which renders values nobody chose to
|
|
107
|
-
# format -- exactly the case this formatter exists for, and the
|
|
108
|
-
# case `fetch_url` relies on when it logs the URL it failed on.
|
|
109
|
-
return redact_diagnostic_text(super().format(record))
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
class _OwnerOnlyRotatingFileHandler(RotatingFileHandler):
|
|
113
|
-
"""`RotatingFileHandler`, except the file is created 0600 (batch 144).
|
|
114
|
-
|
|
115
|
-
`RotatingFileHandler` takes no mode, so `logs/app.log` was created
|
|
116
|
-
under the process umask: measured 0o644 on WSL at `umask 022`, against
|
|
117
|
-
`providers.json` and `secrets.json` at 0o600, every one of which this
|
|
118
|
-
project argues for at length. Every other account on the machine could
|
|
119
|
-
read a file that holds, per `docs/PRIVACY.md`, "INFO and above by
|
|
120
|
-
default ... including provider/HTTP traces ... may still contain query
|
|
121
|
-
text" -- and, until the vocabulary was fixed in this same batch, a third
|
|
122
|
-
party's cleartext password whenever one arrived inside a URL.
|
|
123
|
-
|
|
124
|
-
The project has already filed this exact shape at this exact severity
|
|
125
|
-
for the file one directory over. Issue #19 was "providers.json is
|
|
126
|
-
written with no explicit mode, so plaintext API keys land 0644", and
|
|
127
|
-
its argument survives verbatim in `credentials.py`.
|
|
128
|
-
|
|
129
|
-
0600 AT CREATION, never `open()` then `os.chmod` -- the rule
|
|
130
|
-
`credentials.py` states and the trap it names, which is that a
|
|
131
|
-
post-write chmod leaves a window in which the file is on disk under
|
|
132
|
-
the umask. `_open` is the one place to do it because ROTATION creates
|
|
133
|
-
files too: `doRollover` renames `app.log` to `app.log.1` and then
|
|
134
|
-
calls `_open` for a fresh one, so a mode applied once at construction
|
|
135
|
-
would be correct for one file out of four. A rename carries the mode,
|
|
136
|
-
so the three backups inherit 0600 from the file they used to be.
|
|
137
|
-
|
|
138
|
-
AN EXISTING LOOSE FILE IS LEFT ALONE, deliberately, and this is the
|
|
139
|
-
one place this handler differs from `credentials.py`'s sibling
|
|
140
|
-
(`test_rewriting_an_existing_file_repairs_a_loose_mode` pins that the
|
|
141
|
-
provider store REPAIRS a 0644 file on its next write). The log's path
|
|
142
|
-
is not the harness's to decide: `configure_logging` records that
|
|
143
|
-
"the user can still point AGENT_LOG_FILE anywhere they want", so a
|
|
144
|
-
file at that path may be one the user made group-readable on purpose,
|
|
145
|
-
and silently tightening it would be this module overruling them. The
|
|
146
|
-
existing file ages out on its own instead -- it becomes `.1`, then
|
|
147
|
-
`.2`, `.3`, and is deleted -- while everything created from here on is
|
|
148
|
-
0600.
|
|
149
|
-
|
|
150
|
-
Windows is a no-op, exactly as `credentials.py` records for its
|
|
151
|
-
sibling: `os.open`'s mode there controls only the read-only flag.
|
|
152
|
-
"""
|
|
153
|
-
|
|
154
|
-
def _open(self):
|
|
155
|
-
# The flags are derived rather than hardcoded so the mode argument
|
|
156
|
-
# cannot quietly disagree with `self.mode`. In practice this is
|
|
157
|
-
# always append -- `RotatingFileHandler` forces "a" whenever
|
|
158
|
-
# `maxBytes` is set, which `configure_logging` always sets.
|
|
159
|
-
flags = os.O_WRONLY | os.O_CREAT
|
|
160
|
-
flags |= os.O_TRUNC if "w" in self.mode else os.O_APPEND
|
|
161
|
-
fd = os.open(self.baseFilename, flags, 0o600)
|
|
162
|
-
try:
|
|
163
|
-
return os.fdopen(fd, self.mode, encoding=self.encoding,
|
|
164
|
-
errors=self.errors)
|
|
165
|
-
except BaseException:
|
|
166
|
-
# `os.fdopen` takes ownership on success only, so a failure
|
|
167
|
-
# here would leak the descriptor for the life of the process.
|
|
168
|
-
os.close(fd)
|
|
169
|
-
raise
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
def _resolve_level(level):
|
|
173
|
-
"""Accepts a logging level as int, a name string ("INFO"), a numeric
|
|
174
|
-
string ("20"), or None (in which case AGENT_LOG_LEVEL / the default
|
|
175
|
-
is consulted)."""
|
|
176
|
-
from_env = level is None
|
|
177
|
-
if from_env:
|
|
178
|
-
level = os.environ.get("AGENT_LOG_LEVEL", _DEFAULT_LEVEL)
|
|
179
|
-
if isinstance(level, str):
|
|
180
|
-
# First try a named level ("INFO" -> logging.INFO); fall back to
|
|
181
|
-
# parsing as an int ("20").
|
|
182
|
-
named = getattr(logging, level.upper(), None)
|
|
183
|
-
if isinstance(named, int):
|
|
184
|
-
return named
|
|
185
|
-
try:
|
|
186
|
-
return int(level)
|
|
187
|
-
except ValueError:
|
|
188
|
-
# A typo in AGENT_LOG_LEVEL warns and falls back, as an
|
|
189
|
-
# unwritable AGENT_LOG_FILE does below (batch 219, #212). It
|
|
190
|
-
# used to raise, and this is called before anything can log:
|
|
191
|
-
# `verbose` made startup one traceback, from the subsystem that
|
|
192
|
-
# would have reported the mistake legibly. A bad level handed in
|
|
193
|
-
# by a CALLER still raises, because that is a bug in the code.
|
|
194
|
-
if not from_env:
|
|
195
|
-
raise
|
|
196
|
-
print(
|
|
197
|
-
f"Warning: AGENT_LOG_LEVEL={level!r} is not a logging level "
|
|
198
|
-
f"(use DEBUG, INFO, WARNING, ERROR, CRITICAL or a number); "
|
|
199
|
-
f"logging at {_DEFAULT_LEVEL}.",
|
|
200
|
-
file=sys.stderr,
|
|
201
|
-
)
|
|
202
|
-
return getattr(logging, _DEFAULT_LEVEL)
|
|
203
|
-
return int(level)
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
def configure_logging(
|
|
207
|
-
level=None,
|
|
208
|
-
*,
|
|
209
|
-
log_file=None,
|
|
210
|
-
max_bytes=_DEFAULT_MAX_BYTES,
|
|
211
|
-
backup_count=_DEFAULT_BACKUP_COUNT,
|
|
212
|
-
stderr=True,
|
|
213
|
-
) -> None:
|
|
214
|
-
"""Configure the root logger with a stderr handler and a rotating
|
|
215
|
-
file handler.
|
|
216
|
-
|
|
217
|
-
Safe to call more than once: each call clears existing root handlers
|
|
218
|
-
before re-attaching, so repeated invocations (e.g. across tests)
|
|
219
|
-
don't stack duplicates and double-emit lines.
|
|
220
|
-
|
|
221
|
-
Parameters
|
|
222
|
-
----------
|
|
223
|
-
level:
|
|
224
|
-
Logging level as int, name ("INFO"), numeric string ("20"), or
|
|
225
|
-
None. None falls back to the AGENT_LOG_LEVEL env var, then the
|
|
226
|
-
INFO default.
|
|
227
|
-
log_file:
|
|
228
|
-
Path to the rotating log file. None falls back to the
|
|
229
|
-
AGENT_LOG_FILE env var, then "logs/app.log". Parent directory is
|
|
230
|
-
created if missing.
|
|
231
|
-
max_bytes, backup_count:
|
|
232
|
-
RotatingFileHandler rotation parameters.
|
|
233
|
-
stderr:
|
|
234
|
-
Attach the stderr handler. FALSE FOR THE TUI, and this is not a
|
|
235
|
-
preference. Textual owns the terminal and repaints it; anything
|
|
236
|
-
written to stderr lands on top of the rendered screen, sits over
|
|
237
|
-
the input bar and behind the panels, and disappears at the next
|
|
238
|
-
repaint -- so the message is both corrupting and unreadable. The
|
|
239
|
-
rotating file keeps everything at full detail either way, and
|
|
240
|
-
tui/app.py additionally routes WARNING+ into the transcript so
|
|
241
|
-
nothing that matters is only in a file.
|
|
242
|
-
"""
|
|
243
|
-
resolved_level = _resolve_level(level)
|
|
244
|
-
formatter = _RedactingFormatter(_DEFAULT_FORMAT, datefmt=_DEFAULT_DATEFMT)
|
|
245
|
-
|
|
246
|
-
root = logging.getLogger()
|
|
247
|
-
root.setLevel(resolved_level)
|
|
248
|
-
|
|
249
|
-
# Clear any pre-existing handlers so a repeated call does not attach
|
|
250
|
-
# a second stderr and a second file handler on top of the first.
|
|
251
|
-
for handler in list(root.handlers):
|
|
252
|
-
root.removeHandler(handler)
|
|
253
|
-
try:
|
|
254
|
-
handler.close()
|
|
255
|
-
except OSError:
|
|
256
|
-
# Closing should not fail loudly, but if a handler is in a
|
|
257
|
-
# weird state we still want to proceed with fresh handlers.
|
|
258
|
-
pass
|
|
259
|
-
|
|
260
|
-
if stderr:
|
|
261
|
-
stderr_handler = logging.StreamHandler(stream=sys.stderr)
|
|
262
|
-
stderr_handler.setFormatter(formatter)
|
|
263
|
-
root.addHandler(stderr_handler)
|
|
264
|
-
|
|
265
|
-
if log_file is None:
|
|
266
|
-
log_file = os.environ.get("AGENT_LOG_FILE", _DEFAULT_LOG_FILE)
|
|
267
|
-
# Normalize relative paths and embedded ../ so makedirs and
|
|
268
|
-
# RotatingFileHandler see a clean absolute path. Not a containment
|
|
269
|
-
# check -- the user can still point AGENT_LOG_FILE anywhere they
|
|
270
|
-
# want; this just prevents accidental path-traversal via relative
|
|
271
|
-
# segments in env-configured paths.
|
|
272
|
-
log_file = os.path.abspath(log_file)
|
|
273
|
-
log_dir = os.path.dirname(log_file)
|
|
274
|
-
try:
|
|
275
|
-
if log_dir and not os.path.isdir(log_dir):
|
|
276
|
-
os.makedirs(log_dir, exist_ok=True)
|
|
277
|
-
|
|
278
|
-
file_handler = _OwnerOnlyRotatingFileHandler(
|
|
279
|
-
log_file,
|
|
280
|
-
maxBytes=max_bytes,
|
|
281
|
-
backupCount=backup_count,
|
|
282
|
-
encoding="utf-8",
|
|
283
|
-
)
|
|
284
|
-
file_handler.setFormatter(formatter)
|
|
285
|
-
root.addHandler(file_handler)
|
|
286
|
-
except OSError as e:
|
|
287
|
-
# Logging to a file is non-essential. If the directory can't be
|
|
288
|
-
# created or the file can't be opened (read-only filesystem,
|
|
289
|
-
# permission denied, disk full), fall back to stderr-only rather
|
|
290
|
-
# than crashing the program at startup.
|
|
291
|
-
print(
|
|
292
|
-
f"Warning: could not create log file at {log_file} ({e}); "
|
|
293
|
-
f"logging to stderr only.",
|
|
294
|
-
file=sys.stderr,
|
|
295
|
-
)
|
|
1
|
+
"""logging_setup.py
|
|
2
|
+
|
|
3
|
+
Call configure_logging() ONCE, early, from the entry point (main.py once
|
|
4
|
+
ROADMAP.md §1's CLI lands), before any module that might log. Not called
|
|
5
|
+
on import, so tests and library-style usage don't get surprise handlers.
|
|
6
|
+
|
|
7
|
+
Diverges from ROADMAP.md §2's literal "stream=sys.stderr" decision in two
|
|
8
|
+
narrow ways, by explicit product decision:
|
|
9
|
+
1. Also writes to a rotating file (logs/app.log by default) so tool
|
|
10
|
+
retries survive CLI exit and process close.
|
|
11
|
+
2. Level is env-overridable via AGENT_LOG_LEVEL instead of a hardcoded
|
|
12
|
+
logging.INFO default. AGENT_LOG_FILE overrides the file path.
|
|
13
|
+
|
|
14
|
+
Idempotent on repeated calls so test runs that invoke configure_logging
|
|
15
|
+
more than once don't stack duplicate handlers and double-emit lines.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import logging
|
|
19
|
+
import os
|
|
20
|
+
import sys
|
|
21
|
+
from logging.handlers import RotatingFileHandler
|
|
22
|
+
|
|
23
|
+
# Top-level, not inside format(): this runs on every log record. There is
|
|
24
|
+
# no cycle to dodge, and the reason is the direction rather than a leaf:
|
|
25
|
+
# `safety/policy_enforcement.py` imports `security.posture` and
|
|
26
|
+
# `security.secrets` and nothing else from this project, the edge runs
|
|
27
|
+
# safety -> security and never the reverse, and batch 37 kept THIS module
|
|
28
|
+
# out of `security/` precisely so it could sit above both. The sentence
|
|
29
|
+
# here used to say policy_enforcement was "a leaf (stdlib imports only,
|
|
30
|
+
# nothing from this project)", which stopped being true when value
|
|
31
|
+
# redaction landed; batch 144 corrected the sentence, not the import.
|
|
32
|
+
#
|
|
33
|
+
# ONE NAME, not two. The formatter used to compose
|
|
34
|
+
# `redact_live_values(redact_secrets(...))` itself, which was both a
|
|
35
|
+
# smaller vocabulary than tool output's and the opposite order from the
|
|
36
|
+
# other site that composed the same two passes (batch 144).
|
|
37
|
+
from safety.policy_enforcement import redact_diagnostic_text
|
|
38
|
+
|
|
39
|
+
__all__ = ["configure_logging"]
|
|
40
|
+
|
|
41
|
+
_DEFAULT_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
|
|
42
|
+
_DEFAULT_DATEFMT = "%Y-%m-%d %H:%M:%S"
|
|
43
|
+
|
|
44
|
+
_DEFAULT_LOG_FILE = "logs/app.log"
|
|
45
|
+
_DEFAULT_LEVEL = "INFO"
|
|
46
|
+
|
|
47
|
+
# Rotation defaults live here, not in config.py: this file owns the
|
|
48
|
+
# logging concern end-to-end (mirrors how ROADMAP.md §2 keeps the whole
|
|
49
|
+
# logging subsystem in one file). Bump via the function signature if
|
|
50
|
+
# needed; there is no project-wide reason to surface them as config yet.
|
|
51
|
+
_DEFAULT_MAX_BYTES = 1_000_000
|
|
52
|
+
_DEFAULT_BACKUP_COUNT = 3
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class _RedactingFormatter(logging.Formatter):
|
|
56
|
+
"""Secret redaction on the LOG, which was the fourth sink (#132).
|
|
57
|
+
|
|
58
|
+
README enumerates three: "before it reaches the model, the transcript
|
|
59
|
+
or the database". The log is a fourth, and it was never an
|
|
60
|
+
acknowledged exception -- just an unlisted one.
|
|
61
|
+
|
|
62
|
+
`registry.dispatch()` states this exact threat in the comment above
|
|
63
|
+
its redaction call: "an exception message routinely carries the
|
|
64
|
+
request that produced it, and for an HTTP client that means a URL
|
|
65
|
+
with an API key in the query string. Redacting only the success path
|
|
66
|
+
would make a failing tool the way secrets escape." It then calls
|
|
67
|
+
`logger.exception(...)` THIRTEEN LINES EARLIER, so the unredacted
|
|
68
|
+
message and traceback were already on disk by the time
|
|
69
|
+
check_output_policy saw the result. Same string, two sinks, one
|
|
70
|
+
redacted.
|
|
71
|
+
|
|
72
|
+
A FORMATTER, not a logging.Filter, and that is load-bearing.
|
|
73
|
+
`logger.exception` renders the traceback at format time, from
|
|
74
|
+
`record.exc_info` -- and the traceback is precisely what carries the
|
|
75
|
+
upstream URL. A filter rewriting `record.msg` never sees it.
|
|
76
|
+
Formatting the record first and redacting the finished string covers
|
|
77
|
+
the message, the arguments and the traceback in one pass.
|
|
78
|
+
|
|
79
|
+
Fixed here rather than at dispatch's call site because dispatch is
|
|
80
|
+
not the only site: `fetch_url` logs a warning naming the URL it
|
|
81
|
+
failed on, and any future `logger.*` inherits this for free. Fixing
|
|
82
|
+
the one site the issue measured would have left the class open --
|
|
83
|
+
the same producer-vs-consumer argument #47 makes one file over.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def format(self, record):
|
|
87
|
+
# SS61, on the sink this docstring calls the fourth one. The
|
|
88
|
+
# harness's OWN secrets come out unconditionally and FIRST, which
|
|
89
|
+
# is `redact_diagnostic_text`'s order and no longer this line's
|
|
90
|
+
# choice: a value this process put into a subprocess and then
|
|
91
|
+
# logged is a leak nobody opted into, and app.log is the sink that
|
|
92
|
+
# KEEPS it across runs.
|
|
93
|
+
#
|
|
94
|
+
# THE SAME VOCABULARY AS TOOL OUTPUT since batch 144.
|
|
95
|
+
# This composed `redact_secrets` alone -- the seven vendor tokens
|
|
96
|
+
# -- so a quoted `password =` assignment, a `<password>` element
|
|
97
|
+
# and a URL's userinfo all reached the file in cleartext while the
|
|
98
|
+
# identical string was redacted on its way to the model. Driven,
|
|
99
|
+
# four of issue #167's seven cases were redacted here and three
|
|
100
|
+
# were not; the four are the control that proves this formatter
|
|
101
|
+
# was attached and firing while it happened.
|
|
102
|
+
#
|
|
103
|
+
# Belt and braces rather than redundancy: nothing is supposed to
|
|
104
|
+
# log a secret at all, `Secret` has no `__str__` to print, and
|
|
105
|
+
# `lock()` names its reason rather than quoting anything. This
|
|
106
|
+
# catches the traceback, which renders values nobody chose to
|
|
107
|
+
# format -- exactly the case this formatter exists for, and the
|
|
108
|
+
# case `fetch_url` relies on when it logs the URL it failed on.
|
|
109
|
+
return redact_diagnostic_text(super().format(record))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class _OwnerOnlyRotatingFileHandler(RotatingFileHandler):
|
|
113
|
+
"""`RotatingFileHandler`, except the file is created 0600 (batch 144).
|
|
114
|
+
|
|
115
|
+
`RotatingFileHandler` takes no mode, so `logs/app.log` was created
|
|
116
|
+
under the process umask: measured 0o644 on WSL at `umask 022`, against
|
|
117
|
+
`providers.json` and `secrets.json` at 0o600, every one of which this
|
|
118
|
+
project argues for at length. Every other account on the machine could
|
|
119
|
+
read a file that holds, per `docs/PRIVACY.md`, "INFO and above by
|
|
120
|
+
default ... including provider/HTTP traces ... may still contain query
|
|
121
|
+
text" -- and, until the vocabulary was fixed in this same batch, a third
|
|
122
|
+
party's cleartext password whenever one arrived inside a URL.
|
|
123
|
+
|
|
124
|
+
The project has already filed this exact shape at this exact severity
|
|
125
|
+
for the file one directory over. Issue #19 was "providers.json is
|
|
126
|
+
written with no explicit mode, so plaintext API keys land 0644", and
|
|
127
|
+
its argument survives verbatim in `credentials.py`.
|
|
128
|
+
|
|
129
|
+
0600 AT CREATION, never `open()` then `os.chmod` -- the rule
|
|
130
|
+
`credentials.py` states and the trap it names, which is that a
|
|
131
|
+
post-write chmod leaves a window in which the file is on disk under
|
|
132
|
+
the umask. `_open` is the one place to do it because ROTATION creates
|
|
133
|
+
files too: `doRollover` renames `app.log` to `app.log.1` and then
|
|
134
|
+
calls `_open` for a fresh one, so a mode applied once at construction
|
|
135
|
+
would be correct for one file out of four. A rename carries the mode,
|
|
136
|
+
so the three backups inherit 0600 from the file they used to be.
|
|
137
|
+
|
|
138
|
+
AN EXISTING LOOSE FILE IS LEFT ALONE, deliberately, and this is the
|
|
139
|
+
one place this handler differs from `credentials.py`'s sibling
|
|
140
|
+
(`test_rewriting_an_existing_file_repairs_a_loose_mode` pins that the
|
|
141
|
+
provider store REPAIRS a 0644 file on its next write). The log's path
|
|
142
|
+
is not the harness's to decide: `configure_logging` records that
|
|
143
|
+
"the user can still point AGENT_LOG_FILE anywhere they want", so a
|
|
144
|
+
file at that path may be one the user made group-readable on purpose,
|
|
145
|
+
and silently tightening it would be this module overruling them. The
|
|
146
|
+
existing file ages out on its own instead -- it becomes `.1`, then
|
|
147
|
+
`.2`, `.3`, and is deleted -- while everything created from here on is
|
|
148
|
+
0600.
|
|
149
|
+
|
|
150
|
+
Windows is a no-op, exactly as `credentials.py` records for its
|
|
151
|
+
sibling: `os.open`'s mode there controls only the read-only flag.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
def _open(self):
|
|
155
|
+
# The flags are derived rather than hardcoded so the mode argument
|
|
156
|
+
# cannot quietly disagree with `self.mode`. In practice this is
|
|
157
|
+
# always append -- `RotatingFileHandler` forces "a" whenever
|
|
158
|
+
# `maxBytes` is set, which `configure_logging` always sets.
|
|
159
|
+
flags = os.O_WRONLY | os.O_CREAT
|
|
160
|
+
flags |= os.O_TRUNC if "w" in self.mode else os.O_APPEND
|
|
161
|
+
fd = os.open(self.baseFilename, flags, 0o600)
|
|
162
|
+
try:
|
|
163
|
+
return os.fdopen(fd, self.mode, encoding=self.encoding,
|
|
164
|
+
errors=self.errors)
|
|
165
|
+
except BaseException:
|
|
166
|
+
# `os.fdopen` takes ownership on success only, so a failure
|
|
167
|
+
# here would leak the descriptor for the life of the process.
|
|
168
|
+
os.close(fd)
|
|
169
|
+
raise
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _resolve_level(level):
|
|
173
|
+
"""Accepts a logging level as int, a name string ("INFO"), a numeric
|
|
174
|
+
string ("20"), or None (in which case AGENT_LOG_LEVEL / the default
|
|
175
|
+
is consulted)."""
|
|
176
|
+
from_env = level is None
|
|
177
|
+
if from_env:
|
|
178
|
+
level = os.environ.get("AGENT_LOG_LEVEL", _DEFAULT_LEVEL)
|
|
179
|
+
if isinstance(level, str):
|
|
180
|
+
# First try a named level ("INFO" -> logging.INFO); fall back to
|
|
181
|
+
# parsing as an int ("20").
|
|
182
|
+
named = getattr(logging, level.upper(), None)
|
|
183
|
+
if isinstance(named, int):
|
|
184
|
+
return named
|
|
185
|
+
try:
|
|
186
|
+
return int(level)
|
|
187
|
+
except ValueError:
|
|
188
|
+
# A typo in AGENT_LOG_LEVEL warns and falls back, as an
|
|
189
|
+
# unwritable AGENT_LOG_FILE does below (batch 219, #212). It
|
|
190
|
+
# used to raise, and this is called before anything can log:
|
|
191
|
+
# `verbose` made startup one traceback, from the subsystem that
|
|
192
|
+
# would have reported the mistake legibly. A bad level handed in
|
|
193
|
+
# by a CALLER still raises, because that is a bug in the code.
|
|
194
|
+
if not from_env:
|
|
195
|
+
raise
|
|
196
|
+
print(
|
|
197
|
+
f"Warning: AGENT_LOG_LEVEL={level!r} is not a logging level "
|
|
198
|
+
f"(use DEBUG, INFO, WARNING, ERROR, CRITICAL or a number); "
|
|
199
|
+
f"logging at {_DEFAULT_LEVEL}.",
|
|
200
|
+
file=sys.stderr,
|
|
201
|
+
)
|
|
202
|
+
return getattr(logging, _DEFAULT_LEVEL)
|
|
203
|
+
return int(level)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def configure_logging(
|
|
207
|
+
level=None,
|
|
208
|
+
*,
|
|
209
|
+
log_file=None,
|
|
210
|
+
max_bytes=_DEFAULT_MAX_BYTES,
|
|
211
|
+
backup_count=_DEFAULT_BACKUP_COUNT,
|
|
212
|
+
stderr=True,
|
|
213
|
+
) -> None:
|
|
214
|
+
"""Configure the root logger with a stderr handler and a rotating
|
|
215
|
+
file handler.
|
|
216
|
+
|
|
217
|
+
Safe to call more than once: each call clears existing root handlers
|
|
218
|
+
before re-attaching, so repeated invocations (e.g. across tests)
|
|
219
|
+
don't stack duplicates and double-emit lines.
|
|
220
|
+
|
|
221
|
+
Parameters
|
|
222
|
+
----------
|
|
223
|
+
level:
|
|
224
|
+
Logging level as int, name ("INFO"), numeric string ("20"), or
|
|
225
|
+
None. None falls back to the AGENT_LOG_LEVEL env var, then the
|
|
226
|
+
INFO default.
|
|
227
|
+
log_file:
|
|
228
|
+
Path to the rotating log file. None falls back to the
|
|
229
|
+
AGENT_LOG_FILE env var, then "logs/app.log". Parent directory is
|
|
230
|
+
created if missing.
|
|
231
|
+
max_bytes, backup_count:
|
|
232
|
+
RotatingFileHandler rotation parameters.
|
|
233
|
+
stderr:
|
|
234
|
+
Attach the stderr handler. FALSE FOR THE TUI, and this is not a
|
|
235
|
+
preference. Textual owns the terminal and repaints it; anything
|
|
236
|
+
written to stderr lands on top of the rendered screen, sits over
|
|
237
|
+
the input bar and behind the panels, and disappears at the next
|
|
238
|
+
repaint -- so the message is both corrupting and unreadable. The
|
|
239
|
+
rotating file keeps everything at full detail either way, and
|
|
240
|
+
tui/app.py additionally routes WARNING+ into the transcript so
|
|
241
|
+
nothing that matters is only in a file.
|
|
242
|
+
"""
|
|
243
|
+
resolved_level = _resolve_level(level)
|
|
244
|
+
formatter = _RedactingFormatter(_DEFAULT_FORMAT, datefmt=_DEFAULT_DATEFMT)
|
|
245
|
+
|
|
246
|
+
root = logging.getLogger()
|
|
247
|
+
root.setLevel(resolved_level)
|
|
248
|
+
|
|
249
|
+
# Clear any pre-existing handlers so a repeated call does not attach
|
|
250
|
+
# a second stderr and a second file handler on top of the first.
|
|
251
|
+
for handler in list(root.handlers):
|
|
252
|
+
root.removeHandler(handler)
|
|
253
|
+
try:
|
|
254
|
+
handler.close()
|
|
255
|
+
except OSError:
|
|
256
|
+
# Closing should not fail loudly, but if a handler is in a
|
|
257
|
+
# weird state we still want to proceed with fresh handlers.
|
|
258
|
+
pass
|
|
259
|
+
|
|
260
|
+
if stderr:
|
|
261
|
+
stderr_handler = logging.StreamHandler(stream=sys.stderr)
|
|
262
|
+
stderr_handler.setFormatter(formatter)
|
|
263
|
+
root.addHandler(stderr_handler)
|
|
264
|
+
|
|
265
|
+
if log_file is None:
|
|
266
|
+
log_file = os.environ.get("AGENT_LOG_FILE", _DEFAULT_LOG_FILE)
|
|
267
|
+
# Normalize relative paths and embedded ../ so makedirs and
|
|
268
|
+
# RotatingFileHandler see a clean absolute path. Not a containment
|
|
269
|
+
# check -- the user can still point AGENT_LOG_FILE anywhere they
|
|
270
|
+
# want; this just prevents accidental path-traversal via relative
|
|
271
|
+
# segments in env-configured paths.
|
|
272
|
+
log_file = os.path.abspath(log_file)
|
|
273
|
+
log_dir = os.path.dirname(log_file)
|
|
274
|
+
try:
|
|
275
|
+
if log_dir and not os.path.isdir(log_dir):
|
|
276
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
277
|
+
|
|
278
|
+
file_handler = _OwnerOnlyRotatingFileHandler(
|
|
279
|
+
log_file,
|
|
280
|
+
maxBytes=max_bytes,
|
|
281
|
+
backupCount=backup_count,
|
|
282
|
+
encoding="utf-8",
|
|
283
|
+
)
|
|
284
|
+
file_handler.setFormatter(formatter)
|
|
285
|
+
root.addHandler(file_handler)
|
|
286
|
+
except OSError as e:
|
|
287
|
+
# Logging to a file is non-essential. If the directory can't be
|
|
288
|
+
# created or the file can't be opened (read-only filesystem,
|
|
289
|
+
# permission denied, disk full), fall back to stderr-only rather
|
|
290
|
+
# than crashing the program at startup.
|
|
291
|
+
print(
|
|
292
|
+
f"Warning: could not create log file at {log_file} ({e}); "
|
|
293
|
+
f"logging to stderr only.",
|
|
294
|
+
file=sys.stderr,
|
|
295
|
+
)
|