@yahyahammad/venastine 0.1.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +249 -93
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +280 -23
- package/agents/tui_commands.py +31 -21
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -0
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1249 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +1735 -959
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1353
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +315 -167
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1672 -198
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +21 -14
- package/package.json +303 -73
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +149 -100
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +1082 -682
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +1338 -1005
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +374 -29
- package/tui/app.py +5281 -397
- package/tui/app.tcss +290 -9
- package/tui/commands.py +26 -61
- package/tui/diffs.py +286 -265
- package/tui/history.py +128 -0
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -0
- package/tui/meters.py +364 -0
- package/tui/screens.py +346 -92
- package/tui/themes.py +599 -366
- package/tui/widgets.py +2673 -116
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/json_store.py
CHANGED
|
@@ -3,6 +3,22 @@ json_store.py
|
|
|
3
3
|
|
|
4
4
|
The mechanics four user-tier JSON stores had four copies of.
|
|
5
5
|
|
|
6
|
+
AND THE FILE TOOLS' TEXT WRITES SINCE BATCH 146, which is why
|
|
7
|
+
`_replace_through_temp` exists under both public writers rather than the
|
|
8
|
+
dance living inside `write_json_atomic`. `write_text_atomic` says what it
|
|
9
|
+
needs the dance FOR, and it is not durability: `os.replace` replaces a name
|
|
10
|
+
rather than following it, which is the only thing that stops a write through
|
|
11
|
+
a hardlinked document name landing on the file the link shares. A sixth copy
|
|
12
|
+
of a temp-and-rename would have been the obvious way to get that, and this
|
|
13
|
+
module exists because four copies of it had already drifted.
|
|
14
|
+
|
|
15
|
+
AND THE PROJECT LOADER'S READS SINCE BATCH 219. `read_regular_text` is the
|
|
16
|
+
read half of the same kind of argument: a cloned project's `.venastine/`
|
|
17
|
+
files and root `AGENTS.md` are read by the trust prompt before anyone has
|
|
18
|
+
consented to them, and a skill symlinked to `/dev/zero` was read until
|
|
19
|
+
memory ran out. Six read sites in two packages needed the one guard, which
|
|
20
|
+
is the shape this module was made for.
|
|
21
|
+
|
|
6
22
|
WHAT THIS IS AND IS NOT. It owns *how* a small JSON file is read and
|
|
7
23
|
written -- the temp-file-plus-replace dance, the version guard, the
|
|
8
24
|
fail-soft read -- and nothing about what any of them MEAN. Each store keeps
|
|
@@ -44,14 +60,30 @@ Each caller keeps the policy it already had.
|
|
|
44
60
|
|
|
45
61
|
from __future__ import annotations
|
|
46
62
|
|
|
63
|
+
import errno
|
|
47
64
|
import json
|
|
48
65
|
import logging
|
|
49
66
|
import os
|
|
67
|
+
import stat
|
|
50
68
|
from typing import Optional
|
|
51
69
|
|
|
52
70
|
logger = logging.getLogger(__name__)
|
|
53
71
|
|
|
54
72
|
|
|
73
|
+
class DestinationInUse(PermissionError):
|
|
74
|
+
"""Windows refused the rename because another program has the file open.
|
|
75
|
+
|
|
76
|
+
Batch 219 (#242). This and a destination that is genuinely not writable
|
|
77
|
+
both arrive from `os.replace` as `PermissionError` WinError 5, and the
|
|
78
|
+
remedies are opposite: close it in the other program, or you may not
|
|
79
|
+
write here. Measured on Windows: a destination another process holds
|
|
80
|
+
open for reading still opens for `r+b` from here, and a read-only one
|
|
81
|
+
does not -- so `_held_open_elsewhere` tells them apart without writing a
|
|
82
|
+
byte. A subclass, so every `except OSError` that already reports a
|
|
83
|
+
failed write reports this one too, now saying what to do.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
|
|
55
87
|
def write_json_atomic(path: str, payload, *, mode: Optional[int] = None,
|
|
56
88
|
indent: int = 2, sort_keys: bool = False,
|
|
57
89
|
trailing_newline: bool = False) -> None:
|
|
@@ -63,10 +95,18 @@ def write_json_atomic(path: str, payload, *, mode: Optional[int] = None,
|
|
|
63
95
|
`mode` sets permissions on the TEMP FILE, not on the destination (#19).
|
|
64
96
|
`os.replace` carries the source file's mode across, so setting it
|
|
65
97
|
afterwards would be both a no-op and a lie, and setting it on the
|
|
66
|
-
destination beforehand is worse -- replace overwrites that too.
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
98
|
+
destination beforehand is worse -- replace overwrites that too.
|
|
99
|
+
|
|
100
|
+
EVERY CALLER HOLDING A CREDENTIAL PASSES ONE, and two of them pass it
|
|
101
|
+
for the reason the trust store explicitly does NOT: `credentials.py` and
|
|
102
|
+
`security/secrets.py` hold secrets, and 0600-at-creation is the property
|
|
103
|
+
their own comments argue for at length. `mcp_client/config.py` is the
|
|
104
|
+
fourth, for the `Authorization` header batch 140 named in
|
|
105
|
+
`.venastine/mcp.json`. This used to say "THREE CALLERS PASS ONE" and was written
|
|
106
|
+
when three did -- a count in prose with nothing able to notice the
|
|
107
|
+
fourth, which is the shape batch 143 closed twice on another surface and
|
|
108
|
+
answered the same way: the argument never needed the number, so it is
|
|
109
|
+
gone rather than corrected (batch 144).
|
|
70
110
|
|
|
71
111
|
`trailing_newline` is off by default so every existing store writes
|
|
72
112
|
the same bytes it always did. The four here are machine state nobody
|
|
@@ -75,22 +115,202 @@ def write_json_atomic(path: str, payload, *, mode: Optional[int] = None,
|
|
|
75
115
|
committed file with no final newline carries git's "No newline at end
|
|
76
116
|
of file" marker in every diff of it, forever after.
|
|
77
117
|
"""
|
|
78
|
-
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
79
|
-
tmp = f"{path}.tmp"
|
|
80
|
-
|
|
81
118
|
def _dump(handle) -> None:
|
|
82
119
|
json.dump(payload, handle, indent=indent, sort_keys=sort_keys)
|
|
83
120
|
if trailing_newline:
|
|
84
121
|
handle.write("\n")
|
|
85
122
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
123
|
+
# newline=None, which is what this has always passed and therefore what
|
|
124
|
+
# every stored file already holds: `json.dump` with an indent writes
|
|
125
|
+
# "\n" between its lines and the stream translates them, so on Windows
|
|
126
|
+
# these stores are CRLF on disk. Not a property worth having, and not
|
|
127
|
+
# one worth changing either -- it would rewrite the bytes of four user
|
|
128
|
+
# -tier stores to no end. The text caller below wants the opposite and
|
|
129
|
+
# says so there.
|
|
130
|
+
_replace_through_temp(path, _dump, mode=mode, newline=None)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def write_text_atomic(path: str, text: str, *, newline: str = "") -> None:
|
|
134
|
+
"""Write `text` to `path`, atomically and byte-for-byte. Raises OSError.
|
|
135
|
+
|
|
136
|
+
THE FIFTH CONSUMER OF THE DANCE ABOVE, and the first that is not a JSON
|
|
137
|
+
store (batch 146). The file tools reach it for two reasons that are not
|
|
138
|
+
the ones the JSON stores have:
|
|
139
|
+
|
|
140
|
+
`newline=""` by default, because a tool that was handed content must
|
|
141
|
+
write the content it was handed. `open(path, "w")` translates every
|
|
142
|
+
"\n" to `os.linesep`, so on Windows `write` could not reproduce bytes
|
|
143
|
+
it had just read, and one `write_project_doc` call against an LF
|
|
144
|
+
document rewrote every line ending in it -- 2,910 of them in this
|
|
145
|
+
repository's own AGENTS.md -- under an approval that had been shown a
|
|
146
|
+
diff of one line.
|
|
147
|
+
|
|
148
|
+
And REPLACES RATHER THAN FOLLOWS, which is the property `.venastine/`
|
|
149
|
+
depends on. `os.path.realpath` cannot see a hardlink -- a hardlink is
|
|
150
|
+
not a link to anything, it IS the file -- so a containment test on the
|
|
151
|
+
resolved path admits a write through a hardlinked name, and a `write`
|
|
152
|
+
through one put `{"mcpServers": ...}` into a protected `settings.json`
|
|
153
|
+
whose direct spelling is refused. `os.replace` onto the name breaks the
|
|
154
|
+
link instead: the other name keeps its bytes and this one becomes a new
|
|
155
|
+
file. Measured both ways round.
|
|
156
|
+
|
|
157
|
+
The cost, stated because the caller cannot see it from here: a
|
|
158
|
+
destination another process merely has OPEN fails the rename on Windows,
|
|
159
|
+
where a truncating write would have succeeded. That is the same trade
|
|
160
|
+
`write_json_atomic` documents above, and the same answer -- a write that
|
|
161
|
+
fails loudly beats one that half-succeeds.
|
|
162
|
+
"""
|
|
163
|
+
_replace_through_temp(path, lambda handle: handle.write(text),
|
|
164
|
+
newline=newline)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _replace_through_temp(path: str, write_body, *,
|
|
168
|
+
mode=None, newline=None) -> None:
|
|
169
|
+
"""Write through a unique temp file and `os.replace` it onto `path`.
|
|
170
|
+
|
|
171
|
+
ONE COPY OF THE MECHANICS, which is this module's whole reason for
|
|
172
|
+
existing -- `write_body` is handed an open text stream and writes
|
|
173
|
+
whatever the caller means; everything else here is the dance, and every
|
|
174
|
+
paragraph below was written against a failure that actually happened.
|
|
175
|
+
|
|
176
|
+
GUARDED, because `os.makedirs("")` raises rather than doing nothing
|
|
177
|
+
and `os.path.dirname` returns "" for a bare filename. No caller could
|
|
178
|
+
reach it while all six built an absolute path, but `AGENT_SECRETS_FILE`
|
|
179
|
+
and `AGENT_PROVIDERS_FILE` are documented overrides a user may set to
|
|
180
|
+
one -- so the two secret stores joining this helper (batch 108) is what
|
|
181
|
+
made a path SHAPE able to decide whether the write works.
|
|
182
|
+
"""
|
|
183
|
+
parent = os.path.dirname(path)
|
|
184
|
+
if parent:
|
|
185
|
+
os.makedirs(parent, exist_ok=True)
|
|
186
|
+
|
|
187
|
+
# A UNIQUE NAME, not a fixed `<path>.tmp` sibling (batch 144).
|
|
188
|
+
# `config_edit.write` reached this conclusion for `config.yaml` and
|
|
189
|
+
# wrote down why -- "two writers raced on the one name, and a failed
|
|
190
|
+
# `os.replace` left the leftover in the install tree forever" -- and
|
|
191
|
+
# this helper, extracted so these mechanics would have exactly ONE
|
|
192
|
+
# copy, had neither half of the answer. One question, two answers, and
|
|
193
|
+
# nothing that could tell you they disagreed.
|
|
194
|
+
#
|
|
195
|
+
# Two instances in one directory is a supported configuration:
|
|
196
|
+
# `docs/PRIVACY.md` says "everything is relative to the directory you
|
|
197
|
+
# run in".
|
|
198
|
+
tmp = "%s.%s.tmp" % (path, os.urandom(4).hex())
|
|
199
|
+
|
|
200
|
+
# THE TEMP FILE IS SWEPT ON FAILURE (batch 108). Without this a write
|
|
201
|
+
# that dies inside `_dump` leaves its temp file behind for good: the
|
|
202
|
+
# destination is correctly untouched, which is the whole point, but the
|
|
203
|
+
# debris is not. It is worst for the two stores that joined this helper
|
|
204
|
+
# in the same batch -- a half-written SEALED ENVELOPE sitting next to
|
|
205
|
+
# the secrets file, and a partial provider file next to the real one.
|
|
206
|
+
#
|
|
207
|
+
# AND THE RENAME IS INSIDE IT (batch 144). `os.replace` used to
|
|
208
|
+
# be the statement after this handler, so the sweep covered the failure
|
|
209
|
+
# the comment above names and not the one next to it -- which is the
|
|
210
|
+
# repository's own prompt for defect shape 6: "what actually raises
|
|
211
|
+
# this, and what raises the thing next to it?"
|
|
212
|
+
#
|
|
213
|
+
# The rename's failure is not hypothetical and not symmetrical across
|
|
214
|
+
# platforms. On Windows a plain `open()` takes no FILE_SHARE_DELETE, so
|
|
215
|
+
# any other process merely READING the destination makes `os.replace`
|
|
216
|
+
# fail -- an editor, a sync client, a backup, a virus scanner, or a
|
|
217
|
+
# second instance in the same directory. Driven, with a happy-path
|
|
218
|
+
# control: `PermissionError WinError 5`, destination correctly intact,
|
|
219
|
+
# and `providers.json.tmp` left holding EVERY stored provider key in
|
|
220
|
+
# cleartext, because `save_credentials` reads the whole file, mutates
|
|
221
|
+
# one entry and writes all of it back. The POSIX arm is clean -- rename
|
|
222
|
+
# there ignores open handles and the destination's mode -- and the
|
|
223
|
+
# process-death window between the write and the rename is on both.
|
|
224
|
+
#
|
|
225
|
+
# Hygiene, never error policy: the original exception is re-raised
|
|
226
|
+
# unchanged, because "errors belong to the caller" above is the rule
|
|
227
|
+
# and a cleanup that swallowed one would be the opposite of it. The
|
|
228
|
+
# unlink is itself guarded, so a failure to tidy up cannot replace the
|
|
229
|
+
# failure worth reporting. The one rewording (batch 219) keeps that
|
|
230
|
+
# rule: a held-open destination is still a PermissionError, chained to
|
|
231
|
+
# the original, and only its message changes -- from one that reads as
|
|
232
|
+
# a permissions problem to the remedy, which is closing the file.
|
|
233
|
+
#
|
|
234
|
+
# ONE `os.open` rather than two branches, and `0o666` is not a new
|
|
235
|
+
# policy: it is the mode `open(path, "w")` passes, so the umask
|
|
236
|
+
# subtracts from it exactly as before. Measured on WSL at `umask 022`,
|
|
237
|
+
# both forms give 0o644 -- while `os.open` with NO mode argument gives
|
|
238
|
+
# 0o755, which is why the argument is explicit rather than omitted.
|
|
239
|
+
# `O_EXCL` on a unique name is what makes the sweep below safe: the
|
|
240
|
+
# file this unlinks is always one this call created, never another
|
|
241
|
+
# writer's.
|
|
242
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
243
|
+
fd = os.open(tmp, flags, 0o666 if mode is None else mode)
|
|
244
|
+
try:
|
|
245
|
+
with os.fdopen(fd, "w", encoding="utf-8", newline=newline) as f:
|
|
246
|
+
write_body(f)
|
|
247
|
+
os.replace(tmp, path)
|
|
248
|
+
except BaseException as exc:
|
|
249
|
+
try:
|
|
250
|
+
os.unlink(tmp)
|
|
251
|
+
except OSError:
|
|
252
|
+
pass # the temp file may not exist; the original error is what is raised
|
|
253
|
+
if _held_open_elsewhere(path, exc):
|
|
254
|
+
# No path in the message: every caller prefixes its own ("Could
|
|
255
|
+
# not write <path>: ..."), and a path here printed it twice.
|
|
256
|
+
raise DestinationInUse(
|
|
257
|
+
"it is open in another program (an editor, a sync client, a "
|
|
258
|
+
"backup or a virus scanner), so Windows will not replace it. "
|
|
259
|
+
"Close it there and try again.") from exc
|
|
260
|
+
raise
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _held_open_elsewhere(path: str, exc: BaseException) -> bool:
|
|
264
|
+
"""Whether a failed rename onto `path` was another process's open handle.
|
|
265
|
+
|
|
266
|
+
Windows only: a POSIX rename ignores open handles. WinError 32 is a
|
|
267
|
+
sharing violation and says so itself; WinError 5 is ambiguous, and the
|
|
268
|
+
`r+b` open settles it (see `DestinationInUse`). The probe writes
|
|
269
|
+
nothing, and a destination that has gone or is a directory fails it.
|
|
270
|
+
"""
|
|
271
|
+
if os.name != "nt" or not isinstance(exc, PermissionError):
|
|
272
|
+
return False
|
|
273
|
+
if getattr(exc, "winerror", None) == 32:
|
|
274
|
+
return True
|
|
275
|
+
try:
|
|
276
|
+
with open(path, "r+b"):
|
|
277
|
+
return True
|
|
278
|
+
except OSError as probe:
|
|
279
|
+
return getattr(probe, "winerror", None) == 32
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def read_regular_text(path: str, encoding: str = "utf-8-sig") -> str:
|
|
283
|
+
"""The whole text of `path`, which must be a regular file. Raises OSError.
|
|
284
|
+
|
|
285
|
+
For content a cloned project brought with it (batch 219, #214), which
|
|
286
|
+
the trust prompt reads BEFORE anyone has consented to it. Git stores a
|
|
287
|
+
symlink, `open()` follows one, and `/dev/zero`, `/dev/urandom` and
|
|
288
|
+
`/dev/tty` exist on every POSIX machine: measured, the listing read a
|
|
289
|
+
skill linked to `/dev/zero` until a 1 GiB cap raised MemoryError,
|
|
290
|
+
while the trust hash -- which skips non-regular files for exactly this
|
|
291
|
+
reason -- stepped over it. A FIFO is the other half: `open()` on one
|
|
292
|
+
blocks until a writer appears.
|
|
293
|
+
|
|
294
|
+
So the file is opened NON-BLOCKING and checked on the open descriptor.
|
|
295
|
+
A FIFO's open then returns at once, and `fstat` rather than a `stat` of
|
|
296
|
+
the name leaves no window for the name to be swapped between the check
|
|
297
|
+
and the read. A non-regular file raises OSError, which is what every
|
|
298
|
+
reader of project content already reports or skips. `O_BINARY` stops
|
|
299
|
+
Windows' C runtime translating the bytes under the decoder, which
|
|
300
|
+
`open()` would have done for itself.
|
|
301
|
+
"""
|
|
302
|
+
flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)
|
|
303
|
+
fd = os.open(path, flags)
|
|
304
|
+
try:
|
|
305
|
+
regular = stat.S_ISREG(os.fstat(fd).st_mode)
|
|
306
|
+
except BaseException:
|
|
307
|
+
os.close(fd)
|
|
308
|
+
raise
|
|
309
|
+
if not regular:
|
|
310
|
+
os.close(fd)
|
|
311
|
+
raise OSError(errno.EINVAL, "not a regular file", path)
|
|
312
|
+
with open(fd, encoding=encoding) as f:
|
|
313
|
+
return f.read()
|
|
94
314
|
|
|
95
315
|
|
|
96
316
|
def read_versioned(path: str, version: int, fallback_note: str = "") -> dict:
|
|
@@ -112,7 +332,7 @@ def read_versioned(path: str, version: int, fallback_note: str = "") -> dict:
|
|
|
112
332
|
if not os.path.exists(path):
|
|
113
333
|
return {}
|
|
114
334
|
try:
|
|
115
|
-
with open(path,
|
|
335
|
+
with open(path, encoding="utf-8-sig") as f:
|
|
116
336
|
data = json.load(f)
|
|
117
337
|
except (OSError, ValueError):
|
|
118
338
|
logger.debug("Could not read %s; %s", path,
|
package/logging_setup.py
CHANGED
|
@@ -20,10 +20,21 @@ import os
|
|
|
20
20
|
import sys
|
|
21
21
|
from logging.handlers import RotatingFileHandler
|
|
22
22
|
|
|
23
|
-
# Top-level, not inside format(): this runs on every log record
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
|
|
23
|
+
# Top-level, not inside format(): this runs on every log record. There is
|
|
24
|
+
# no cycle to dodge, and the reason is the direction rather than a leaf:
|
|
25
|
+
# `safety/policy_enforcement.py` imports `security.posture` and
|
|
26
|
+
# `security.secrets` and nothing else from this project, the edge runs
|
|
27
|
+
# safety -> security and never the reverse, and batch 37 kept THIS module
|
|
28
|
+
# out of `security/` precisely so it could sit above both. The sentence
|
|
29
|
+
# here used to say policy_enforcement was "a leaf (stdlib imports only,
|
|
30
|
+
# nothing from this project)", which stopped being true when value
|
|
31
|
+
# redaction landed; batch 144 corrected the sentence, not the import.
|
|
32
|
+
#
|
|
33
|
+
# ONE NAME, not two. The formatter used to compose
|
|
34
|
+
# `redact_live_values(redact_secrets(...))` itself, which was both a
|
|
35
|
+
# smaller vocabulary than tool output's and the opposite order from the
|
|
36
|
+
# other site that composed the same two passes (batch 144).
|
|
37
|
+
from safety.policy_enforcement import redact_diagnostic_text
|
|
27
38
|
|
|
28
39
|
__all__ = ["configure_logging"]
|
|
29
40
|
|
|
@@ -73,23 +84,122 @@ class _RedactingFormatter(logging.Formatter):
|
|
|
73
84
|
"""
|
|
74
85
|
|
|
75
86
|
def format(self, record):
|
|
76
|
-
|
|
87
|
+
# SS61, on the sink this docstring calls the fourth one. The
|
|
88
|
+
# harness's OWN secrets come out unconditionally and FIRST, which
|
|
89
|
+
# is `redact_diagnostic_text`'s order and no longer this line's
|
|
90
|
+
# choice: a value this process put into a subprocess and then
|
|
91
|
+
# logged is a leak nobody opted into, and app.log is the sink that
|
|
92
|
+
# KEEPS it across runs.
|
|
93
|
+
#
|
|
94
|
+
# THE SAME VOCABULARY AS TOOL OUTPUT since batch 144.
|
|
95
|
+
# This composed `redact_secrets` alone -- the seven vendor tokens
|
|
96
|
+
# -- so a quoted `password =` assignment, a `<password>` element
|
|
97
|
+
# and a URL's userinfo all reached the file in cleartext while the
|
|
98
|
+
# identical string was redacted on its way to the model. Driven,
|
|
99
|
+
# four of issue #167's seven cases were redacted here and three
|
|
100
|
+
# were not; the four are the control that proves this formatter
|
|
101
|
+
# was attached and firing while it happened.
|
|
102
|
+
#
|
|
103
|
+
# Belt and braces rather than redundancy: nothing is supposed to
|
|
104
|
+
# log a secret at all, `Secret` has no `__str__` to print, and
|
|
105
|
+
# `lock()` names its reason rather than quoting anything. This
|
|
106
|
+
# catches the traceback, which renders values nobody chose to
|
|
107
|
+
# format -- exactly the case this formatter exists for, and the
|
|
108
|
+
# case `fetch_url` relies on when it logs the URL it failed on.
|
|
109
|
+
return redact_diagnostic_text(super().format(record))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class _OwnerOnlyRotatingFileHandler(RotatingFileHandler):
|
|
113
|
+
"""`RotatingFileHandler`, except the file is created 0600 (batch 144).
|
|
114
|
+
|
|
115
|
+
`RotatingFileHandler` takes no mode, so `logs/app.log` was created
|
|
116
|
+
under the process umask: measured 0o644 on WSL at `umask 022`, against
|
|
117
|
+
`providers.json` and `secrets.json` at 0o600, every one of which this
|
|
118
|
+
project argues for at length. Every other account on the machine could
|
|
119
|
+
read a file that holds, per `docs/PRIVACY.md`, "INFO and above by
|
|
120
|
+
default ... including provider/HTTP traces ... may still contain query
|
|
121
|
+
text" -- and, until the vocabulary was fixed in this same batch, a third
|
|
122
|
+
party's cleartext password whenever one arrived inside a URL.
|
|
123
|
+
|
|
124
|
+
The project has already filed this exact shape at this exact severity
|
|
125
|
+
for the file one directory over. Issue #19 was "providers.json is
|
|
126
|
+
written with no explicit mode, so plaintext API keys land 0644", and
|
|
127
|
+
its argument survives verbatim in `credentials.py`.
|
|
128
|
+
|
|
129
|
+
0600 AT CREATION, never `open()` then `os.chmod` -- the rule
|
|
130
|
+
`credentials.py` states and the trap it names, which is that a
|
|
131
|
+
post-write chmod leaves a window in which the file is on disk under
|
|
132
|
+
the umask. `_open` is the one place to do it because ROTATION creates
|
|
133
|
+
files too: `doRollover` renames `app.log` to `app.log.1` and then
|
|
134
|
+
calls `_open` for a fresh one, so a mode applied once at construction
|
|
135
|
+
would be correct for one file out of four. A rename carries the mode,
|
|
136
|
+
so the three backups inherit 0600 from the file they used to be.
|
|
137
|
+
|
|
138
|
+
AN EXISTING LOOSE FILE IS LEFT ALONE, deliberately, and this is the
|
|
139
|
+
one place this handler differs from `credentials.py`'s sibling
|
|
140
|
+
(`test_rewriting_an_existing_file_repairs_a_loose_mode` pins that the
|
|
141
|
+
provider store REPAIRS a 0644 file on its next write). The log's path
|
|
142
|
+
is not the harness's to decide: `configure_logging` records that
|
|
143
|
+
"the user can still point AGENT_LOG_FILE anywhere they want", so a
|
|
144
|
+
file at that path may be one the user made group-readable on purpose,
|
|
145
|
+
and silently tightening it would be this module overruling them. The
|
|
146
|
+
existing file ages out on its own instead -- it becomes `.1`, then
|
|
147
|
+
`.2`, `.3`, and is deleted -- while everything created from here on is
|
|
148
|
+
0600.
|
|
149
|
+
|
|
150
|
+
Windows is a no-op, exactly as `credentials.py` records for its
|
|
151
|
+
sibling: `os.open`'s mode there controls only the read-only flag.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
def _open(self):
|
|
155
|
+
# The flags are derived rather than hardcoded so the mode argument
|
|
156
|
+
# cannot quietly disagree with `self.mode`. In practice this is
|
|
157
|
+
# always append -- `RotatingFileHandler` forces "a" whenever
|
|
158
|
+
# `maxBytes` is set, which `configure_logging` always sets.
|
|
159
|
+
flags = os.O_WRONLY | os.O_CREAT
|
|
160
|
+
flags |= os.O_TRUNC if "w" in self.mode else os.O_APPEND
|
|
161
|
+
fd = os.open(self.baseFilename, flags, 0o600)
|
|
162
|
+
try:
|
|
163
|
+
return os.fdopen(fd, self.mode, encoding=self.encoding,
|
|
164
|
+
errors=self.errors)
|
|
165
|
+
except BaseException:
|
|
166
|
+
# `os.fdopen` takes ownership on success only, so a failure
|
|
167
|
+
# here would leak the descriptor for the life of the process.
|
|
168
|
+
os.close(fd)
|
|
169
|
+
raise
|
|
77
170
|
|
|
78
171
|
|
|
79
172
|
def _resolve_level(level):
|
|
80
173
|
"""Accepts a logging level as int, a name string ("INFO"), a numeric
|
|
81
174
|
string ("20"), or None (in which case AGENT_LOG_LEVEL / the default
|
|
82
175
|
is consulted)."""
|
|
83
|
-
|
|
176
|
+
from_env = level is None
|
|
177
|
+
if from_env:
|
|
84
178
|
level = os.environ.get("AGENT_LOG_LEVEL", _DEFAULT_LEVEL)
|
|
85
179
|
if isinstance(level, str):
|
|
86
180
|
# First try a named level ("INFO" -> logging.INFO); fall back to
|
|
87
|
-
# parsing as an int ("20").
|
|
88
|
-
# is the right failure mode for a misconfigured env var.
|
|
181
|
+
# parsing as an int ("20").
|
|
89
182
|
named = getattr(logging, level.upper(), None)
|
|
90
183
|
if isinstance(named, int):
|
|
91
184
|
return named
|
|
92
|
-
|
|
185
|
+
try:
|
|
186
|
+
return int(level)
|
|
187
|
+
except ValueError:
|
|
188
|
+
# A typo in AGENT_LOG_LEVEL warns and falls back, as an
|
|
189
|
+
# unwritable AGENT_LOG_FILE does below (batch 219, #212). It
|
|
190
|
+
# used to raise, and this is called before anything can log:
|
|
191
|
+
# `verbose` made startup one traceback, from the subsystem that
|
|
192
|
+
# would have reported the mistake legibly. A bad level handed in
|
|
193
|
+
# by a CALLER still raises, because that is a bug in the code.
|
|
194
|
+
if not from_env:
|
|
195
|
+
raise
|
|
196
|
+
print(
|
|
197
|
+
f"Warning: AGENT_LOG_LEVEL={level!r} is not a logging level "
|
|
198
|
+
f"(use DEBUG, INFO, WARNING, ERROR, CRITICAL or a number); "
|
|
199
|
+
f"logging at {_DEFAULT_LEVEL}.",
|
|
200
|
+
file=sys.stderr,
|
|
201
|
+
)
|
|
202
|
+
return getattr(logging, _DEFAULT_LEVEL)
|
|
93
203
|
return int(level)
|
|
94
204
|
|
|
95
205
|
|
|
@@ -142,7 +252,7 @@ def configure_logging(
|
|
|
142
252
|
root.removeHandler(handler)
|
|
143
253
|
try:
|
|
144
254
|
handler.close()
|
|
145
|
-
except
|
|
255
|
+
except OSError:
|
|
146
256
|
# Closing should not fail loudly, but if a handler is in a
|
|
147
257
|
# weird state we still want to proceed with fresh handlers.
|
|
148
258
|
pass
|
|
@@ -165,7 +275,7 @@ def configure_logging(
|
|
|
165
275
|
if log_dir and not os.path.isdir(log_dir):
|
|
166
276
|
os.makedirs(log_dir, exist_ok=True)
|
|
167
277
|
|
|
168
|
-
file_handler =
|
|
278
|
+
file_handler = _OwnerOnlyRotatingFileHandler(
|
|
169
279
|
log_file,
|
|
170
280
|
maxBytes=max_bytes,
|
|
171
281
|
backupCount=backup_count,
|