@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -0,0 +1,855 @@
1
+ """
2
+ security/secrets.py
3
+
4
+ ROADMAP_v3 §49 slice 5a (SS56-SS62). The ONE place this harness holds a
5
+ secret it did not receive from the user at the moment of use.
6
+
7
+ WHY THIS EXISTS
8
+ ---------------
9
+ Slice 4 shipped SSH holding no secret at all: SS47 chose key-and-agent auth
10
+ precisely so there was nothing to protect, and an encrypted key with no
11
+ agent simply FAILED. That is a fine answer right up until the user has an
12
+ encrypted key, or a host that wants a password, or -- slice 5b -- a sudo
13
+ prompt on the far side of a connection. All three need a value kept between
14
+ the moment the user types it and the moment a subprocess needs it.
15
+
16
+ WHAT THE GUARANTEE IS, AND IS NOT (SS56)
17
+ ----------------------------------------
18
+ Stated precisely, because a half-understood control is worse than none
19
+ (UN4), and because this module is the one whose failure is silent.
20
+
21
+ It DOES defend against:
22
+
23
+ * the AGENT reading the file. A shell command on WSL, on SSH, or on the
24
+ host fallback can `cat` this file and gets ciphertext. That is the
25
+ threat this harness actually has, and it is why an OS keystore was
26
+ rejected: DPAPI decrypts for any process running as this user, and the
27
+ agent's shell is a process running as this user;
28
+ * the file leaving the machine -- a backup, a sync client, a stolen
29
+ disk. Without the passphrase it is 32 bytes of salt and a blob;
30
+ * a downgrade of the KDF parameters, which are covered by the AEAD's
31
+ associated data rather than merely stored beside the ciphertext. An
32
+ edited `n` does not weaken the next unlock, it fails it.
33
+
34
+ It does NOT defend against:
35
+
36
+ * arbitrary in-process Python, which `security/posture.py` already
37
+ concedes can rebind anything in this project, this module included;
38
+ * a keylogger, or anything watching the user type the passphrase;
39
+ * a crash dump taken while the store is unlocked. See `Secret` below --
40
+ the plaintext is zeroed on lock, but CPython hands out immutable
41
+ copies that nothing can reach, and that is measured rather than
42
+ assumed.
43
+
44
+ WHAT IS MEASURED HERE, NOT READ FROM DOCUMENTATION (batch 103)
45
+ --------------------------------------------------------------
46
+ * `hashlib.scrypt` REFUSES N=2**15 with the default `maxmem` --
47
+ "ValueError: [digital envelope routines] memory limit exceeded". The
48
+ default is not generous enough for any parameter worth using, so
49
+ `maxmem` is passed explicitly and `_SCRYPT_MAXMEM` is derived from the
50
+ parameters rather than being a constant someone has to remember to
51
+ move. Without it this module would raise on the first unlock on every
52
+ machine, which is the kind of failure that gets "fixed" by lowering N;
53
+ * N=2**15, r=8, p=1 costs 142 ms on the machine this was built on,
54
+ N=2**14 costs 73 ms and N=2**16 costs 286 ms. The parameter is a
55
+ measurement, not a number copied from a blog post, and it is stored in
56
+ the envelope so raising it later does not strand an existing file;
57
+ * zeroing a `bytearray` really does clear the bytes -- read back through
58
+ `ctypes.string_at` at the same address, before and after;
59
+ * `bytes(buf)` and `AESGCM.decrypt(...)` both return IMMUTABLE objects,
60
+ so every conversion makes a copy nothing can zero. That is why
61
+ `Secret` holds a `bytearray` and why `reveal()` says what it costs;
62
+ * `InvalidTag` carries no message at all. A wrong passphrase, a flipped
63
+ tag bit and an edited `n` are indistinguishable from the exception,
64
+ which is the behaviour this module wants: see `unlock`.
65
+ """
66
+
67
+ import atexit
68
+ import base64
69
+ import contextlib
70
+ import hashlib
71
+ import hmac
72
+ import json
73
+ import logging
74
+ import os
75
+ import threading
76
+ from typing import Optional
77
+
78
+ from cryptography.exceptions import InvalidTag
79
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
80
+
81
+ # A ROOT module, and no new edge: json_store.py's own docstring names
82
+ # `security/` as one of the packages it sits at the root to be reachable
83
+ # from, beside `config`, `storage`, `database` and `credentials`.
84
+ from json_store import write_json_atomic
85
+ from security import protected_paths
86
+
87
+ logger = logging.getLogger(__name__)
88
+
89
+ #: The envelope's shape. Bumping this is how a future KDF arrives without
90
+ #: guessing at an old file's meaning.
91
+ ENVELOPE_VERSION = 1
92
+
93
+ KDF_SCRYPT = "scrypt"
94
+ #: Measured on this machine at 142 ms (N=2**14 is 73 ms, N=2**16 is 286 ms).
95
+ #: Stored in the envelope, so a file written under one parameter still opens
96
+ #: after this constant moves.
97
+ SCRYPT_N = 1 << 15
98
+ SCRYPT_R = 8
99
+ SCRYPT_P = 1
100
+ KEY_BYTES = 32
101
+ SALT_BYTES = 16
102
+ NONCE_BYTES = 12
103
+
104
+ #: `hashlib.scrypt` REFUSES anything above N=2**14 with its default limit,
105
+ #: and the error names "memory limit exceeded" rather than the parameter.
106
+ #: Derived rather than hardcoded so the two cannot drift: scrypt's working
107
+ #: set is 128*N*r bytes, and the doubling is the slack OpenSSL wants on top.
108
+ def _scrypt_maxmem(n: int, r: int) -> int:
109
+ # No floor under this, and that is a deliberate removal rather than an
110
+ # omission. One was written here -- for a tiny `n` the derived figure
111
+ # IS below OpenSSL's working set, and it refuses with the same "memory
112
+ # limit exceeded" a too-large parameter gives -- and the mutation pass
113
+ # showed it can never fire: `_derive` is reached from `_seal` with
114
+ # SCRYPT_N, and from `_open` only after MIN_SCRYPT_N has already
115
+ # refused anything smaller. A guard that cannot fire still reads as
116
+ # the thing protecting you.
117
+ return 128 * n * r * 2
118
+
119
+
120
+ #: The weakest KDF this build will DERIVE against, whatever a file asks for.
121
+ #: SS56's downgrade defence has two independent halves and this is the
122
+ #: first: a file naming a weaker parameter is refused by policy, before any
123
+ #: key is derived. The second is the AEAD, which binds the parameters as
124
+ #: associated data so an edited one fails the tag as well. Either alone
125
+ #: would do; a security property with one implementation is a security
126
+ #: property with one typo between it and nothing.
127
+ MIN_SCRYPT_N = 1 << 14
128
+
129
+
130
+ #: What a redacted secret reads as. The same marker
131
+ #: `safety/policy_enforcement.py` already uses, imported from nowhere: this
132
+ #: module may not depend on `safety/`, and a second constant is pinned
133
+ #: against the first by a test rather than shared by an import that would
134
+ #: invert the layering `protected_paths` sets out.
135
+ REDACTED = "[REDACTED]"
136
+
137
+ #: The floor `put` enforces. Not a password-strength opinion -- it is about
138
+ #: SS61's value redaction, which substitutes a stored value over every
139
+ #: occurrence of it in every tool result and every log line: a
140
+ #: one-character secret would destroy the output rather than merely redact
141
+ #: it.
142
+ #:
143
+ #: WHAT IT DOES NOT DO, and this sentence used to claim otherwise (batch
144
+ #: 144). It does not make that redaction survivable in general. At 4
145
+ #: the floor admits `'true'`, `'null'`, `'name'` and `'aaaa'`, and a stored
146
+ #: `'true'` turns every JSON boolean in every tool result into `[REDACTED]`
147
+ #: -- measured. Raising it closes no class either: at 4 a stored `'pass'`
148
+ #: destroys every Python `pass` statement and at 8 `'password'` destroys
149
+ #: every build file, because length is a proxy for how often a value occurs
150
+ #: in ordinary text and a poor one. A four-character RANDOM password occurs
151
+ #: in nothing.
152
+ #:
153
+ #: So the floor stays 4 (the owner's ruling, batch 144) and `put` closes
154
+ #: the cases that have no legitimate reading at all -- a value that is
155
+ #: empty or nothing but whitespace -- rather than a bigger number being
156
+ #: chosen to look like a guarantee. Those are the ones with the worst
157
+ #: blast radius anyway: four spaces replaces every four-space indent in
158
+ #: every file the model is shown.
159
+ MIN_SECRET_LENGTH = 4
160
+
161
+
162
+ class SecretsError(Exception):
163
+ """Base for everything this module refuses to do."""
164
+
165
+
166
+ class StoreLocked(SecretsError):
167
+ """The store is not unlocked, so there is nothing to read."""
168
+
169
+
170
+ class WrongPassphrase(SecretsError):
171
+ """The envelope did not open. Deliberately does not say why."""
172
+
173
+
174
+ class StoreUnreadable(SecretsError):
175
+ """The file is not a store this build can open, WHATEVER the passphrase.
176
+
177
+ The distinction `WrongPassphrase` cannot carry, and it is a type rather
178
+ than a convention because the convention had already failed twice
179
+ (batch 108).
180
+
181
+ IN THE TESTS: `test_an_unknown_version_refuses_rather_than_guessing`
182
+ asserted `SecretsError`, which `WrongPassphrase` subclasses -- so
183
+ deleting the version check entirely still passed, because the AEAD then
184
+ failed the tag for a different reason. The mutation pass found it, and
185
+ the repair was `assert not isinstance(excinfo.value, WrongPassphrase)`
186
+ written out at each site.
187
+
188
+ IN PRODUCTION: both callers that report an unlock failure catch
189
+ `SecretsError` and print it. A recovery message added to that branch
190
+ would fire on a MISTYPED PASSPHRASE and tell someone to delete a
191
+ perfectly good store -- the exact footgun, in the exact shape the tests
192
+ had already met.
193
+
194
+ So the two are separated where they are RAISED, and a caller cannot get
195
+ the ordering wrong because there is no ordering to get wrong. Every
196
+ structural fault is one of these: an unreadable or non-JSON file, a
197
+ version or KDF this build does not know, malformed base64, key
198
+ derivation parameters below `MIN_SCRYPT_N`, and a plaintext that
199
+ decrypts cleanly and is not an entry table. A wrong passphrase is none
200
+ of them, and neither is a tampered ciphertext -- those are
201
+ indistinguishable by design (SS56) and both stay `WrongPassphrase`.
202
+
203
+ Subclasses `SecretsError`, so every existing `except` and every
204
+ `pytest.raises(SecretsError)` keeps working unchanged.
205
+ """
206
+
207
+
208
+ class NoStore(SecretsError):
209
+ """No store file exists yet."""
210
+
211
+
212
+ class Secret:
213
+ """A value that cannot be printed by accident (SS62).
214
+
215
+ The cheapest control in this batch and the one that prevents the
216
+ largest class of leak, because every other control in this module
217
+ assumes the value stays where it was put. An f-string, a `%s`, a
218
+ `logger.debug`, a pytest assertion diff and a traceback all reach for
219
+ `__str__` or `__repr__`, and all of them get the marker.
220
+
221
+ The bytes come out through `reveal()` and nowhere else, which makes
222
+ every REAL use greppable -- `grep -rn 'reveal()'` is the audit -- and
223
+ every accidental one impossible.
224
+
225
+ `reveal()` returns `bytes`, which is IMMUTABLE and therefore a copy
226
+ nothing can zero afterwards. That is measured, not assumed, and it is
227
+ the reason the method is named for what it does rather than called
228
+ `value`: a caller should be able to see at the call site that they
229
+ have just made a copy of a secret that outlives `lock()`. Hold it for
230
+ as short a time as possible.
231
+ """
232
+
233
+ __slots__ = ("_buf",)
234
+
235
+ def __init__(self, raw) -> None:
236
+ if isinstance(raw, str):
237
+ raw = raw.encode("utf-8")
238
+ self._buf = bytearray(raw)
239
+
240
+ def reveal(self) -> bytes:
241
+ """The plaintext, as an immutable copy this module cannot zero."""
242
+ return bytes(self._buf)
243
+
244
+ def zero(self) -> None:
245
+ """Overwrite the buffer in place. Verified with `ctypes.string_at`
246
+ against the same address (batch 103), which is the only way to know
247
+ it happened rather than hoping the name means what it says."""
248
+ for index in range(len(self._buf)):
249
+ self._buf[index] = 0
250
+
251
+ # -- everything below exists to make an accidental disclosure impossible
252
+
253
+ def __str__(self) -> str:
254
+ return REDACTED
255
+
256
+ def __repr__(self) -> str:
257
+ return REDACTED
258
+
259
+ def __format__(self, spec: str) -> str:
260
+ return REDACTED
261
+
262
+ def __eq__(self, other) -> bool:
263
+ if not isinstance(other, Secret):
264
+ return NotImplemented
265
+ return hmac.compare_digest(bytes(self._buf), bytes(other._buf))
266
+
267
+ #: Unhashable, so it cannot become a dict key whose repr is dumped.
268
+ __hash__ = None
269
+
270
+ def __reduce__(self):
271
+ raise TypeError("a Secret cannot be pickled")
272
+
273
+ def __copy__(self):
274
+ raise TypeError("a Secret cannot be copied")
275
+
276
+ def __deepcopy__(self, memo):
277
+ raise TypeError("a Secret cannot be copied")
278
+
279
+
280
+ # ---------------------------------------------------------------------------
281
+ # ---- Where the file lives -------------------------------------------------
282
+ # ---------------------------------------------------------------------------
283
+
284
+ def store_path() -> str:
285
+ """`~/.config/venastine/secrets.json`, resolved at CALL time.
286
+
287
+ Call time rather than import time, matching `tui/preferences.py` and
288
+ `core/model_windows.py` rather than `credentials.py`: a moved HOME is
289
+ picked up, and a test can redirect HOME instead of monkeypatching a
290
+ module attribute. `AGENT_SECRETS_FILE` is the explicit override, named
291
+ to match `AGENT_PROVIDERS_FILE` and `AGENT_ENV_FILE`.
292
+
293
+ The directory is already a PROTECTED PATH
294
+ (`security/protected_paths.py`), so the shell gate refuses a command
295
+ naming it and the file tools deny it outright. That is not what keeps
296
+ the secret safe -- the encryption is -- but it means the agent cannot
297
+ quietly delete the store either.
298
+ """
299
+ override = os.environ.get("AGENT_SECRETS_FILE", "")
300
+ if override:
301
+ return override
302
+ return os.path.join(protected_paths.user_config_dir(), "secrets.json")
303
+
304
+
305
+ def exists() -> bool:
306
+ """Whether a store file is present. Reads no content and needs no
307
+ passphrase, so every "is this configured" caller can ask."""
308
+ return os.path.exists(store_path())
309
+
310
+
311
+ def recovery_hint() -> str:
312
+ """What to tell someone holding a file this build cannot open.
313
+
314
+ ONE copy, for `--secrets` and `/secrets` both, because the two are the
315
+ same sentence and the CLI one is read by someone who has already tried
316
+ the TUI one.
317
+
318
+ IT NAMES THE PATH AND DELETES NOTHING. A store that will not open is
319
+ not necessarily a store with nothing in it -- it may have been written
320
+ by a different build, or be recoverable by hand from a backup -- and
321
+ an encrypted file is one this harness cannot inspect to find out. So
322
+ the refusal is a refusal, and removing the file stays the user's
323
+ action. `create` refuses to overwrite for the same reason, which is
324
+ what makes saying the path out loud necessary rather than merely
325
+ helpful: without it the next step is undiscoverable, in a directory
326
+ the shell gate and the file tools both refuse to touch.
327
+
328
+ IT NAMES TWO CAUSES, and the second one is the live one (batch 144).
329
+ This said the usual cause was "a write that was interrupted
330
+ before batch 108 made them atomic" -- which tells someone already
331
+ confused that their build is old or their file predates a fix, when
332
+ the write that produced it may have happened on this build thirty
333
+ seconds ago. `create` is deliberately NOT routed through `_write`, for
334
+ the reason its own docstring gives, so it is a single non-atomic write
335
+ and a crash inside it leaves exactly this state: `exists()` true,
336
+ `unlock` refusing structurally, `create` refusing to overwrite.
337
+ Driven, and the message quoted above is what the probe got back.
338
+
339
+ That is a sentence, not a defect in `create`: the remedy here was
340
+ always correct, and `O_EXCL` should not become `os.replace`, which
341
+ overwrites without complaint and is the TOCTOU `create` closed.
342
+ """
343
+ return (
344
+ "This file is not a secrets store this build can open, and that is "
345
+ "not a wrong passphrase -- the envelope itself is unreadable. Two "
346
+ "causes are known: a write interrupted before batch 108 made the "
347
+ "rewrites atomic, or an interrupted first creation, which is a "
348
+ "single non-atomic write by design and is still one on this build. "
349
+ "Nothing here will delete it for you: if you have no backup of it, "
350
+ "remove %s by hand and run the command again to start a new store."
351
+ % store_path()
352
+ )
353
+
354
+
355
+ # ---------------------------------------------------------------------------
356
+ # ---- The envelope ---------------------------------------------------------
357
+ # ---------------------------------------------------------------------------
358
+
359
+ def _b64(raw: bytes) -> str:
360
+ return base64.b64encode(raw).decode("ascii")
361
+
362
+
363
+ def _unb64(text: str) -> bytes:
364
+ return base64.b64decode(text.encode("ascii"), validate=True)
365
+
366
+
367
+ def _header(envelope: dict) -> dict:
368
+ """Everything except the ciphertext."""
369
+ return {key: value for key, value in envelope.items() if key != "ct"}
370
+
371
+
372
+ def _aad(envelope: dict) -> bytes:
373
+ """The associated data the ciphertext is bound to: the whole header.
374
+
375
+ WHAT THIS ACTUALLY BUYS, corrected by the mutation pass rather than
376
+ left as it was first written. The claim here used to be that the AAD
377
+ is what makes a downgraded `n` fail. Measured, it is NOT: every field
378
+ in the header today either feeds the key derivation (`salt`, `n`, `r`,
379
+ `p`) or is checked outright (`version`, `kdf`), so an edit to any of
380
+ them already fails without the AAD -- through a different key, or
381
+ through an explicit refusal. Replacing this function's return with
382
+ `b""` broke no test, which is how the overstatement was found.
383
+
384
+ What it buys is that the property does not DEPEND on that coincidence.
385
+ A header field added later -- a key id, a rotation counter, a hint
386
+ about which agent may use the entry -- would not feed the KDF, and
387
+ without this it would be an instruction the file gives its reader,
388
+ editable by anyone who can write the file. Inside the AAD it is
389
+ covered from the moment it exists, rather than from the moment
390
+ somebody remembers to check it. `test_an_unrecognised_header_field_is
391
+ _still_covered` is that case, written as the thing this is for.
392
+
393
+ The DOWNGRADE defence is `MIN_SCRYPT_N` plus the derivation itself;
394
+ this is the third answer, and it is the one that survives a change
395
+ nobody has made yet.
396
+
397
+ Canonical JSON with sorted keys, so the bytes do not depend on dict
398
+ ordering across versions.
399
+ """
400
+ return json.dumps(_header(envelope), sort_keys=True,
401
+ separators=(",", ":")).encode("utf-8")
402
+
403
+
404
+ def _derive(passphrase: bytes, salt: bytes, n: int, r: int, p: int) -> bytes:
405
+ """scrypt, with `maxmem` passed EXPLICITLY.
406
+
407
+ Measured: the default refuses N=2**15 with "memory limit exceeded", so
408
+ a version of this call without `maxmem` raises on the first unlock on
409
+ every machine. That failure looks like a bad parameter rather than a
410
+ library default, which is exactly how it gets "fixed" by lowering N.
411
+ """
412
+ return hashlib.scrypt(passphrase, salt=salt, n=n, r=r, p=p,
413
+ dklen=KEY_BYTES, maxmem=_scrypt_maxmem(n, r))
414
+
415
+
416
+ def _seal(entries: dict, passphrase: bytes) -> dict:
417
+ salt = os.urandom(SALT_BYTES)
418
+ nonce = os.urandom(NONCE_BYTES)
419
+ envelope = {
420
+ "version": ENVELOPE_VERSION,
421
+ "kdf": KDF_SCRYPT,
422
+ "n": SCRYPT_N,
423
+ "r": SCRYPT_R,
424
+ "p": SCRYPT_P,
425
+ "salt": _b64(salt),
426
+ "nonce": _b64(nonce),
427
+ }
428
+ key = _derive(passphrase, salt, SCRYPT_N, SCRYPT_R, SCRYPT_P)
429
+ plaintext = json.dumps(entries, sort_keys=True).encode("utf-8")
430
+ envelope["ct"] = _b64(AESGCM(key).encrypt(nonce, plaintext,
431
+ _aad(envelope)))
432
+ return envelope
433
+
434
+
435
+ def _open(envelope: dict, passphrase: bytes) -> dict:
436
+ if envelope.get("version") != ENVELOPE_VERSION:
437
+ raise StoreUnreadable(
438
+ "secrets file is version %r; this build writes version %d"
439
+ % (envelope.get("version"), ENVELOPE_VERSION))
440
+ if envelope.get("kdf") != KDF_SCRYPT:
441
+ raise StoreUnreadable("secrets file uses an unknown KDF: %r"
442
+ % (envelope.get("kdf"),))
443
+ try:
444
+ salt = _unb64(envelope["salt"])
445
+ nonce = _unb64(envelope["nonce"])
446
+ ciphertext = _unb64(envelope["ct"])
447
+ n = int(envelope["n"])
448
+ r = int(envelope["r"])
449
+ p = int(envelope["p"])
450
+ except (KeyError, ValueError, TypeError) as exc:
451
+ raise StoreUnreadable("secrets file is malformed: %s" % exc) from exc
452
+ if n < MIN_SCRYPT_N or r < 1 or p < 1 or n & (n - 1):
453
+ raise StoreUnreadable(
454
+ "secrets file asks for a weaker key derivation than this build "
455
+ "accepts (n=%r, r=%r, p=%r); it will not be opened" % (n, r, p))
456
+ try:
457
+ key = _derive(passphrase, salt, n, r, p)
458
+ except ValueError as exc:
459
+ # Anything the KDF itself refuses is a malformed FILE, not a
460
+ # programming error to propagate raw out of `unlock`.
461
+ raise StoreUnreadable(
462
+ "secrets file's key derivation parameters were rejected: %s"
463
+ % exc) from exc
464
+ try:
465
+ plaintext = AESGCM(key).decrypt(nonce, ciphertext, _aad(envelope))
466
+ except InvalidTag as exc:
467
+ # MEASURED: InvalidTag carries no message, and the three causes --
468
+ # wrong passphrase, tampered ciphertext, edited KDF parameter --
469
+ # are indistinguishable here. That is the right shape rather than a
470
+ # limitation: telling the two apart would tell someone holding a
471
+ # stolen file that their guess was structurally right.
472
+ raise WrongPassphrase(
473
+ "the passphrase did not open the secrets file") from exc
474
+ try:
475
+ entries = json.loads(plaintext.decode("utf-8"))
476
+ except (ValueError, UnicodeDecodeError) as exc:
477
+ raise StoreUnreadable("secrets file decrypted to something that is not "
478
+ "an entry table") from exc
479
+ if not isinstance(entries, dict):
480
+ raise StoreUnreadable("secrets file decrypted to something that is not "
481
+ "an entry table")
482
+ return entries
483
+
484
+
485
+ # ---------------------------------------------------------------------------
486
+ # ---- The unlocked state ---------------------------------------------------
487
+ # ---------------------------------------------------------------------------
488
+
489
+ class _Unlocked:
490
+ """What a session holds between `unlock()` and `lock()`.
491
+
492
+ `stamp` is how SS60's "the store file changed under us" trigger is
493
+ answered without re-reading the file on every get: the (mtime, size)
494
+ the file had when it was opened. A rotation from another process, or
495
+ an editor writing over it, means what is in memory is no longer what
496
+ is on disk -- and continuing to serve the old values would make the
497
+ two silently disagree.
498
+ """
499
+
500
+ __slots__ = ("passphrase", "entries", "stamp")
501
+
502
+ def __init__(self, passphrase: bytearray, entries: dict, stamp) -> None:
503
+ self.passphrase = passphrase
504
+ self.entries = entries
505
+ self.stamp = stamp
506
+
507
+ def zero(self) -> None:
508
+ for index in range(len(self.passphrase)):
509
+ self.passphrase[index] = 0
510
+ for secret in self.entries.values():
511
+ secret.zero()
512
+ self.entries.clear()
513
+
514
+
515
+ _state: Optional[_Unlocked] = None
516
+ #: Re-entrant, because `lock()` is called from inside `guarded()` blocks
517
+ #: that already hold it, and from `atexit` while another thread may be in a
518
+ #: read.
519
+ _state_lock = threading.RLock()
520
+
521
+
522
+ def _stamp_of(path: str):
523
+ try:
524
+ info = os.stat(path)
525
+ except OSError:
526
+ return None
527
+ return (info.st_mtime_ns, info.st_size)
528
+
529
+
530
+ def is_unlocked() -> bool:
531
+ with _state_lock:
532
+ return _state is not None
533
+
534
+
535
+ def lock(reason: str = "") -> bool:
536
+ """Drop every plaintext this process holds. Returns whether anything
537
+ was actually held.
538
+
539
+ SS60's single exit. Every trigger calls THIS -- the slash command, the
540
+ CLI subcommand, quit, `atexit`, `guarded()`'s exception path, the
541
+ askpass listener's refusals, and a store file that changed underneath.
542
+ One function rather than each of them clearing what it happens to know
543
+ about, which is how one of them ends up clearing less.
544
+
545
+ Never raises: it is called from `atexit` and from exception handlers,
546
+ and a lock that can fail is a lock that sometimes does not happen.
547
+ """
548
+ global _state
549
+ with _state_lock:
550
+ held = _state is not None
551
+ if held:
552
+ try:
553
+ _state.zero()
554
+ except Exception: # pragma: no cover
555
+ logger.error("secrets: zeroing failed during lock",
556
+ exc_info=True)
557
+ _state = None
558
+ if held and reason:
559
+ # NAMES the reason, never a value (AGENTS.md: a breadcrumb
560
+ # quoting a value is one log rotation away from a durable
561
+ # secret beside the redacted sinks).
562
+ logger.info("secrets: locked (%s)", reason)
563
+ return held
564
+
565
+
566
+ atexit.register(lock, "process exit")
567
+
568
+
569
+ def unlock(passphrase: str) -> None:
570
+ """Open the store for this process.
571
+
572
+ Raises `NoStore` when there is no file, and `WrongPassphrase` when the
573
+ envelope does not open -- which, measured, is the same exception for a
574
+ wrong passphrase, a tampered ciphertext and an edited KDF parameter.
575
+ """
576
+ path = store_path()
577
+ if not os.path.exists(path):
578
+ raise NoStore("no secrets file at %s" % path)
579
+ try:
580
+ with open(path, encoding="utf-8") as handle:
581
+ envelope = json.load(handle)
582
+ except (OSError, ValueError) as exc:
583
+ raise StoreUnreadable("secrets file could not be read: %s" % exc) from exc
584
+ if not isinstance(envelope, dict):
585
+ raise StoreUnreadable("secrets file is malformed")
586
+ raw = passphrase.encode("utf-8")
587
+ entries = _open(envelope, raw)
588
+ global _state
589
+ with _state_lock:
590
+ lock()
591
+ _state = _Unlocked(
592
+ bytearray(raw),
593
+ {name: Secret(value) for name, value in entries.items()},
594
+ _stamp_of(path))
595
+ logger.info("secrets: unlocked, %d entries", len(entries))
596
+
597
+
598
+ def create(passphrase: str) -> None:
599
+ """Write a new, empty store. Refuses to overwrite an existing one.
600
+
601
+ THE REFUSAL IS THE WRITE, not a check above it (batch 108). This was
602
+ `if os.path.exists(path): raise`, which is a TOCTOU -- and one that got
603
+ sharper rather than softer when `_write` became atomic, because
604
+ `os.replace` overwrites without complaint, so the check was the only
605
+ thing standing between a second `create` and somebody's store.
606
+ `O_EXCL` asks the filesystem the same question indivisibly.
607
+
608
+ So this does NOT go through `_write`. Atomic replace is the right shape
609
+ for `_flush`, which rewrites a file that already has contents many
610
+ times; exclusive creation is the right shape for the one write whose
611
+ whole contract is that the file must not already be there. A crash
612
+ inside this one leaves a truncated file that never held anything, which
613
+ is the case `unlock`'s structural refusal names.
614
+ """
615
+ path = store_path()
616
+ parent = os.path.dirname(path)
617
+ if parent:
618
+ os.makedirs(parent, exist_ok=True)
619
+ envelope = _seal({}, passphrase.encode("utf-8"))
620
+ try:
621
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
622
+ except FileExistsError as exc:
623
+ raise SecretsError(
624
+ "a secrets file already exists at %s" % path) from exc
625
+ with os.fdopen(fd, "w", encoding="utf-8") as handle:
626
+ json.dump(envelope, handle, indent=2, sort_keys=True)
627
+ unlock(passphrase)
628
+
629
+
630
+ def _write(envelope: dict, path: str) -> None:
631
+ """Write the envelope to a file created 0600, ATOMICALLY.
632
+
633
+ 0600 AT CREATION, never `open()` then `os.chmod` -- the same rule and
634
+ the same reason as `credentials._write_secret_json`, whose comment
635
+ records that a post-write chmod leaves a window in which the file is on
636
+ disk with the umask's permissions. On Windows the mode is a no-op and
637
+ the protection is the ACL on the user profile directory; that is
638
+ recorded there too, and is why this file is encrypted rather than
639
+ relying on the mode at all.
640
+
641
+ THE OTHER HALF, AND IT WAS MISSING UNTIL BATCH 108. This wrote
642
+ `O_TRUNC` straight over the live store, and `_flush` calls it on every
643
+ `put` and `delete` -- so a crash, a full disk or a kill between the
644
+ truncate and the flush left a half-written envelope. That is worse than
645
+ losing the write: `exists()` stays True, so `unlock` is attempted and
646
+ fails structurally, and `create` refuses to overwrite -- leaving a user
647
+ who cannot store a secret again without deleting a file by hand from a
648
+ directory the shell gate and the file tools both refuse to touch.
649
+
650
+ `json_store.write_json_atomic` is temp-file-plus-`os.replace` and
651
+ already existed, with the mode applied to the TEMP file because replace
652
+ carries it across. Its own docstring is where this argument is made --
653
+ "it leaves the file empty for the length of the write and permanently
654
+ damaged if the process dies inside it" -- and it names `security/` as
655
+ one of the packages it sits at the root to be reachable from. So the
656
+ two files in this project that hold credentials were the last two
657
+ JSON stores still using the pattern that module was extracted to
658
+ delete, and this is not a new writer but the one that was already
659
+ there.
660
+ """
661
+ write_json_atomic(path, envelope, mode=0o600, indent=2, sort_keys=True)
662
+
663
+
664
+ def _require_unlocked() -> _Unlocked:
665
+ with _state_lock:
666
+ if _state is None:
667
+ raise StoreLocked("the secrets store is locked")
668
+ path = store_path()
669
+ if _stamp_of(path) != _state.stamp:
670
+ # SS60. What is in memory is no longer what is on disk, so
671
+ # somebody rotated, replaced or removed the file. Continuing to
672
+ # serve the values held here would make the two silently
673
+ # disagree; locking makes the next use ask.
674
+ lock("store file changed on disk")
675
+ raise StoreLocked(
676
+ "the secrets file changed on disk; the store was locked")
677
+ return _state
678
+
679
+
680
+ def names() -> list:
681
+ """Every entry name, sorted. NEVER a value -- this is the only thing a
682
+ listing surface may read, which is what lets `/secrets list` be written
683
+ without a way to print a password."""
684
+ return sorted(_require_unlocked().entries)
685
+
686
+
687
+ def get(name: str) -> Optional[Secret]:
688
+ """The entry, or None when there is no such name."""
689
+ return _require_unlocked().entries.get(name)
690
+
691
+
692
+ def put(name: str, value: str) -> None:
693
+ """Add or replace one entry and rewrite the file.
694
+
695
+ Named `put` rather than `set` so the module does not shadow the
696
+ builtin at every internal call site.
697
+ """
698
+ if not name or not name.strip():
699
+ raise SecretsError("a secret needs a name")
700
+ if not value.strip():
701
+ # THE SAME CHECK THE NAME GETS ONE LINE UP (batch 144).
702
+ # Refused rather than stripped-and-stored: a password with a
703
+ # trailing space is still that password, and silently altering a
704
+ # value the user cannot see -- SS59 masks this input in both the
705
+ # TUI modal and the CLI -- would be worse than refusing it.
706
+ # `test_a_value_may_still_contain_other_whitespace` pins that
707
+ # INTERNAL whitespace stays permitted, which is why this is
708
+ # `strip()` and not a character class.
709
+ #
710
+ # Refusing at the door is where the constraint can still be
711
+ # explained to whoever is typing, which is the argument the
712
+ # newline check below already makes. The symptom otherwise is
713
+ # undiagnosable from the other end: `[REDACTED]` starts appearing
714
+ # in unrelated places, in every sink at once, with nothing
715
+ # connecting it to a stored secret.
716
+ raise SecretsError(
717
+ "a secret cannot be empty or only whitespace: a stored value is "
718
+ "substituted over every occurrence of it in every tool result "
719
+ "and every log line, unconditionally, so four spaces would "
720
+ "replace every four-space indent the model is shown with "
721
+ "[REDACTED]")
722
+ if len(value) < MIN_SECRET_LENGTH:
723
+ raise SecretsError(
724
+ "a secret must be at least %d characters: shorter values are "
725
+ "substituted over every occurrence in every tool result and "
726
+ "would destroy the output rather than redact it"
727
+ % MIN_SECRET_LENGTH)
728
+ if "\n" in value or "\r" in value:
729
+ # A LINE, NOT A BLOB (batch 117). `sandbox._sudo_password_line` is
730
+ # `reveal() + b"\n"` and `sudo -S` reads exactly one line, so a
731
+ # stored newline hands sudo the text BEFORE it and drops the rest.
732
+ # It fails closed -- wrong password, and `_SUDO_STDIN_GUARD` keeps
733
+ # the remainder off the command line -- so the cost is an auth
734
+ # failure that quarantines the entry and names nothing useful.
735
+ # Refusing at the door is where the constraint can still be
736
+ # explained to whoever is typing.
737
+ raise SecretsError(
738
+ "a secret cannot contain a newline: it is written to a "
739
+ "subprocess as a single line (sudo -S reads one), so a value "
740
+ "with a line break in it would be delivered truncated")
741
+ with _state_lock:
742
+ state = _require_unlocked()
743
+ previous = state.entries.get(name)
744
+ state.entries[name] = Secret(value)
745
+ try:
746
+ _flush(state)
747
+ except BaseException:
748
+ # MEMORY FOLLOWS DISK (batch 108). This used to zero the old
749
+ # Secret and install the new one BEFORE the write, so a flush
750
+ # that failed left the process holding a value that is not in
751
+ # the file -- reported as an error to the user and usable for
752
+ # the rest of the run, then gone at restart. `_require_unlocked`
753
+ # cannot notice: it compares the stamp against the file, and a
754
+ # failed write leaves both unchanged.
755
+ if previous is None:
756
+ state.entries.pop(name, None)
757
+ else:
758
+ state.entries[name] = previous
759
+ raise
760
+ # AFTER the write, and only on success: zeroing the old value is
761
+ # the irreversible half, so it waits until the new one is really
762
+ # on disk.
763
+ if previous is not None:
764
+ previous.zero()
765
+
766
+
767
+ def delete(name: str) -> bool:
768
+ """Remove one entry. Returns whether it was there."""
769
+ with _state_lock:
770
+ state = _require_unlocked()
771
+ secret = state.entries.get(name)
772
+ if secret is None:
773
+ return False
774
+ del state.entries[name]
775
+ try:
776
+ _flush(state)
777
+ except BaseException:
778
+ # `put`'s rule, and the same reason: a delete that failed to
779
+ # reach the disk must not have happened in memory either.
780
+ state.entries[name] = secret
781
+ raise
782
+ secret.zero()
783
+ return True
784
+
785
+
786
+ def _flush(state: _Unlocked) -> None:
787
+ """Re-seal and rewrite, then re-stamp.
788
+
789
+ A FRESH salt and nonce every write, which is not merely tidy: AES-GCM
790
+ is catastrophically broken by a repeated (key, nonce) pair, and a store
791
+ that kept its nonce across writes would reuse one on every edit.
792
+ """
793
+ path = store_path()
794
+ entries = {name: secret.reveal().decode("utf-8")
795
+ for name, secret in state.entries.items()}
796
+ _write(_seal(entries, bytes(state.passphrase)), path)
797
+ state.stamp = _stamp_of(path)
798
+
799
+
800
+ @contextlib.contextmanager
801
+ def guarded(what: str):
802
+ """Run a block that uses secrets, and LOCK if anything escapes it.
803
+
804
+ SS60's "an exception escaping any secret consumer" trigger, as a
805
+ context manager rather than a rule each consumer remembers. An
806
+ exception here means the harness no longer knows what state the
807
+ subprocess, the connection or the store is in, and holding plaintext
808
+ through an unknown state is exactly the condition the owner asked to
809
+ lock on.
810
+
811
+ Re-raises. This decides what to KEEP, not what to report.
812
+ """
813
+ try:
814
+ yield
815
+ except BaseException:
816
+ lock("exception during %s" % what)
817
+ raise
818
+
819
+
820
+ # ---------------------------------------------------------------------------
821
+ # ---- Value redaction (SS61) -----------------------------------------------
822
+ # ---------------------------------------------------------------------------
823
+
824
+ def redact_live_values(text: str) -> str:
825
+ """Replace any live secret VALUE in *text* with the marker.
826
+
827
+ UNCONDITIONAL -- it does not consult `redact_tool_outputs`, and that is
828
+ the decision rather than an oversight. The master switch governs
829
+ pattern GUESSES about other people's credentials, and a user who turns
830
+ it off has chosen to see those raw. This is a value the harness itself
831
+ put into a process, coming back out; handing it to the model is the
832
+ harness leaking its own secret, and no switch asks for that. It follows
833
+ the depth cap, which `redaction_enabled()` already documents as staying
834
+ fail-closed because it is structure rather than content judgment.
835
+
836
+ The values never leave this module: `safety/policy_enforcement.py`
837
+ calls THIS rather than asking for a list to compare against, so there
838
+ is exactly one place holding plaintext and the redactor is not it.
839
+
840
+ Longest first, so a secret that contains another is replaced whole
841
+ rather than leaving the tail of the longer one behind.
842
+ """
843
+ if not text:
844
+ return text
845
+ with _state_lock:
846
+ if _state is None:
847
+ return text
848
+ values = sorted(
849
+ (secret.reveal().decode("utf-8", "replace")
850
+ for secret in _state.entries.values()),
851
+ key=len, reverse=True)
852
+ for value in values:
853
+ if value and value in text:
854
+ text = text.replace(value, REDACTED)
855
+ return text