@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
package/json_store.py CHANGED
@@ -1,343 +1,343 @@
1
- """
2
- json_store.py
3
-
4
- The mechanics four user-tier JSON stores had four copies of.
5
-
6
- AND THE FILE TOOLS' TEXT WRITES SINCE BATCH 146, which is why
7
- `_replace_through_temp` exists under both public writers rather than the
8
- dance living inside `write_json_atomic`. `write_text_atomic` says what it
9
- needs the dance FOR, and it is not durability: `os.replace` replaces a name
10
- rather than following it, which is the only thing that stops a write through
11
- a hardlinked document name landing on the file the link shares. A sixth copy
12
- of a temp-and-rename would have been the obvious way to get that, and this
13
- module exists because four copies of it had already drifted.
14
-
15
- AND THE PROJECT LOADER'S READS SINCE BATCH 219. `read_regular_text` is the
16
- read half of the same kind of argument: a cloned project's `.venastine/`
17
- files and root `AGENTS.md` are read by the trust prompt before anyone has
18
- consented to them, and a skill symlinked to `/dev/zero` was read until
19
- memory ran out. Six read sites in two packages needed the one guard, which
20
- is the shape this module was made for.
21
-
22
- WHAT THIS IS AND IS NOT. It owns *how* a small JSON file is read and
23
- written -- the temp-file-plus-replace dance, the version guard, the
24
- fail-soft read -- and nothing about what any of them MEAN. Each store keeps
25
- its own module, its own path, its own STORE_VERSION and its own posture,
26
- because those are the parts that were decided separately and are argued at
27
- length where they live:
28
-
29
- core/workspace_trust.py trusted_projects.json fails CLOSED, 0600,
30
- unversioned (a flat path -> hash mapping)
31
- mcp_client/config.py known_mcp_servers.json versioned, with F3's
32
- legacy-v1 re-ask arm
33
- core/model_windows.py model_windows.json versioned, fail soft
34
- tui/preferences.py ui_preferences.json versioned, fail soft,
35
- read-modify-write (RM4)
36
-
37
- WHY IT IS AT THE ROOT. `core/model_windows.py` is read by
38
- `core/compaction.py`; `tui/preferences.py` states that it imports nothing
39
- from `core/`; `mcp_client/` and `security/` each document a reason not to
40
- reach into `core/` either. A root module is the one place all four can
41
- already see -- `config`, `storage`, `database` and `credentials` are
42
- imported from everywhere for the same reason -- so sharing this costs no
43
- new edge in any direction.
44
-
45
- WHAT THE COPIES HAD ALREADY DRIFTED INTO. Three of the four wrote through a
46
- temp file and `os.replace`, each spending four to eight docstring lines on
47
- why a truncate-then-write is unacceptable: it leaves the file empty for the
48
- length of the write and permanently damaged if the process dies inside it.
49
- The fourth -- `mcp_client`'s, which stores a record of CONSENT, the same
50
- class of data as the trust store -- wrote with a bare `open(path, "w")` and
51
- said nothing. That is what a comment-enforced convention decays into, and
52
- extracting the write is what makes it stop being a convention.
53
-
54
- ERRORS BELONG TO THE CALLER. `write_json_atomic` raises; it does not decide
55
- whether a failed write is a WARNING the user should see (model_windows,
56
- preferences: yes, and return False) or an exception (workspace_trust: yes,
57
- because a trust grant that silently did not persist is worse than a crash).
58
- Each caller keeps the policy it already had.
59
- """
60
-
61
- from __future__ import annotations
62
-
63
- import errno
64
- import json
65
- import logging
66
- import os
67
- import stat
68
- from typing import Optional
69
-
70
- logger = logging.getLogger(__name__)
71
-
72
-
73
- class DestinationInUse(PermissionError):
74
- """Windows refused the rename because another program has the file open.
75
-
76
- Batch 219 (#242). This and a destination that is genuinely not writable
77
- both arrive from `os.replace` as `PermissionError` WinError 5, and the
78
- remedies are opposite: close it in the other program, or you may not
79
- write here. Measured on Windows: a destination another process holds
80
- open for reading still opens for `r+b` from here, and a read-only one
81
- does not -- so `_held_open_elsewhere` tells them apart without writing a
82
- byte. A subclass, so every `except OSError` that already reports a
83
- failed write reports this one too, now saying what to do.
84
- """
85
-
86
-
87
- def write_json_atomic(path: str, payload, *, mode: Optional[int] = None,
88
- indent: int = 2, sort_keys: bool = False,
89
- trailing_newline: bool = False) -> None:
90
- """Write `payload` as JSON to `path`, atomically. Raises OSError.
91
-
92
- Temp file plus `os.replace`, which is atomic on POSIX and on Windows.
93
- Parent directories are created.
94
-
95
- `mode` sets permissions on the TEMP FILE, not on the destination (#19).
96
- `os.replace` carries the source file's mode across, so setting it
97
- afterwards would be both a no-op and a lie, and setting it on the
98
- destination beforehand is worse -- replace overwrites that too.
99
-
100
- EVERY CALLER HOLDING A CREDENTIAL PASSES ONE, and two of them pass it
101
- for the reason the trust store explicitly does NOT: `credentials.py` and
102
- `security/secrets.py` hold secrets, and 0600-at-creation is the property
103
- their own comments argue for at length. `mcp_client/config.py` is the
104
- fourth, for the `Authorization` header batch 140 named in
105
- `.venastine/mcp.json`. This used to say "THREE CALLERS PASS ONE" and was written
106
- when three did -- a count in prose with nothing able to notice the
107
- fourth, which is the shape batch 143 closed twice on another surface and
108
- answered the same way: the argument never needed the number, so it is
109
- gone rather than corrected (batch 144).
110
-
111
- `trailing_newline` is off by default so every existing store writes
112
- the same bytes it always did. The four here are machine state nobody
113
- opens; §24's `/init --config` scaffold is the first file written
114
- through this that a human is meant to edit and COMMIT, and a
115
- committed file with no final newline carries git's "No newline at end
116
- of file" marker in every diff of it, forever after.
117
- """
118
- def _dump(handle) -> None:
119
- json.dump(payload, handle, indent=indent, sort_keys=sort_keys)
120
- if trailing_newline:
121
- handle.write("\n")
122
-
123
- # newline=None, which is what this has always passed and therefore what
124
- # every stored file already holds: `json.dump` with an indent writes
125
- # "\n" between its lines and the stream translates them, so on Windows
126
- # these stores are CRLF on disk. Not a property worth having, and not
127
- # one worth changing either -- it would rewrite the bytes of four user
128
- # -tier stores to no end. The text caller below wants the opposite and
129
- # says so there.
130
- _replace_through_temp(path, _dump, mode=mode, newline=None)
131
-
132
-
133
- def write_text_atomic(path: str, text: str, *, newline: str = "") -> None:
134
- """Write `text` to `path`, atomically and byte-for-byte. Raises OSError.
135
-
136
- THE FIFTH CONSUMER OF THE DANCE ABOVE, and the first that is not a JSON
137
- store (batch 146). The file tools reach it for two reasons that are not
138
- the ones the JSON stores have:
139
-
140
- `newline=""` by default, because a tool that was handed content must
141
- write the content it was handed. `open(path, "w")` translates every
142
- "\n" to `os.linesep`, so on Windows `write` could not reproduce bytes
143
- it had just read, and one `write_project_doc` call against an LF
144
- document rewrote every line ending in it -- 2,910 of them in this
145
- repository's own AGENTS.md -- under an approval that had been shown a
146
- diff of one line.
147
-
148
- And REPLACES RATHER THAN FOLLOWS, which is the property `.venastine/`
149
- depends on. `os.path.realpath` cannot see a hardlink -- a hardlink is
150
- not a link to anything, it IS the file -- so a containment test on the
151
- resolved path admits a write through a hardlinked name, and a `write`
152
- through one put `{"mcpServers": ...}` into a protected `settings.json`
153
- whose direct spelling is refused. `os.replace` onto the name breaks the
154
- link instead: the other name keeps its bytes and this one becomes a new
155
- file. Measured both ways round.
156
-
157
- The cost, stated because the caller cannot see it from here: a
158
- destination another process merely has OPEN fails the rename on Windows,
159
- where a truncating write would have succeeded. That is the same trade
160
- `write_json_atomic` documents above, and the same answer -- a write that
161
- fails loudly beats one that half-succeeds.
162
- """
163
- _replace_through_temp(path, lambda handle: handle.write(text),
164
- newline=newline)
165
-
166
-
167
- def _replace_through_temp(path: str, write_body, *,
168
- mode=None, newline=None) -> None:
169
- """Write through a unique temp file and `os.replace` it onto `path`.
170
-
171
- ONE COPY OF THE MECHANICS, which is this module's whole reason for
172
- existing -- `write_body` is handed an open text stream and writes
173
- whatever the caller means; everything else here is the dance, and every
174
- paragraph below was written against a failure that actually happened.
175
-
176
- GUARDED, because `os.makedirs("")` raises rather than doing nothing
177
- and `os.path.dirname` returns "" for a bare filename. No caller could
178
- reach it while all six built an absolute path, but `AGENT_SECRETS_FILE`
179
- and `AGENT_PROVIDERS_FILE` are documented overrides a user may set to
180
- one -- so the two secret stores joining this helper (batch 108) is what
181
- made a path SHAPE able to decide whether the write works.
182
- """
183
- parent = os.path.dirname(path)
184
- if parent:
185
- os.makedirs(parent, exist_ok=True)
186
-
187
- # A UNIQUE NAME, not a fixed `<path>.tmp` sibling (batch 144).
188
- # `config_edit.write` reached this conclusion for `config.yaml` and
189
- # wrote down why -- "two writers raced on the one name, and a failed
190
- # `os.replace` left the leftover in the install tree forever" -- and
191
- # this helper, extracted so these mechanics would have exactly ONE
192
- # copy, had neither half of the answer. One question, two answers, and
193
- # nothing that could tell you they disagreed.
194
- #
195
- # Two instances in one directory is a supported configuration:
196
- # `docs/PRIVACY.md` says "everything is relative to the directory you
197
- # run in".
198
- tmp = "%s.%s.tmp" % (path, os.urandom(4).hex())
199
-
200
- # THE TEMP FILE IS SWEPT ON FAILURE (batch 108). Without this a write
201
- # that dies inside `_dump` leaves its temp file behind for good: the
202
- # destination is correctly untouched, which is the whole point, but the
203
- # debris is not. It is worst for the two stores that joined this helper
204
- # in the same batch -- a half-written SEALED ENVELOPE sitting next to
205
- # the secrets file, and a partial provider file next to the real one.
206
- #
207
- # AND THE RENAME IS INSIDE IT (batch 144). `os.replace` used to
208
- # be the statement after this handler, so the sweep covered the failure
209
- # the comment above names and not the one next to it -- which is the
210
- # repository's own prompt for defect shape 6: "what actually raises
211
- # this, and what raises the thing next to it?"
212
- #
213
- # The rename's failure is not hypothetical and not symmetrical across
214
- # platforms. On Windows a plain `open()` takes no FILE_SHARE_DELETE, so
215
- # any other process merely READING the destination makes `os.replace`
216
- # fail -- an editor, a sync client, a backup, a virus scanner, or a
217
- # second instance in the same directory. Driven, with a happy-path
218
- # control: `PermissionError WinError 5`, destination correctly intact,
219
- # and `providers.json.tmp` left holding EVERY stored provider key in
220
- # cleartext, because `save_credentials` reads the whole file, mutates
221
- # one entry and writes all of it back. The POSIX arm is clean -- rename
222
- # there ignores open handles and the destination's mode -- and the
223
- # process-death window between the write and the rename is on both.
224
- #
225
- # Hygiene, never error policy: the original exception is re-raised
226
- # unchanged, because "errors belong to the caller" above is the rule
227
- # and a cleanup that swallowed one would be the opposite of it. The
228
- # unlink is itself guarded, so a failure to tidy up cannot replace the
229
- # failure worth reporting. The one rewording (batch 219) keeps that
230
- # rule: a held-open destination is still a PermissionError, chained to
231
- # the original, and only its message changes -- from one that reads as
232
- # a permissions problem to the remedy, which is closing the file.
233
- #
234
- # ONE `os.open` rather than two branches, and `0o666` is not a new
235
- # policy: it is the mode `open(path, "w")` passes, so the umask
236
- # subtracts from it exactly as before. Measured on WSL at `umask 022`,
237
- # both forms give 0o644 -- while `os.open` with NO mode argument gives
238
- # 0o755, which is why the argument is explicit rather than omitted.
239
- # `O_EXCL` on a unique name is what makes the sweep below safe: the
240
- # file this unlinks is always one this call created, never another
241
- # writer's.
242
- flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
243
- fd = os.open(tmp, flags, 0o666 if mode is None else mode)
244
- try:
245
- with os.fdopen(fd, "w", encoding="utf-8", newline=newline) as f:
246
- write_body(f)
247
- os.replace(tmp, path)
248
- except BaseException as exc:
249
- try:
250
- os.unlink(tmp)
251
- except OSError:
252
- pass # the temp file may not exist; the original error is what is raised
253
- if _held_open_elsewhere(path, exc):
254
- # No path in the message: every caller prefixes its own ("Could
255
- # not write <path>: ..."), and a path here printed it twice.
256
- raise DestinationInUse(
257
- "it is open in another program (an editor, a sync client, a "
258
- "backup or a virus scanner), so Windows will not replace it. "
259
- "Close it there and try again.") from exc
260
- raise
261
-
262
-
263
- def _held_open_elsewhere(path: str, exc: BaseException) -> bool:
264
- """Whether a failed rename onto `path` was another process's open handle.
265
-
266
- Windows only: a POSIX rename ignores open handles. WinError 32 is a
267
- sharing violation and says so itself; WinError 5 is ambiguous, and the
268
- `r+b` open settles it (see `DestinationInUse`). The probe writes
269
- nothing, and a destination that has gone or is a directory fails it.
270
- """
271
- if os.name != "nt" or not isinstance(exc, PermissionError):
272
- return False
273
- if getattr(exc, "winerror", None) == 32:
274
- return True
275
- try:
276
- with open(path, "r+b"):
277
- return True
278
- except OSError as probe:
279
- return getattr(probe, "winerror", None) == 32
280
-
281
-
282
- def read_regular_text(path: str, encoding: str = "utf-8-sig") -> str:
283
- """The whole text of `path`, which must be a regular file. Raises OSError.
284
-
285
- For content a cloned project brought with it (batch 219, #214), which
286
- the trust prompt reads BEFORE anyone has consented to it. Git stores a
287
- symlink, `open()` follows one, and `/dev/zero`, `/dev/urandom` and
288
- `/dev/tty` exist on every POSIX machine: measured, the listing read a
289
- skill linked to `/dev/zero` until a 1 GiB cap raised MemoryError,
290
- while the trust hash -- which skips non-regular files for exactly this
291
- reason -- stepped over it. A FIFO is the other half: `open()` on one
292
- blocks until a writer appears.
293
-
294
- So the file is opened NON-BLOCKING and checked on the open descriptor.
295
- A FIFO's open then returns at once, and `fstat` rather than a `stat` of
296
- the name leaves no window for the name to be swapped between the check
297
- and the read. A non-regular file raises OSError, which is what every
298
- reader of project content already reports or skips. `O_BINARY` stops
299
- Windows' C runtime translating the bytes under the decoder, which
300
- `open()` would have done for itself.
301
- """
302
- flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)
303
- fd = os.open(path, flags)
304
- try:
305
- regular = stat.S_ISREG(os.fstat(fd).st_mode)
306
- except BaseException:
307
- os.close(fd)
308
- raise
309
- if not regular:
310
- os.close(fd)
311
- raise OSError(errno.EINVAL, "not a regular file", path)
312
- with open(fd, encoding=encoding) as f:
313
- return f.read()
314
-
315
-
316
- def read_versioned(path: str, version: int, fallback_note: str = "") -> dict:
317
- """The store at `path` if this build can use it, `{}` otherwise.
318
-
319
- `{}` covers every unhappy case -- a missing file, an unreadable one, a
320
- non-object, and a version this build does not know. Fail SOFT, and that
321
- is a property of the callers this serves rather than of JSON: forgetting
322
- a remembered window or a remembered theme costs one launch on the
323
- derived value, and there is no security property to preserve. The trust
324
- store also fails soft in the *sense* that a damaged file reads as empty,
325
- but empty means "nothing is trusted" there, so it is fail-CLOSED and
326
- keeps its own reader.
327
-
328
- `fallback_note` is the caller's own clause for the debug line, so
329
- "using derived context windows" and "starting on the configured
330
- preferences" survive being one function.
331
- """
332
- if not os.path.exists(path):
333
- return {}
334
- try:
335
- with open(path, encoding="utf-8-sig") as f:
336
- data = json.load(f)
337
- except (OSError, ValueError):
338
- logger.debug("Could not read %s; %s", path,
339
- fallback_note or "falling back to defaults.")
340
- return {}
341
- if not isinstance(data, dict) or data.get("version") != version:
342
- return {}
343
- return data
1
+ """
2
+ json_store.py
3
+
4
+ The mechanics four user-tier JSON stores had four copies of.
5
+
6
+ AND THE FILE TOOLS' TEXT WRITES SINCE BATCH 146, which is why
7
+ `_replace_through_temp` exists under both public writers rather than the
8
+ dance living inside `write_json_atomic`. `write_text_atomic` says what it
9
+ needs the dance FOR, and it is not durability: `os.replace` replaces a name
10
+ rather than following it, which is the only thing that stops a write through
11
+ a hardlinked document name landing on the file the link shares. A sixth copy
12
+ of a temp-and-rename would have been the obvious way to get that, and this
13
+ module exists because four copies of it had already drifted.
14
+
15
+ AND THE PROJECT LOADER'S READS SINCE BATCH 219. `read_regular_text` is the
16
+ read half of the same kind of argument: a cloned project's `.venastine/`
17
+ files and root `AGENTS.md` are read by the trust prompt before anyone has
18
+ consented to them, and a skill symlinked to `/dev/zero` was read until
19
+ memory ran out. Six read sites in two packages needed the one guard, which
20
+ is the shape this module was made for.
21
+
22
+ WHAT THIS IS AND IS NOT. It owns *how* a small JSON file is read and
23
+ written -- the temp-file-plus-replace dance, the version guard, the
24
+ fail-soft read -- and nothing about what any of them MEAN. Each store keeps
25
+ its own module, its own path, its own STORE_VERSION and its own posture,
26
+ because those are the parts that were decided separately and are argued at
27
+ length where they live:
28
+
29
+ core/workspace_trust.py trusted_projects.json fails CLOSED, 0600,
30
+ unversioned (a flat path -> hash mapping)
31
+ mcp_client/config.py known_mcp_servers.json versioned, with F3's
32
+ legacy-v1 re-ask arm
33
+ core/model_windows.py model_windows.json versioned, fail soft
34
+ tui/preferences.py ui_preferences.json versioned, fail soft,
35
+ read-modify-write (RM4)
36
+
37
+ WHY IT IS AT THE ROOT. `core/model_windows.py` is read by
38
+ `core/compaction.py`; `tui/preferences.py` states that it imports nothing
39
+ from `core/`; `mcp_client/` and `security/` each document a reason not to
40
+ reach into `core/` either. A root module is the one place all four can
41
+ already see -- `config`, `storage`, `database` and `credentials` are
42
+ imported from everywhere for the same reason -- so sharing this costs no
43
+ new edge in any direction.
44
+
45
+ WHAT THE COPIES HAD ALREADY DRIFTED INTO. Three of the four wrote through a
46
+ temp file and `os.replace`, each spending four to eight docstring lines on
47
+ why a truncate-then-write is unacceptable: it leaves the file empty for the
48
+ length of the write and permanently damaged if the process dies inside it.
49
+ The fourth -- `mcp_client`'s, which stores a record of CONSENT, the same
50
+ class of data as the trust store -- wrote with a bare `open(path, "w")` and
51
+ said nothing. That is what a comment-enforced convention decays into, and
52
+ extracting the write is what makes it stop being a convention.
53
+
54
+ ERRORS BELONG TO THE CALLER. `write_json_atomic` raises; it does not decide
55
+ whether a failed write is a WARNING the user should see (model_windows,
56
+ preferences: yes, and return False) or an exception (workspace_trust: yes,
57
+ because a trust grant that silently did not persist is worse than a crash).
58
+ Each caller keeps the policy it already had.
59
+ """
60
+
61
+ from __future__ import annotations
62
+
63
+ import errno
64
+ import json
65
+ import logging
66
+ import os
67
+ import stat
68
+ from typing import Optional
69
+
70
+ logger = logging.getLogger(__name__)
71
+
72
+
73
+ class DestinationInUse(PermissionError):
74
+ """Windows refused the rename because another program has the file open.
75
+
76
+ Batch 219 (#242). This and a destination that is genuinely not writable
77
+ both arrive from `os.replace` as `PermissionError` WinError 5, and the
78
+ remedies are opposite: close it in the other program, or you may not
79
+ write here. Measured on Windows: a destination another process holds
80
+ open for reading still opens for `r+b` from here, and a read-only one
81
+ does not -- so `_held_open_elsewhere` tells them apart without writing a
82
+ byte. A subclass, so every `except OSError` that already reports a
83
+ failed write reports this one too, now saying what to do.
84
+ """
85
+
86
+
87
+ def write_json_atomic(path: str, payload, *, mode: Optional[int] = None,
88
+ indent: int = 2, sort_keys: bool = False,
89
+ trailing_newline: bool = False) -> None:
90
+ """Write `payload` as JSON to `path`, atomically. Raises OSError.
91
+
92
+ Temp file plus `os.replace`, which is atomic on POSIX and on Windows.
93
+ Parent directories are created.
94
+
95
+ `mode` sets permissions on the TEMP FILE, not on the destination (#19).
96
+ `os.replace` carries the source file's mode across, so setting it
97
+ afterwards would be both a no-op and a lie, and setting it on the
98
+ destination beforehand is worse -- replace overwrites that too.
99
+
100
+ EVERY CALLER HOLDING A CREDENTIAL PASSES ONE, and two of them pass it
101
+ for the reason the trust store explicitly does NOT: `credentials.py` and
102
+ `security/secrets.py` hold secrets, and 0600-at-creation is the property
103
+ their own comments argue for at length. `mcp_client/config.py` is the
104
+ fourth, for the `Authorization` header batch 140 named in
105
+ `.venastine/mcp.json`. This used to say "THREE CALLERS PASS ONE" and was written
106
+ when three did -- a count in prose with nothing able to notice the
107
+ fourth, which is the shape batch 143 closed twice on another surface and
108
+ answered the same way: the argument never needed the number, so it is
109
+ gone rather than corrected (batch 144).
110
+
111
+ `trailing_newline` is off by default so every existing store writes
112
+ the same bytes it always did. The four here are machine state nobody
113
+ opens; §24's `/init --config` scaffold is the first file written
114
+ through this that a human is meant to edit and COMMIT, and a
115
+ committed file with no final newline carries git's "No newline at end
116
+ of file" marker in every diff of it, forever after.
117
+ """
118
+ def _dump(handle) -> None:
119
+ json.dump(payload, handle, indent=indent, sort_keys=sort_keys)
120
+ if trailing_newline:
121
+ handle.write("\n")
122
+
123
+ # newline=None, which is what this has always passed and therefore what
124
+ # every stored file already holds: `json.dump` with an indent writes
125
+ # "\n" between its lines and the stream translates them, so on Windows
126
+ # these stores are CRLF on disk. Not a property worth having, and not
127
+ # one worth changing either -- it would rewrite the bytes of four user
128
+ # -tier stores to no end. The text caller below wants the opposite and
129
+ # says so there.
130
+ _replace_through_temp(path, _dump, mode=mode, newline=None)
131
+
132
+
133
+ def write_text_atomic(path: str, text: str, *, newline: str = "") -> None:
134
+ """Write `text` to `path`, atomically and byte-for-byte. Raises OSError.
135
+
136
+ THE FIFTH CONSUMER OF THE DANCE ABOVE, and the first that is not a JSON
137
+ store (batch 146). The file tools reach it for two reasons that are not
138
+ the ones the JSON stores have:
139
+
140
+ `newline=""` by default, because a tool that was handed content must
141
+ write the content it was handed. `open(path, "w")` translates every
142
+ "\n" to `os.linesep`, so on Windows `write` could not reproduce bytes
143
+ it had just read, and one `write_project_doc` call against an LF
144
+ document rewrote every line ending in it -- 2,910 of them in this
145
+ repository's own AGENTS.md -- under an approval that had been shown a
146
+ diff of one line.
147
+
148
+ And REPLACES RATHER THAN FOLLOWS, which is the property `.venastine/`
149
+ depends on. `os.path.realpath` cannot see a hardlink -- a hardlink is
150
+ not a link to anything, it IS the file -- so a containment test on the
151
+ resolved path admits a write through a hardlinked name, and a `write`
152
+ through one put `{"mcpServers": ...}` into a protected `settings.json`
153
+ whose direct spelling is refused. `os.replace` onto the name breaks the
154
+ link instead: the other name keeps its bytes and this one becomes a new
155
+ file. Measured both ways round.
156
+
157
+ The cost, stated because the caller cannot see it from here: a
158
+ destination another process merely has OPEN fails the rename on Windows,
159
+ where a truncating write would have succeeded. That is the same trade
160
+ `write_json_atomic` documents above, and the same answer -- a write that
161
+ fails loudly beats one that half-succeeds.
162
+ """
163
+ _replace_through_temp(path, lambda handle: handle.write(text),
164
+ newline=newline)
165
+
166
+
167
+ def _replace_through_temp(path: str, write_body, *,
168
+ mode=None, newline=None) -> None:
169
+ """Write through a unique temp file and `os.replace` it onto `path`.
170
+
171
+ ONE COPY OF THE MECHANICS, which is this module's whole reason for
172
+ existing -- `write_body` is handed an open text stream and writes
173
+ whatever the caller means; everything else here is the dance, and every
174
+ paragraph below was written against a failure that actually happened.
175
+
176
+ GUARDED, because `os.makedirs("")` raises rather than doing nothing
177
+ and `os.path.dirname` returns "" for a bare filename. No caller could
178
+ reach it while all six built an absolute path, but `AGENT_SECRETS_FILE`
179
+ and `AGENT_PROVIDERS_FILE` are documented overrides a user may set to
180
+ one -- so the two secret stores joining this helper (batch 108) is what
181
+ made a path SHAPE able to decide whether the write works.
182
+ """
183
+ parent = os.path.dirname(path)
184
+ if parent:
185
+ os.makedirs(parent, exist_ok=True)
186
+
187
+ # A UNIQUE NAME, not a fixed `<path>.tmp` sibling (batch 144).
188
+ # `config_edit.write` reached this conclusion for `config.yaml` and
189
+ # wrote down why -- "two writers raced on the one name, and a failed
190
+ # `os.replace` left the leftover in the install tree forever" -- and
191
+ # this helper, extracted so these mechanics would have exactly ONE
192
+ # copy, had neither half of the answer. One question, two answers, and
193
+ # nothing that could tell you they disagreed.
194
+ #
195
+ # Two instances in one directory is a supported configuration:
196
+ # `docs/PRIVACY.md` says "everything is relative to the directory you
197
+ # run in".
198
+ tmp = "%s.%s.tmp" % (path, os.urandom(4).hex())
199
+
200
+ # THE TEMP FILE IS SWEPT ON FAILURE (batch 108). Without this a write
201
+ # that dies inside `_dump` leaves its temp file behind for good: the
202
+ # destination is correctly untouched, which is the whole point, but the
203
+ # debris is not. It is worst for the two stores that joined this helper
204
+ # in the same batch -- a half-written SEALED ENVELOPE sitting next to
205
+ # the secrets file, and a partial provider file next to the real one.
206
+ #
207
+ # AND THE RENAME IS INSIDE IT (batch 144). `os.replace` used to
208
+ # be the statement after this handler, so the sweep covered the failure
209
+ # the comment above names and not the one next to it -- which is the
210
+ # repository's own prompt for defect shape 6: "what actually raises
211
+ # this, and what raises the thing next to it?"
212
+ #
213
+ # The rename's failure is not hypothetical and not symmetrical across
214
+ # platforms. On Windows a plain `open()` takes no FILE_SHARE_DELETE, so
215
+ # any other process merely READING the destination makes `os.replace`
216
+ # fail -- an editor, a sync client, a backup, a virus scanner, or a
217
+ # second instance in the same directory. Driven, with a happy-path
218
+ # control: `PermissionError WinError 5`, destination correctly intact,
219
+ # and `providers.json.tmp` left holding EVERY stored provider key in
220
+ # cleartext, because `save_credentials` reads the whole file, mutates
221
+ # one entry and writes all of it back. The POSIX arm is clean -- rename
222
+ # there ignores open handles and the destination's mode -- and the
223
+ # process-death window between the write and the rename is on both.
224
+ #
225
+ # Hygiene, never error policy: the original exception is re-raised
226
+ # unchanged, because "errors belong to the caller" above is the rule
227
+ # and a cleanup that swallowed one would be the opposite of it. The
228
+ # unlink is itself guarded, so a failure to tidy up cannot replace the
229
+ # failure worth reporting. The one rewording (batch 219) keeps that
230
+ # rule: a held-open destination is still a PermissionError, chained to
231
+ # the original, and only its message changes -- from one that reads as
232
+ # a permissions problem to the remedy, which is closing the file.
233
+ #
234
+ # ONE `os.open` rather than two branches, and `0o666` is not a new
235
+ # policy: it is the mode `open(path, "w")` passes, so the umask
236
+ # subtracts from it exactly as before. Measured on WSL at `umask 022`,
237
+ # both forms give 0o644 -- while `os.open` with NO mode argument gives
238
+ # 0o755, which is why the argument is explicit rather than omitted.
239
+ # `O_EXCL` on a unique name is what makes the sweep below safe: the
240
+ # file this unlinks is always one this call created, never another
241
+ # writer's.
242
+ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
243
+ fd = os.open(tmp, flags, 0o666 if mode is None else mode)
244
+ try:
245
+ with os.fdopen(fd, "w", encoding="utf-8", newline=newline) as f:
246
+ write_body(f)
247
+ os.replace(tmp, path)
248
+ except BaseException as exc:
249
+ try:
250
+ os.unlink(tmp)
251
+ except OSError:
252
+ pass # the temp file may not exist; the original error is what is raised
253
+ if _held_open_elsewhere(path, exc):
254
+ # No path in the message: every caller prefixes its own ("Could
255
+ # not write <path>: ..."), and a path here printed it twice.
256
+ raise DestinationInUse(
257
+ "it is open in another program (an editor, a sync client, a "
258
+ "backup or a virus scanner), so Windows will not replace it. "
259
+ "Close it there and try again.") from exc
260
+ raise
261
+
262
+
263
+ def _held_open_elsewhere(path: str, exc: BaseException) -> bool:
264
+ """Whether a failed rename onto `path` was another process's open handle.
265
+
266
+ Windows only: a POSIX rename ignores open handles. WinError 32 is a
267
+ sharing violation and says so itself; WinError 5 is ambiguous, and the
268
+ `r+b` open settles it (see `DestinationInUse`). The probe writes
269
+ nothing, and a destination that has gone or is a directory fails it.
270
+ """
271
+ if os.name != "nt" or not isinstance(exc, PermissionError):
272
+ return False
273
+ if getattr(exc, "winerror", None) == 32:
274
+ return True
275
+ try:
276
+ with open(path, "r+b"):
277
+ return True
278
+ except OSError as probe:
279
+ return getattr(probe, "winerror", None) == 32
280
+
281
+
282
+ def read_regular_text(path: str, encoding: str = "utf-8-sig") -> str:
283
+ """The whole text of `path`, which must be a regular file. Raises OSError.
284
+
285
+ For content a cloned project brought with it (batch 219, #214), which
286
+ the trust prompt reads BEFORE anyone has consented to it. Git stores a
287
+ symlink, `open()` follows one, and `/dev/zero`, `/dev/urandom` and
288
+ `/dev/tty` exist on every POSIX machine: measured, the listing read a
289
+ skill linked to `/dev/zero` until a 1 GiB cap raised MemoryError,
290
+ while the trust hash -- which skips non-regular files for exactly this
291
+ reason -- stepped over it. A FIFO is the other half: `open()` on one
292
+ blocks until a writer appears.
293
+
294
+ So the file is opened NON-BLOCKING and checked on the open descriptor.
295
+ A FIFO's open then returns at once, and `fstat` rather than a `stat` of
296
+ the name leaves no window for the name to be swapped between the check
297
+ and the read. A non-regular file raises OSError, which is what every
298
+ reader of project content already reports or skips. `O_BINARY` stops
299
+ Windows' C runtime translating the bytes under the decoder, which
300
+ `open()` would have done for itself.
301
+ """
302
+ flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)
303
+ fd = os.open(path, flags)
304
+ try:
305
+ regular = stat.S_ISREG(os.fstat(fd).st_mode)
306
+ except BaseException:
307
+ os.close(fd)
308
+ raise
309
+ if not regular:
310
+ os.close(fd)
311
+ raise OSError(errno.EINVAL, "not a regular file", path)
312
+ with open(fd, encoding=encoding) as f:
313
+ return f.read()
314
+
315
+
316
+ def read_versioned(path: str, version: int, fallback_note: str = "") -> dict:
317
+ """The store at `path` if this build can use it, `{}` otherwise.
318
+
319
+ `{}` covers every unhappy case -- a missing file, an unreadable one, a
320
+ non-object, and a version this build does not know. Fail SOFT, and that
321
+ is a property of the callers this serves rather than of JSON: forgetting
322
+ a remembered window or a remembered theme costs one launch on the
323
+ derived value, and there is no security property to preserve. The trust
324
+ store also fails soft in the *sense* that a damaged file reads as empty,
325
+ but empty means "nothing is trusted" there, so it is fail-CLOSED and
326
+ keeps its own reader.
327
+
328
+ `fallback_note` is the caller's own clause for the debug line, so
329
+ "using derived context windows" and "starting on the configured
330
+ preferences" survive being one function.
331
+ """
332
+ if not os.path.exists(path):
333
+ return {}
334
+ try:
335
+ with open(path, encoding="utf-8-sig") as f:
336
+ data = json.load(f)
337
+ except (OSError, ValueError):
338
+ logger.debug("Could not read %s; %s", path,
339
+ fallback_note or "falling back to defaults.")
340
+ return {}
341
+ if not isinstance(data, dict) or data.get("version") != version:
342
+ return {}
343
+ return data