@yahyahammad/venastine 0.2.0 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -4
- package/CONFIG_ARCHITECTURE.md +2030 -0
- package/LICENSE +201 -201
- package/NOTICE +1 -1
- package/README.md +1034 -895
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -88
- package/agents/builtin/review.md +74 -74
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +68 -2
- package/agents/subagent_tool.py +472 -238
- package/agents/tui_commands.py +238 -151
- package/bin/venastine.mjs +150 -4
- package/bin/venastine.py +71 -0
- package/cli_commands.py +838 -0
- package/config.py +95 -1198
- package/config.yaml +1103 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +971 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -118
- package/core/ansi.py +452 -0
- package/core/approval.py +137 -115
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +2715 -1559
- package/core/commands.py +460 -0
- package/core/compaction.py +1735 -971
- package/core/config_loader.py +1982 -1563
- package/core/connect.py +686 -0
- package/core/events.py +123 -61
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +411 -328
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1490
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +190 -176
- package/core/provider_errors.py +234 -0
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -468
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +1561 -1382
- package/core/reasoning/output_writer.py +345 -286
- package/core/reasoning/payload_validation.py +524 -398
- package/core/reasoning/pipeline_storage.py +88 -22
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +458 -434
- package/core/reasoning/source_corpus.py +433 -327
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -183
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +148 -20
- package/database.py +333 -252
- package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +50 -9
- package/json_store.py +343 -123
- package/logging_setup.py +295 -185
- package/main.py +3433 -1961
- package/mcp_client/client.py +733 -499
- package/mcp_client/config.py +126 -21
- package/mcp_client/registration.py +343 -182
- package/memories/manager.py +147 -132
- package/memories/tui_commands.py +319 -244
- package/package.json +303 -73
- package/project_init/config_files.py +282 -278
- package/project_init/generator.py +728 -564
- package/project_init/manifest.py +18 -6
- package/project_init/tui_commands.py +147 -144
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +2 -0
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -262
- package/providers.json.example +4 -2
- package/pyproject.toml +149 -100
- package/requirements.txt +295 -72
- package/safety/policy_enforcement.py +1082 -746
- package/security/askpass.py +419 -0
- package/security/capability.py +207 -177
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +237 -190
- package/security/posture.py +124 -12
- package/security/protected_paths.py +501 -309
- package/security/sandbox.py +4108 -985
- package/security/secrets.py +855 -0
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -111
- package/storage.py +1338 -1005
- package/tools/base.py +161 -1
- package/tools/builtin/_math_common.py +667 -377
- package/tools/builtin/_net_common.py +225 -19
- package/tools/builtin/arxiv.py +497 -39
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +371 -153
- package/tools/builtin/file_ops.py +720 -412
- package/tools/builtin/geometry.py +48 -12
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +76 -66
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +227 -193
- package/tools/builtin/probability_stats.py +12 -16
- package/tools/builtin/project_docs.py +464 -405
- package/tools/builtin/remember.py +175 -158
- package/tools/builtin/shell.py +980 -461
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +149 -147
- package/tools/builtin/web_search.py +191 -13
- package/tools/context.py +159 -136
- package/tools/isolation.py +107 -34
- package/tools/registry.py +1242 -925
- package/tui/app.py +8318 -4100
- package/tui/app.tcss +747 -556
- package/tui/commands.py +26 -170
- package/tui/diffs.py +34 -13
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -1279
- package/tui/meters.py +364 -268
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -943
- package/tui/themes.py +599 -559
- package/tui/widgets.py +3813 -2332
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/json_store.py
CHANGED
|
@@ -1,123 +1,343 @@
|
|
|
1
|
-
"""
|
|
2
|
-
json_store.py
|
|
3
|
-
|
|
4
|
-
The mechanics four user-tier JSON stores had four copies of.
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
1
|
+
"""
|
|
2
|
+
json_store.py
|
|
3
|
+
|
|
4
|
+
The mechanics four user-tier JSON stores had four copies of.
|
|
5
|
+
|
|
6
|
+
AND THE FILE TOOLS' TEXT WRITES SINCE BATCH 146, which is why
|
|
7
|
+
`_replace_through_temp` exists under both public writers rather than the
|
|
8
|
+
dance living inside `write_json_atomic`. `write_text_atomic` says what it
|
|
9
|
+
needs the dance FOR, and it is not durability: `os.replace` replaces a name
|
|
10
|
+
rather than following it, which is the only thing that stops a write through
|
|
11
|
+
a hardlinked document name landing on the file the link shares. A sixth copy
|
|
12
|
+
of a temp-and-rename would have been the obvious way to get that, and this
|
|
13
|
+
module exists because four copies of it had already drifted.
|
|
14
|
+
|
|
15
|
+
AND THE PROJECT LOADER'S READS SINCE BATCH 219. `read_regular_text` is the
|
|
16
|
+
read half of the same kind of argument: a cloned project's `.venastine/`
|
|
17
|
+
files and root `AGENTS.md` are read by the trust prompt before anyone has
|
|
18
|
+
consented to them, and a skill symlinked to `/dev/zero` was read until
|
|
19
|
+
memory ran out. Six read sites in two packages needed the one guard, which
|
|
20
|
+
is the shape this module was made for.
|
|
21
|
+
|
|
22
|
+
WHAT THIS IS AND IS NOT. It owns *how* a small JSON file is read and
|
|
23
|
+
written -- the temp-file-plus-replace dance, the version guard, the
|
|
24
|
+
fail-soft read -- and nothing about what any of them MEAN. Each store keeps
|
|
25
|
+
its own module, its own path, its own STORE_VERSION and its own posture,
|
|
26
|
+
because those are the parts that were decided separately and are argued at
|
|
27
|
+
length where they live:
|
|
28
|
+
|
|
29
|
+
core/workspace_trust.py trusted_projects.json fails CLOSED, 0600,
|
|
30
|
+
unversioned (a flat path -> hash mapping)
|
|
31
|
+
mcp_client/config.py known_mcp_servers.json versioned, with F3's
|
|
32
|
+
legacy-v1 re-ask arm
|
|
33
|
+
core/model_windows.py model_windows.json versioned, fail soft
|
|
34
|
+
tui/preferences.py ui_preferences.json versioned, fail soft,
|
|
35
|
+
read-modify-write (RM4)
|
|
36
|
+
|
|
37
|
+
WHY IT IS AT THE ROOT. `core/model_windows.py` is read by
|
|
38
|
+
`core/compaction.py`; `tui/preferences.py` states that it imports nothing
|
|
39
|
+
from `core/`; `mcp_client/` and `security/` each document a reason not to
|
|
40
|
+
reach into `core/` either. A root module is the one place all four can
|
|
41
|
+
already see -- `config`, `storage`, `database` and `credentials` are
|
|
42
|
+
imported from everywhere for the same reason -- so sharing this costs no
|
|
43
|
+
new edge in any direction.
|
|
44
|
+
|
|
45
|
+
WHAT THE COPIES HAD ALREADY DRIFTED INTO. Three of the four wrote through a
|
|
46
|
+
temp file and `os.replace`, each spending four to eight docstring lines on
|
|
47
|
+
why a truncate-then-write is unacceptable: it leaves the file empty for the
|
|
48
|
+
length of the write and permanently damaged if the process dies inside it.
|
|
49
|
+
The fourth -- `mcp_client`'s, which stores a record of CONSENT, the same
|
|
50
|
+
class of data as the trust store -- wrote with a bare `open(path, "w")` and
|
|
51
|
+
said nothing. That is what a comment-enforced convention decays into, and
|
|
52
|
+
extracting the write is what makes it stop being a convention.
|
|
53
|
+
|
|
54
|
+
ERRORS BELONG TO THE CALLER. `write_json_atomic` raises; it does not decide
|
|
55
|
+
whether a failed write is a WARNING the user should see (model_windows,
|
|
56
|
+
preferences: yes, and return False) or an exception (workspace_trust: yes,
|
|
57
|
+
because a trust grant that silently did not persist is worse than a crash).
|
|
58
|
+
Each caller keeps the policy it already had.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
from __future__ import annotations
|
|
62
|
+
|
|
63
|
+
import errno
|
|
64
|
+
import json
|
|
65
|
+
import logging
|
|
66
|
+
import os
|
|
67
|
+
import stat
|
|
68
|
+
from typing import Optional
|
|
69
|
+
|
|
70
|
+
logger = logging.getLogger(__name__)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class DestinationInUse(PermissionError):
|
|
74
|
+
"""Windows refused the rename because another program has the file open.
|
|
75
|
+
|
|
76
|
+
Batch 219 (#242). This and a destination that is genuinely not writable
|
|
77
|
+
both arrive from `os.replace` as `PermissionError` WinError 5, and the
|
|
78
|
+
remedies are opposite: close it in the other program, or you may not
|
|
79
|
+
write here. Measured on Windows: a destination another process holds
|
|
80
|
+
open for reading still opens for `r+b` from here, and a read-only one
|
|
81
|
+
does not -- so `_held_open_elsewhere` tells them apart without writing a
|
|
82
|
+
byte. A subclass, so every `except OSError` that already reports a
|
|
83
|
+
failed write reports this one too, now saying what to do.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def write_json_atomic(path: str, payload, *, mode: Optional[int] = None,
|
|
88
|
+
indent: int = 2, sort_keys: bool = False,
|
|
89
|
+
trailing_newline: bool = False) -> None:
|
|
90
|
+
"""Write `payload` as JSON to `path`, atomically. Raises OSError.
|
|
91
|
+
|
|
92
|
+
Temp file plus `os.replace`, which is atomic on POSIX and on Windows.
|
|
93
|
+
Parent directories are created.
|
|
94
|
+
|
|
95
|
+
`mode` sets permissions on the TEMP FILE, not on the destination (#19).
|
|
96
|
+
`os.replace` carries the source file's mode across, so setting it
|
|
97
|
+
afterwards would be both a no-op and a lie, and setting it on the
|
|
98
|
+
destination beforehand is worse -- replace overwrites that too.
|
|
99
|
+
|
|
100
|
+
EVERY CALLER HOLDING A CREDENTIAL PASSES ONE, and two of them pass it
|
|
101
|
+
for the reason the trust store explicitly does NOT: `credentials.py` and
|
|
102
|
+
`security/secrets.py` hold secrets, and 0600-at-creation is the property
|
|
103
|
+
their own comments argue for at length. `mcp_client/config.py` is the
|
|
104
|
+
fourth, for the `Authorization` header batch 140 named in
|
|
105
|
+
`.venastine/mcp.json`. This used to say "THREE CALLERS PASS ONE" and was written
|
|
106
|
+
when three did -- a count in prose with nothing able to notice the
|
|
107
|
+
fourth, which is the shape batch 143 closed twice on another surface and
|
|
108
|
+
answered the same way: the argument never needed the number, so it is
|
|
109
|
+
gone rather than corrected (batch 144).
|
|
110
|
+
|
|
111
|
+
`trailing_newline` is off by default so every existing store writes
|
|
112
|
+
the same bytes it always did. The four here are machine state nobody
|
|
113
|
+
opens; §24's `/init --config` scaffold is the first file written
|
|
114
|
+
through this that a human is meant to edit and COMMIT, and a
|
|
115
|
+
committed file with no final newline carries git's "No newline at end
|
|
116
|
+
of file" marker in every diff of it, forever after.
|
|
117
|
+
"""
|
|
118
|
+
def _dump(handle) -> None:
|
|
119
|
+
json.dump(payload, handle, indent=indent, sort_keys=sort_keys)
|
|
120
|
+
if trailing_newline:
|
|
121
|
+
handle.write("\n")
|
|
122
|
+
|
|
123
|
+
# newline=None, which is what this has always passed and therefore what
|
|
124
|
+
# every stored file already holds: `json.dump` with an indent writes
|
|
125
|
+
# "\n" between its lines and the stream translates them, so on Windows
|
|
126
|
+
# these stores are CRLF on disk. Not a property worth having, and not
|
|
127
|
+
# one worth changing either -- it would rewrite the bytes of four user
|
|
128
|
+
# -tier stores to no end. The text caller below wants the opposite and
|
|
129
|
+
# says so there.
|
|
130
|
+
_replace_through_temp(path, _dump, mode=mode, newline=None)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def write_text_atomic(path: str, text: str, *, newline: str = "") -> None:
|
|
134
|
+
"""Write `text` to `path`, atomically and byte-for-byte. Raises OSError.
|
|
135
|
+
|
|
136
|
+
THE FIFTH CONSUMER OF THE DANCE ABOVE, and the first that is not a JSON
|
|
137
|
+
store (batch 146). The file tools reach it for two reasons that are not
|
|
138
|
+
the ones the JSON stores have:
|
|
139
|
+
|
|
140
|
+
`newline=""` by default, because a tool that was handed content must
|
|
141
|
+
write the content it was handed. `open(path, "w")` translates every
|
|
142
|
+
"\n" to `os.linesep`, so on Windows `write` could not reproduce bytes
|
|
143
|
+
it had just read, and one `write_project_doc` call against an LF
|
|
144
|
+
document rewrote every line ending in it -- 2,910 of them in this
|
|
145
|
+
repository's own AGENTS.md -- under an approval that had been shown a
|
|
146
|
+
diff of one line.
|
|
147
|
+
|
|
148
|
+
And REPLACES RATHER THAN FOLLOWS, which is the property `.venastine/`
|
|
149
|
+
depends on. `os.path.realpath` cannot see a hardlink -- a hardlink is
|
|
150
|
+
not a link to anything, it IS the file -- so a containment test on the
|
|
151
|
+
resolved path admits a write through a hardlinked name, and a `write`
|
|
152
|
+
through one put `{"mcpServers": ...}` into a protected `settings.json`
|
|
153
|
+
whose direct spelling is refused. `os.replace` onto the name breaks the
|
|
154
|
+
link instead: the other name keeps its bytes and this one becomes a new
|
|
155
|
+
file. Measured both ways round.
|
|
156
|
+
|
|
157
|
+
The cost, stated because the caller cannot see it from here: a
|
|
158
|
+
destination another process merely has OPEN fails the rename on Windows,
|
|
159
|
+
where a truncating write would have succeeded. That is the same trade
|
|
160
|
+
`write_json_atomic` documents above, and the same answer -- a write that
|
|
161
|
+
fails loudly beats one that half-succeeds.
|
|
162
|
+
"""
|
|
163
|
+
_replace_through_temp(path, lambda handle: handle.write(text),
|
|
164
|
+
newline=newline)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _replace_through_temp(path: str, write_body, *,
|
|
168
|
+
mode=None, newline=None) -> None:
|
|
169
|
+
"""Write through a unique temp file and `os.replace` it onto `path`.
|
|
170
|
+
|
|
171
|
+
ONE COPY OF THE MECHANICS, which is this module's whole reason for
|
|
172
|
+
existing -- `write_body` is handed an open text stream and writes
|
|
173
|
+
whatever the caller means; everything else here is the dance, and every
|
|
174
|
+
paragraph below was written against a failure that actually happened.
|
|
175
|
+
|
|
176
|
+
GUARDED, because `os.makedirs("")` raises rather than doing nothing
|
|
177
|
+
and `os.path.dirname` returns "" for a bare filename. No caller could
|
|
178
|
+
reach it while all six built an absolute path, but `AGENT_SECRETS_FILE`
|
|
179
|
+
and `AGENT_PROVIDERS_FILE` are documented overrides a user may set to
|
|
180
|
+
one -- so the two secret stores joining this helper (batch 108) is what
|
|
181
|
+
made a path SHAPE able to decide whether the write works.
|
|
182
|
+
"""
|
|
183
|
+
parent = os.path.dirname(path)
|
|
184
|
+
if parent:
|
|
185
|
+
os.makedirs(parent, exist_ok=True)
|
|
186
|
+
|
|
187
|
+
# A UNIQUE NAME, not a fixed `<path>.tmp` sibling (batch 144).
|
|
188
|
+
# `config_edit.write` reached this conclusion for `config.yaml` and
|
|
189
|
+
# wrote down why -- "two writers raced on the one name, and a failed
|
|
190
|
+
# `os.replace` left the leftover in the install tree forever" -- and
|
|
191
|
+
# this helper, extracted so these mechanics would have exactly ONE
|
|
192
|
+
# copy, had neither half of the answer. One question, two answers, and
|
|
193
|
+
# nothing that could tell you they disagreed.
|
|
194
|
+
#
|
|
195
|
+
# Two instances in one directory is a supported configuration:
|
|
196
|
+
# `docs/PRIVACY.md` says "everything is relative to the directory you
|
|
197
|
+
# run in".
|
|
198
|
+
tmp = "%s.%s.tmp" % (path, os.urandom(4).hex())
|
|
199
|
+
|
|
200
|
+
# THE TEMP FILE IS SWEPT ON FAILURE (batch 108). Without this a write
|
|
201
|
+
# that dies inside `_dump` leaves its temp file behind for good: the
|
|
202
|
+
# destination is correctly untouched, which is the whole point, but the
|
|
203
|
+
# debris is not. It is worst for the two stores that joined this helper
|
|
204
|
+
# in the same batch -- a half-written SEALED ENVELOPE sitting next to
|
|
205
|
+
# the secrets file, and a partial provider file next to the real one.
|
|
206
|
+
#
|
|
207
|
+
# AND THE RENAME IS INSIDE IT (batch 144). `os.replace` used to
|
|
208
|
+
# be the statement after this handler, so the sweep covered the failure
|
|
209
|
+
# the comment above names and not the one next to it -- which is the
|
|
210
|
+
# repository's own prompt for defect shape 6: "what actually raises
|
|
211
|
+
# this, and what raises the thing next to it?"
|
|
212
|
+
#
|
|
213
|
+
# The rename's failure is not hypothetical and not symmetrical across
|
|
214
|
+
# platforms. On Windows a plain `open()` takes no FILE_SHARE_DELETE, so
|
|
215
|
+
# any other process merely READING the destination makes `os.replace`
|
|
216
|
+
# fail -- an editor, a sync client, a backup, a virus scanner, or a
|
|
217
|
+
# second instance in the same directory. Driven, with a happy-path
|
|
218
|
+
# control: `PermissionError WinError 5`, destination correctly intact,
|
|
219
|
+
# and `providers.json.tmp` left holding EVERY stored provider key in
|
|
220
|
+
# cleartext, because `save_credentials` reads the whole file, mutates
|
|
221
|
+
# one entry and writes all of it back. The POSIX arm is clean -- rename
|
|
222
|
+
# there ignores open handles and the destination's mode -- and the
|
|
223
|
+
# process-death window between the write and the rename is on both.
|
|
224
|
+
#
|
|
225
|
+
# Hygiene, never error policy: the original exception is re-raised
|
|
226
|
+
# unchanged, because "errors belong to the caller" above is the rule
|
|
227
|
+
# and a cleanup that swallowed one would be the opposite of it. The
|
|
228
|
+
# unlink is itself guarded, so a failure to tidy up cannot replace the
|
|
229
|
+
# failure worth reporting. The one rewording (batch 219) keeps that
|
|
230
|
+
# rule: a held-open destination is still a PermissionError, chained to
|
|
231
|
+
# the original, and only its message changes -- from one that reads as
|
|
232
|
+
# a permissions problem to the remedy, which is closing the file.
|
|
233
|
+
#
|
|
234
|
+
# ONE `os.open` rather than two branches, and `0o666` is not a new
|
|
235
|
+
# policy: it is the mode `open(path, "w")` passes, so the umask
|
|
236
|
+
# subtracts from it exactly as before. Measured on WSL at `umask 022`,
|
|
237
|
+
# both forms give 0o644 -- while `os.open` with NO mode argument gives
|
|
238
|
+
# 0o755, which is why the argument is explicit rather than omitted.
|
|
239
|
+
# `O_EXCL` on a unique name is what makes the sweep below safe: the
|
|
240
|
+
# file this unlinks is always one this call created, never another
|
|
241
|
+
# writer's.
|
|
242
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
243
|
+
fd = os.open(tmp, flags, 0o666 if mode is None else mode)
|
|
244
|
+
try:
|
|
245
|
+
with os.fdopen(fd, "w", encoding="utf-8", newline=newline) as f:
|
|
246
|
+
write_body(f)
|
|
247
|
+
os.replace(tmp, path)
|
|
248
|
+
except BaseException as exc:
|
|
249
|
+
try:
|
|
250
|
+
os.unlink(tmp)
|
|
251
|
+
except OSError:
|
|
252
|
+
pass # the temp file may not exist; the original error is what is raised
|
|
253
|
+
if _held_open_elsewhere(path, exc):
|
|
254
|
+
# No path in the message: every caller prefixes its own ("Could
|
|
255
|
+
# not write <path>: ..."), and a path here printed it twice.
|
|
256
|
+
raise DestinationInUse(
|
|
257
|
+
"it is open in another program (an editor, a sync client, a "
|
|
258
|
+
"backup or a virus scanner), so Windows will not replace it. "
|
|
259
|
+
"Close it there and try again.") from exc
|
|
260
|
+
raise
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _held_open_elsewhere(path: str, exc: BaseException) -> bool:
|
|
264
|
+
"""Whether a failed rename onto `path` was another process's open handle.
|
|
265
|
+
|
|
266
|
+
Windows only: a POSIX rename ignores open handles. WinError 32 is a
|
|
267
|
+
sharing violation and says so itself; WinError 5 is ambiguous, and the
|
|
268
|
+
`r+b` open settles it (see `DestinationInUse`). The probe writes
|
|
269
|
+
nothing, and a destination that has gone or is a directory fails it.
|
|
270
|
+
"""
|
|
271
|
+
if os.name != "nt" or not isinstance(exc, PermissionError):
|
|
272
|
+
return False
|
|
273
|
+
if getattr(exc, "winerror", None) == 32:
|
|
274
|
+
return True
|
|
275
|
+
try:
|
|
276
|
+
with open(path, "r+b"):
|
|
277
|
+
return True
|
|
278
|
+
except OSError as probe:
|
|
279
|
+
return getattr(probe, "winerror", None) == 32
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def read_regular_text(path: str, encoding: str = "utf-8-sig") -> str:
|
|
283
|
+
"""The whole text of `path`, which must be a regular file. Raises OSError.
|
|
284
|
+
|
|
285
|
+
For content a cloned project brought with it (batch 219, #214), which
|
|
286
|
+
the trust prompt reads BEFORE anyone has consented to it. Git stores a
|
|
287
|
+
symlink, `open()` follows one, and `/dev/zero`, `/dev/urandom` and
|
|
288
|
+
`/dev/tty` exist on every POSIX machine: measured, the listing read a
|
|
289
|
+
skill linked to `/dev/zero` until a 1 GiB cap raised MemoryError,
|
|
290
|
+
while the trust hash -- which skips non-regular files for exactly this
|
|
291
|
+
reason -- stepped over it. A FIFO is the other half: `open()` on one
|
|
292
|
+
blocks until a writer appears.
|
|
293
|
+
|
|
294
|
+
So the file is opened NON-BLOCKING and checked on the open descriptor.
|
|
295
|
+
A FIFO's open then returns at once, and `fstat` rather than a `stat` of
|
|
296
|
+
the name leaves no window for the name to be swapped between the check
|
|
297
|
+
and the read. A non-regular file raises OSError, which is what every
|
|
298
|
+
reader of project content already reports or skips. `O_BINARY` stops
|
|
299
|
+
Windows' C runtime translating the bytes under the decoder, which
|
|
300
|
+
`open()` would have done for itself.
|
|
301
|
+
"""
|
|
302
|
+
flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_BINARY", 0)
|
|
303
|
+
fd = os.open(path, flags)
|
|
304
|
+
try:
|
|
305
|
+
regular = stat.S_ISREG(os.fstat(fd).st_mode)
|
|
306
|
+
except BaseException:
|
|
307
|
+
os.close(fd)
|
|
308
|
+
raise
|
|
309
|
+
if not regular:
|
|
310
|
+
os.close(fd)
|
|
311
|
+
raise OSError(errno.EINVAL, "not a regular file", path)
|
|
312
|
+
with open(fd, encoding=encoding) as f:
|
|
313
|
+
return f.read()
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def read_versioned(path: str, version: int, fallback_note: str = "") -> dict:
|
|
317
|
+
"""The store at `path` if this build can use it, `{}` otherwise.
|
|
318
|
+
|
|
319
|
+
`{}` covers every unhappy case -- a missing file, an unreadable one, a
|
|
320
|
+
non-object, and a version this build does not know. Fail SOFT, and that
|
|
321
|
+
is a property of the callers this serves rather than of JSON: forgetting
|
|
322
|
+
a remembered window or a remembered theme costs one launch on the
|
|
323
|
+
derived value, and there is no security property to preserve. The trust
|
|
324
|
+
store also fails soft in the *sense* that a damaged file reads as empty,
|
|
325
|
+
but empty means "nothing is trusted" there, so it is fail-CLOSED and
|
|
326
|
+
keeps its own reader.
|
|
327
|
+
|
|
328
|
+
`fallback_note` is the caller's own clause for the debug line, so
|
|
329
|
+
"using derived context windows" and "starting on the configured
|
|
330
|
+
preferences" survive being one function.
|
|
331
|
+
"""
|
|
332
|
+
if not os.path.exists(path):
|
|
333
|
+
return {}
|
|
334
|
+
try:
|
|
335
|
+
with open(path, encoding="utf-8-sig") as f:
|
|
336
|
+
data = json.load(f)
|
|
337
|
+
except (OSError, ValueError):
|
|
338
|
+
logger.debug("Could not read %s; %s", path,
|
|
339
|
+
fallback_note or "falling back to defaults.")
|
|
340
|
+
return {}
|
|
341
|
+
if not isinstance(data, dict) or data.get("version") != version:
|
|
342
|
+
return {}
|
|
343
|
+
return data
|