@yahyahammad/venastine 0.1.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +249 -93
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +280 -23
  12. package/agents/tui_commands.py +31 -21
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +256 -0
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1249 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +1735 -959
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +2566 -1353
  38. package/core/memory.py +482 -370
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +315 -167
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1672 -198
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +21 -14
  70. package/package.json +303 -73
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +149 -100
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +1082 -682
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +1338 -1005
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +374 -29
  118. package/tui/app.py +5281 -397
  119. package/tui/app.tcss +290 -9
  120. package/tui/commands.py +26 -61
  121. package/tui/diffs.py +286 -265
  122. package/tui/history.py +128 -0
  123. package/tui/hosted_driver.py +102 -0
  124. package/tui/markdown.py +1329 -0
  125. package/tui/meters.py +364 -0
  126. package/tui/screens.py +346 -92
  127. package/tui/themes.py +599 -366
  128. package/tui/widgets.py +2673 -116
  129. package/SECURITY.md +0 -88
  130. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,61 +1,267 @@
1
- """
2
- tools/builtin/_net_common.py
3
-
4
- What the two network-backed search tools share. `_math_common.py`'s
5
- precedent, one directory over: a private module the built-in tools import
6
- from, never registered and never advertised.
7
-
8
- TODAY THAT IS ONE THING: the in-process response cache. `web_search.py` and
9
- `arxiv.py` each carried a `_cache_get` / `_cache_set` pair that was
10
- identical apart from the type of the value being stored, plus a
11
- `CACHE_TTL_S = 300` each, plus a bare dict each. Three copies of an expiry
12
- rule is how one of them comes to be measured in minutes.
13
-
14
- WHAT IS DELIBERATELY *NOT* SHARED. Each tool keeps its OWN instance, so the
15
- two key spaces stay separate -- a `TTLCache` shared between them would let
16
- an arXiv query and a web query collide on an equal key string, which is a
17
- correctness bug rather than a tidiness one. Their `@field_validator`s
18
- (`strip_keywords`, `strip_query`) are also left alone: they are three lines
19
- each, they differ in the message the model is shown, and folding them into
20
- a factory would make the params class harder to read to save two lines.
21
- """
22
-
23
- from __future__ import annotations
24
-
25
- import time
26
- from typing import Any, Optional
27
-
28
-
29
- class TTLCache:
30
- """A tiny time-boxed cache, keyed by whatever string a caller builds.
31
-
32
- Expiry is checked on READ rather than swept: these caches hold a
33
- handful of entries for the length of a research run, so a sweep would
34
- be machinery for a problem neither tool has.
35
-
36
- A stale entry is dropped as it is read, which is what makes a repeated
37
- query after the TTL a real request again rather than a permanent miss
38
- against a growing dict.
39
- """
40
-
41
- def __init__(self, ttl_s: float) -> None:
42
- self.ttl_s = ttl_s
43
- self._entries: dict[str, tuple[float, Any]] = {}
44
-
45
- def get(self, key: str) -> Optional[Any]:
46
- entry = self._entries.get(key)
47
- if not entry:
48
- return None
49
- stored_at, value = entry
50
- if time.time() - stored_at > self.ttl_s:
51
- self._entries.pop(key, None)
52
- return None
53
- return value
54
-
55
- def set(self, key: str, value: Any) -> None:
56
- self._entries[key] = (time.time(), value)
57
-
58
- def clear(self) -> None:
59
- """Drop everything. For tests, which must not inherit a previous
60
- test's answers -- the same reason conftest isolates the stores."""
61
- self._entries.clear()
1
+ """
2
+ tools/builtin/_net_common.py
3
+
4
+ What the two network-backed search tools share. `_math_common.py`'s
5
+ precedent, one directory over: a private module the built-in tools import
6
+ from, never registered and never advertised.
7
+
8
+ TWO THINGS NOW. The first is the in-process response cache: `web_search.py`
9
+ and `arxiv.py` each carried a `_cache_get` / `_cache_set` pair that was
10
+ identical apart from the type of the value being stored, plus a
11
+ `CACHE_TTL_S = 300` each, plus a bare dict each. Three copies of an expiry
12
+ rule is how one of them comes to be measured in minutes.
13
+
14
+ The second is `ThirdPartyText` (batch 156, ROADMAP_v3 §73), which is where
15
+ the two search tools now bound and redact the strings they did not write.
16
+ It is here rather than in either tool for the reason the cache is: both
17
+ were doing the same thing to the same kind of value, and the one that got
18
+ it wrong got it wrong in the half the other had already fixed.
19
+
20
+ There are THREE callers now -- `fetch_url.py` joined them in ROADMAP_v3 §51 --
21
+ and the third is why §52 gave this class a size bound as well as a time one.
22
+
23
+ WHAT IS DELIBERATELY *NOT* SHARED. Each tool keeps its OWN instance, so the
24
+ two key spaces stay separate -- a `TTLCache` shared between them would let
25
+ an arXiv query and a web query collide on an equal key string, which is a
26
+ correctness bug rather than a tidiness one. Their `@field_validator`s
27
+ (`strip_keywords`, `strip_query`) are also left alone: they are three lines
28
+ each, they differ in the message the model is shown, and folding them into
29
+ a factory would make the params class harder to read to save two lines.
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import logging
35
+ import threading
36
+ import time
37
+ from typing import Any, Optional
38
+
39
+ from safety.policy_enforcement import redact_output_text
40
+
41
+ logger = logging.getLogger(__name__)
42
+
43
+ # ROADMAP_v3 §52 (RB6). How many entries any one of these may hold.
44
+ #
45
+ # Derived rather than picked: `config.MAX_ITERATIONS` is 50, so one pass can
46
+ # make at most 50 tool calls, and a `fetch_url` whose URL redirects stores
47
+ # TWO keys for one body (the requested spelling and the answering one). 100
48
+ # keys is therefore the most a single pass can reach, and this sits above it
49
+ # so that the cap bounds a PROCESS -- which runs many passes and many runs --
50
+ # without ever biting inside one.
51
+ #
52
+ # That "without ever biting" is the requirement, not a nicety. A `fetch_url`
53
+ # page-2 request that misses degrades to a re-fetch, which is exactly the
54
+ # spliced-document failure §51 measured and built PG5 to prevent, so an
55
+ # eviction that fires in ordinary use would trade a bounded memory cost for
56
+ # an unbounded correctness one.
57
+ #
58
+ # In entries and not in bytes, because a generic cache cannot measure the
59
+ # size of what a caller hands it. For the caller that made this necessary the
60
+ # two are interchangeable anyway: `fetch_url`'s MAX_CONTENT_BYTES holds each
61
+ # body to ~64 KB whatever the encoding, so PG6's refusal to raise that cap is
62
+ # what turns this count into a memory bound at all.
63
+ MAX_ENTRIES = 128
64
+
65
+
66
+ # ROADMAP_v3 §73 (A58). How long a URL this harness is willing to SHOW the
67
+ # model may be.
68
+ #
69
+ # DERIVED, NOT PICKED, and the derivation is the argument for dropping an
70
+ # over-long URL rather than cutting it: `fetch_url` already bounds the URL
71
+ # the model may ASK for, at `max_length=400` on `FetchURLParams.url`.
72
+ # Measured (batch 156), a 401-character URL comes back from fetch_url's own
73
+ # gate as an "Invalid arguments" refusal while a 400-character one is
74
+ # accepted. So a longer URL is one THIS HARNESS refuses to take, and
75
+ # offering it in a search result is offering a route to nowhere.
76
+ #
77
+ # ONE DEFINITION: `fetch_url` reads it from here rather than spelling 400
78
+ # again. The two were the same number in two files, which is the state a
79
+ # pair of numbers is in just before it stops being a pair.
80
+ MAX_URL_CHARS = 400
81
+
82
+
83
+ class ThirdPartyText:
84
+ """One tool call's worth of strings nobody here wrote, REDACTED and
85
+ THEN cut.
86
+
87
+ ORDER IS THE WHOLE POINT, and it is RB1's point restated for the two
88
+ tools RB1 did not reach. `fetch_url` redacts its whole body above both
89
+ the hold and the slice, because a pattern only matches what it can see
90
+ whole; `web_search` and `arxiv_search` cut first and let
91
+ `registry.dispatch`'s `check_output_policy` redact the piece that was
92
+ kept. A fixed-length credential straddling such a cut is then invisible
93
+ to both halves -- the kept prefix is not the token and the dropped
94
+ suffix is gone. Measured (batch 156) at all three of their cut points:
95
+ a `ghp_` token survived post-hoc redaction at 36 of the 39 offsets it
96
+ could straddle, and redacting the whole value first caught every one.
97
+
98
+ TWO KINDS OF VALUE, because a cut means different things to them (A58).
99
+ `text()` cuts and flags: half an abstract is still an abstract, and PG4
100
+ says the flag is owed. `url()` DROPS and flags: half a URL is not a
101
+ URL, it is a fetchable-looking string that is not the resource, and
102
+ `pdf_url` and arXiv's `full_*_url` pair exist precisely to be the route
103
+ the model takes to recover what was cut. A broken route offered as a
104
+ route is worse than an absent one, which the flag makes visible.
105
+
106
+ TOTAL OVER WHATEVER THE PROVIDER EMITTED, for the reason `tools/base.py`
107
+ gives about tool arguments: a key that is present with a wrong-typed
108
+ value is not an absent key, and `r.get("title", "(untitled)")` returns
109
+ None for `{"title": null}` rather than the default. Both accessors
110
+ answer None for a non-string instead of raising, and the caller decides
111
+ what an absent value means -- which is a decision the two tools make
112
+ differently and should (A61).
113
+
114
+ ONE WARNING PER CALL, not per value (#49, RB2). Redacting here means
115
+ `check_output_policy` finds nothing left to alter, so the obligation to
116
+ say that something WAS altered moves here with the redaction; and a
117
+ feed with twenty-five authors must not produce twenty-five warning
118
+ lines. The count is carried on the instance and `report()` spends it.
119
+ The match is never echoed, for the reason `check_output_policy` and
120
+ `fetch_url._redacted` both state: this function exists because
121
+ something credential-shaped was found, so quoting it to say so would
122
+ put it in app.log and every sink a WARNING reaches.
123
+ """
124
+
125
+ def __init__(self, tool: str) -> None:
126
+ self._tool = tool
127
+ self._redacted = 0
128
+
129
+ def text(self, value: Any, cap: int) -> tuple:
130
+ """`(text, truncated)` -- `value` redacted and THEN cut to `cap`.
131
+
132
+ `(None, False)` for anything that is not a string, so a caller can
133
+ tell "the provider did not give me this" from "it gave me an empty
134
+ one" and choose its own substitute.
135
+
136
+ THE FLAG COUNTS REDACTED CHARACTERS, which is RB3's consequence
137
+ arriving here: redaction can SHRINK a value past the cap, and when
138
+ it does there is nothing further the cut is withholding. A flag
139
+ measured on the raw length would claim there is.
140
+ """
141
+ if not isinstance(value, str):
142
+ return None, False
143
+ clean = self._clean(value)
144
+ return clean[:cap], len(clean) > cap
145
+
146
+ def url(self, value: Any, cap: int = MAX_URL_CHARS) -> tuple:
147
+ """`(url, withheld)` -- a URL past `cap` is DROPPED, not cut (A58).
148
+
149
+ `(None, False)` for a non-string or an empty one: nothing was
150
+ withheld, there was nothing there. `(None, True)` says a URL was
151
+ found and is not being shown, which is the fact the flag carries.
152
+ """
153
+ if not isinstance(value, str) or not value:
154
+ return None, False
155
+ clean = self._clean(value)
156
+ if len(clean) > cap:
157
+ return None, True
158
+ return clean, False
159
+
160
+ def _clean(self, value: str) -> str:
161
+ clean = redact_output_text(value)
162
+ if clean != value:
163
+ self._redacted += 1
164
+ return clean
165
+
166
+ def report(self) -> None:
167
+ """The one warning this call owes (#49), or silence."""
168
+ if self._redacted:
169
+ logger.warning(
170
+ "Output policy altered %s result: credentials replaced in "
171
+ "%d value(s) before they were cut.",
172
+ self._tool, self._redacted)
173
+
174
+
175
+ class TTLCache:
176
+ """A tiny time-boxed cache, keyed by whatever string a caller builds.
177
+
178
+ TWO BOUNDS, ANSWERING DIFFERENT QUESTIONS. The TTL is about FRESHNESS: a
179
+ stale entry is dropped as it is read, which is what makes a repeated
180
+ query after the TTL a real request again rather than a permanent miss
181
+ against a growing dict. `max_entries` is about SIZE, and it is the only
182
+ thing that bounds a cache nothing reads a second time.
183
+
184
+ THE SWEEP ON WRITE IS §52 (RB5), AND THE REASON THIS DOCSTRING USED TO
185
+ GIVE FOR NOT HAVING ONE IS NOW FALSE. It said a sweep "would be machinery
186
+ for a problem neither tool has" -- true of two callers holding result
187
+ lists, and not of the third, which holds bodies. Measured (batch 106):
188
+ 100 fetches of a capped page held 6.3 MB and not one entry was ever
189
+ dropped, because expiry ran only in `get` and nothing read those keys
190
+ again. Expiring on read alone means the TTL bounds staleness and nothing
191
+ whatever bounds memory.
192
+
193
+ READ IS DELIBERATELY UNCHANGED. `get` still expires the one key it
194
+ touches, because that path is what makes a post-TTL query a real request
195
+ again; the sweep is additional, not a replacement, and a cache that only
196
+ swept would keep serving a stale entry until someone else wrote.
197
+ """
198
+
199
+ def __init__(self, ttl_s: float, max_entries: int = MAX_ENTRIES) -> None:
200
+ self.ttl_s = ttl_s
201
+ self.max_entries = max_entries
202
+ self._entries: dict[str, tuple[float, Any]] = {}
203
+ # LOCKED BECAUSE THE REGISTRY ALREADY SAYS IT IS (batch 117).
204
+ # `spawn_subagent` is the one tool with `parallel=True`, and the
205
+ # comment beside that field claims "everything the child's run
206
+ # touches was made concurrency-safe first -- the budget's lock, the
207
+ # serialised question, the per-future authorization". The child
208
+ # reaches this cache through `arxiv_search` and `fetch_url`, and it
209
+ # was the one item on that list without a lock.
210
+ #
211
+ # Single dict operations are atomic under CPython, so a corrupt
212
+ # dict was never the risk. `set` is a read-modify-write -- pop,
213
+ # assign, sweep, then evict while over the cap -- so two children
214
+ # storing at once could lose one entry and evict in an order
215
+ # neither asked for. Mirrors `GrantBudget.take()`, which is the
216
+ # pattern this module should have followed from the start.
217
+ #
218
+ # NOT reentrant, and `_sweep` is the reason: it is called from
219
+ # `set` with the lock already held, so it must never take it.
220
+ self._lock = threading.Lock()
221
+
222
+ def get(self, key: str) -> Optional[Any]:
223
+ with self._lock:
224
+ entry = self._entries.get(key)
225
+ if not entry:
226
+ return None
227
+ stored_at, value = entry
228
+ if time.time() - stored_at > self.ttl_s:
229
+ self._entries.pop(key, None)
230
+ return None
231
+ return value
232
+
233
+ def _sweep(self, now: float) -> None:
234
+ """Drop everything already expired. Write-side only (§52, RB5).
235
+
236
+ CALLED WITH `self._lock` HELD, and it must not take it: the lock is
237
+ a plain `Lock`, so re-entering would deadlock the caller.
238
+ """
239
+ stale = [key for key, (stored_at, _) in self._entries.items()
240
+ if now - stored_at > self.ttl_s]
241
+ for key in stale:
242
+ del self._entries[key]
243
+
244
+ def set(self, key: str, value: Any) -> None:
245
+ with self._lock:
246
+ self._set_locked(key, value)
247
+
248
+ def _set_locked(self, key: str, value: Any) -> None:
249
+ now = time.time()
250
+ # POPPED FIRST so a re-store moves the key to the END of the dict.
251
+ # Assigning to an existing key keeps its original insertion position,
252
+ # which would make the freshest entry the next one evicted -- and the
253
+ # freshest entry is the one a caller is most likely to read next.
254
+ self._entries.pop(key, None)
255
+ self._entries[key] = (now, value)
256
+ self._sweep(now)
257
+ # Insertion order IS age order, since every entry is stamped as it is
258
+ # written and the TTL is uniform. So the oldest key is simply the
259
+ # first one, and no second ordering has to be kept consistent.
260
+ while len(self._entries) > self.max_entries:
261
+ self._entries.pop(next(iter(self._entries)))
262
+
263
+ def clear(self) -> None:
264
+ """Drop everything. For tests, which must not inherit a previous
265
+ test's answers -- the same reason conftest isolates the stores."""
266
+ with self._lock:
267
+ self._entries.clear()