@yahyahammad/venastine 0.1.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +249 -93
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +280 -23
  12. package/agents/tui_commands.py +31 -21
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +256 -0
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1249 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +1735 -959
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +2566 -1353
  38. package/core/memory.py +482 -370
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +315 -167
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1672 -198
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +21 -14
  70. package/package.json +303 -73
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +149 -100
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +1082 -682
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +1338 -1005
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +374 -29
  118. package/tui/app.py +5281 -397
  119. package/tui/app.tcss +290 -9
  120. package/tui/commands.py +26 -61
  121. package/tui/diffs.py +286 -265
  122. package/tui/history.py +128 -0
  123. package/tui/hosted_driver.py +102 -0
  124. package/tui/markdown.py +1329 -0
  125. package/tui/meters.py +364 -0
  126. package/tui/screens.py +346 -92
  127. package/tui/themes.py +599 -366
  128. package/tui/widgets.py +2673 -116
  129. package/SECURITY.md +0 -88
  130. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -5,11 +5,16 @@ import logging
5
5
  import httpx
6
6
  from pydantic import BaseModel, Field
7
7
 
8
- from safety.policy_enforcement import is_url_permitted
8
+ from safety.policy_enforcement import is_url_permitted, redact_output_text
9
+ from tools.builtin._net_common import MAX_URL_CHARS, TTLCache
9
10
 
10
11
  logger = logging.getLogger(__name__)
11
12
 
12
- _TOOL_DESCRIPTION = "Retrieve the contents of a web page using a URL."
13
+ _TOOL_DESCRIPTION = (
14
+ "Retrieve the contents of a web page using a URL. Long pages are "
15
+ "returned one page at a time -- when a result says it was truncated, "
16
+ "its message names the offset to pass for the next page."
17
+ )
13
18
  MAX_CONTENT_CHARS = 5000
14
19
 
15
20
  # ROADMAP_v2 §31 (H7), #55. How many BYTES may be pulled off the wire
@@ -34,9 +39,60 @@ REQUEST_TIMEOUT_S = 8.0
34
39
  # making, and every hop costs a policy check and a DNS resolution.
35
40
  MAX_REDIRECTS = 5
36
41
 
42
+ # ROADMAP_v3 §51 (PG5). How long a fetched body stays available to be paged
43
+ # through. The same 300 s `arxiv.py` and `web_search.py` each use, and for a
44
+ # weaker reason than theirs: they cache to avoid repeating a query, this
45
+ # caches so that page 2 IS page 2.
46
+ #
47
+ # MEASURED (batch 105), which is why this is a cache and not a re-fetch.
48
+ # Five real URLs fetched twice 45 seconds apart -- the interval a model
49
+ # actually pages over, not the milliseconds an eager probe measures:
50
+ # four were byte-identical and `news.ycombinator.com` was not. Its first
51
+ # difference landed at byte 2,019, INSIDE page 1, and the text at
52
+ # offset 5000..10000 differed between the two fetches. So a model that read
53
+ # page 1 and asked for page 2 would have been handed two halves of two
54
+ # different documents, spliced at an arbitrary point, with nothing in the
55
+ # result able to say so. A news front page is exactly what a research run
56
+ # reads.
57
+ #
58
+ # The second measured reason is cost. Paging an 88 KB document by re-fetch
59
+ # is 14 requests for the same body: 1.2 MB off the wire to read 65 KB, and
60
+ # 5.3 MB for `bbc.com/news`. The cache holds at most the byte cap per URL.
61
+ CACHE_TTL_S = 300
62
+
63
+ # Keyed by URL -- BOTH the requested spelling and the one that answered,
64
+ # because a redirect means the model may page with either. Holds
65
+ # (answering url, decoded and REDACTED body, whether the byte cap stopped
66
+ # the read).
67
+ #
68
+ # Redacted, since §52 (RB1). Two things follow and both are wanted: no page
69
+ # cut from this body can split a credential, and the process no longer holds
70
+ # unredacted remote content for the life of the entry -- before §51 the body
71
+ # was a local that died with the call, and holding it extended the lifetime
72
+ # of text nobody here wrote from one function call to 300 seconds.
73
+ #
74
+ # The entry COUNT is bounded by `_net_common.MAX_ENTRIES` (§52, RB6).
75
+ # Measured (batch 106): 100 fetches held 6.3 MB and nothing was ever dropped,
76
+ # because `TTLCache` expired on read alone and nothing read those keys again.
77
+ _body_cache = TTLCache(CACHE_TTL_S)
78
+
37
79
 
38
80
  class FetchURLParams(BaseModel):
39
- url: str = Field(..., description="URL to be fetched", min_length=1, max_length=400)
81
+ # `MAX_URL_CHARS`, not 400 spelled again: batch 156 derived the
82
+ # bound the two SEARCH tools use from this one, because a URL
83
+ # they offer past it is a route this gate refuses. Two files
84
+ # holding one number is the state a pair is in just before it
85
+ # stops being a pair, and the relation is what matters here.
86
+ url: str = Field(..., description="URL to be fetched", min_length=1, max_length=MAX_URL_CHARS)
87
+ offset: int = Field(
88
+ 0,
89
+ ge=0,
90
+ description=(
91
+ "Number of characters to skip before reading (0-based). Use the "
92
+ "offset named in a truncated result's message to read the next "
93
+ "page of a long document."
94
+ ),
95
+ )
40
96
 
41
97
 
42
98
  TOOL_SCHEMA = {
@@ -53,6 +109,13 @@ def _bounded_body(response) -> tuple:
53
109
  extra chunk beyond the cap is what makes `more` honest: without it,
54
110
  a body that happens to end exactly at the cap is indistinguishable
55
111
  from one that was cut off.
112
+
113
+ NOT THE PLACE TO REDACT, though it is the first place the body exists
114
+ (§52). This reads BYTES and the redactor works on decoded strings: a
115
+ credential spanning the boundary between two `iter_bytes` chunks would
116
+ be as invisible here as one spanning two pages is downstream, which is
117
+ the very failure §52 exists to close. Redaction runs once in `run()`,
118
+ on the whole decoded body, above both the cache and the slice.
56
119
  """
57
120
  chunks, total, more = [], 0, False
58
121
  for chunk in response.iter_bytes():
@@ -71,6 +134,125 @@ def _bounded_body(response) -> tuple:
71
134
  return raw.decode("utf-8", errors="replace"), more
72
135
 
73
136
 
137
+ def _page(url: str, body: str, more: bool, offset: int) -> dict:
138
+ """One page of a held body, plus how to reach the next (§51, PG1/PG4).
139
+
140
+ `read`'s vocabulary deliberately (PG1): an offset, a bounded slice, and
141
+ a message naming the offset to use next. One paging idiom across the
142
+ tools that have one, rather than a second one to learn here.
143
+
144
+ THE THREE KEYS THIS ALREADY RETURNED KEEP THEIR EXACT MEANINGS.
145
+ `truncated` is still "there is more than you were given", and at
146
+ offset 0 it computes to precisely what it did before §51 -- the byte
147
+ cap bit, or the body is longer than one page. `source_corpus` and the
148
+ grounding passes read `url` and `content`; the register's constraint is
149
+ that added keys are safe and changed ones are not.
150
+
151
+ EVERY POSITION HERE COUNTS REDACTED CHARACTERS (§52, RB3), because
152
+ `body` arrives redacted. That is the only position space this tool
153
+ speaks, and it is self-consistent: a caller only ever passes back an
154
+ offset this function named. Two consequences, stated rather than left to
155
+ be discovered -- a document containing credentials reports a smaller
156
+ `chars_available` than its source has, and the same URL reports
157
+ different totals with `REDACT_TOOL_OUTPUTS` on and off.
158
+ """
159
+ total = len(body)
160
+ chunk = body[offset:offset + MAX_CONTENT_CHARS]
161
+ next_offset = offset + len(chunk)
162
+ beyond = next_offset < total
163
+
164
+ result = {
165
+ "url": url,
166
+ "content": chunk,
167
+ "truncated": beyond or more,
168
+ "offset": offset,
169
+ "chars_available": total,
170
+ }
171
+
172
+ if offset >= total and total:
173
+ # Past the end is not an error, for `read`'s reason: a model that
174
+ # walked one page too far should be told where the end was, not
175
+ # handed a failure it has to reason about.
176
+ result["message"] = (
177
+ f"Offset {offset} is past the end of this document, which has "
178
+ f"{total} characters. Nothing was returned."
179
+ )
180
+ elif beyond:
181
+ result["message"] = (
182
+ f"Returned characters {offset}-{next_offset} of {total}. "
183
+ f"Use offset={next_offset} to read further."
184
+ )
185
+ elif more:
186
+ # PG6, and the whole of it. The cap is NOT raised, so the honest
187
+ # thing is to say the remainder was never retrieved rather than
188
+ # name an offset that would return nothing. Measured (batch 105):
189
+ # an ordinary Python docs page is 88 KB, so roughly a quarter of it
190
+ # is past this bound, and `bbc.com/news` is 382 KB.
191
+ result["message"] = (
192
+ f"Returned characters {offset}-{next_offset}, the end of the "
193
+ f"readable window. This document was cut at the "
194
+ f"{MAX_CONTENT_BYTES}-byte read limit, so the rest of it was "
195
+ f"never retrieved and no further page exists."
196
+ )
197
+ elif offset:
198
+ result["message"] = (
199
+ f"Returned characters {offset}-{next_offset} of {total}. "
200
+ f"This is the last page."
201
+ )
202
+
203
+ return result
204
+
205
+
206
+ def _redacted(url: str, body: str) -> str:
207
+ """The body with credentials replaced, said once (§52, RB1 and RB2).
208
+
209
+ ORDER IS THE WHOLE POINT. `registry.dispatch` runs every result through
210
+ `check_output_policy`, so this text was always redacted -- but it was
211
+ redacted AFTER the page had been cut out of it, and a pattern only
212
+ matches what it can see whole. Measured (batch 106), a `ghp_` token
213
+ straddling character 5,000 survived both pages untouched, was rejoined by
214
+ the corpus's PG7 append, and landed in `sources/<sha256>.txt` and in the
215
+ window sent to the embedding provider. Every one of the ten patterns is
216
+ defeated by some cut; the fixed-length tokens and both credential shapes
217
+ are defeated by EVERY interior cut, because a prefix of a fixed-length
218
+ token is not that token.
219
+
220
+ This repo had already locked the rule twice. `param_digest` "redacts
221
+ BEFORE truncating" because "truncating first cuts a credential below the
222
+ 20 characters its pattern needs", and `tui/app.py`'s diff block runs on
223
+ whole texts because a secret "would be just as readable split across two
224
+ wrapped rows as on one". §51's paging is a third cut, and the first to
225
+ get the order wrong.
226
+
227
+ RB2, the warning. Redacting here means `check_output_policy` finds
228
+ nothing left to alter, so #49's "nothing this function does is silent"
229
+ would stop covering this tool without anybody noticing. One line at the
230
+ fetch replaces up to fourteen at the pages, and it keeps that function's
231
+ own rule: the match is never echoed, because saying what was found would
232
+ put it in app.log and every sink a WARNING reaches. The URL is named --
233
+ it is what makes the line actionable, and `logging_setup`'s formatter
234
+ redaction is the sink-side guard for a URL carrying userinfo, exactly as
235
+ it is for the `fetch_url failed for %s` line below.
236
+ """
237
+ clean = redact_output_text(body)
238
+ if clean != body:
239
+ logger.warning(
240
+ "Output policy altered fetch_url result: credentials replaced "
241
+ "in the body of %s before it was paged.", url)
242
+ return clean
243
+
244
+
245
+ def _cached(url: str):
246
+ """The held body for `url`, or None. Policy is the CALLER's job."""
247
+ return _body_cache.get(url)
248
+
249
+
250
+ def _hold(requested: str, answering: str, body: str, more: bool) -> None:
251
+ _body_cache.set(requested, (answering, body, more))
252
+ if answering != requested:
253
+ _body_cache.set(answering, (answering, body, more))
254
+
255
+
74
256
  def run(params: dict) -> dict:
75
257
  """Fetch one URL, checking policy BEFORE every request it issues.
76
258
 
@@ -96,6 +278,31 @@ def run(params: dict) -> dict:
96
278
  """
97
279
  parsed = FetchURLParams(**params)
98
280
 
281
+ # §51 (PG5). A page after the first is served from the body the first
282
+ # one held, so that every page of a document comes from ONE fetch and
283
+ # page 2 continues page 1 rather than continuing whatever the server
284
+ # is serving now.
285
+ #
286
+ # THE POLICY CHECK RUNS FIRST AND ALWAYS, cache hit included. A cache
287
+ # consulted before the blocklist is a blocklist with a hole in it: the
288
+ # body was permitted when it was fetched, and `is_url_permitted` is not
289
+ # a constant -- it resolves names, and a run's own configuration can
290
+ # change under it. Skipping it here would rebuild #54 one layer up,
291
+ # where the control is bypassed by a second call rather than a
292
+ # redirect.
293
+ #
294
+ # offset 0 never reads the cache: a fresh read of a page is a fresh
295
+ # read, which is what every caller that passes no offset already
296
+ # expects and what makes the held copy current for the pages after it.
297
+ if parsed.offset:
298
+ refusal = is_url_permitted(parsed.url)
299
+ if refusal:
300
+ return {"error": refusal}
301
+ held = _cached(parsed.url)
302
+ if held is not None:
303
+ answering, body, more = held
304
+ return _page(answering, body, more, parsed.offset)
305
+
99
306
  url = parsed.url
100
307
  for _ in range(MAX_REDIRECTS):
101
308
  refusal = is_url_permitted(url)
@@ -131,23 +338,34 @@ def run(params: dict) -> dict:
131
338
  logger.warning("fetch_url failed for %s: %s", url, e)
132
339
  return {"error": f"Could not fetch URL: {e}"}
133
340
 
134
- return {
135
- "url": url,
136
- "content": body[:MAX_CONTENT_CHARS],
137
- # True when EITHER bound bit: more characters were decoded
138
- # than are returned, or the byte cap stopped the read with
139
- # the server still sending. A silently short page is how a
140
- # grounding pass concludes a source does not say something.
141
- #
142
- # `or more` CANNOT FIRE with the shipped constants, and that
143
- # is deliberate rather than an oversight: 65,536 bytes is at
144
- # least 16,384 characters even at four bytes each, so the
145
- # first term is always already true when the byte cap bites.
146
- # It is here for the day someone raises MAX_CONTENT_CHARS or
147
- # lowers MAX_CONTENT_BYTES, which is exactly when a silent
148
- # under-report would appear. A mutation deleting it survived
149
- # until a test moved the constants to reach it.
150
- "truncated": len(body) > MAX_CONTENT_CHARS or more,
151
- }
341
+ # §52 (RB1). ABOVE BOTH THE CACHE AND THE SLICE, and that position
342
+ # is the fix: the redactor sees the whole document exactly once,
343
+ # while it is still one string. Everything below this line -- the
344
+ # held copy, page 1, and every page a later offset cuts out of it --
345
+ # is a slice of already-clean text.
346
+ body = _redacted(url, body)
347
+
348
+ # Held before the page is cut, so a later offset reads the body
349
+ # this fetch actually retrieved (PG5). Keyed by both spellings,
350
+ # because `url` here is the one that ANSWERED and the model may
351
+ # page with the one it asked for.
352
+ _hold(parsed.url, url, body, more)
353
+
354
+ # `truncated` is still true when EITHER bound bit: more characters
355
+ # were decoded than are returned, or the byte cap stopped the read
356
+ # with the server still sending. A silently short page is how a
357
+ # grounding pass concludes a source does not say something.
358
+ #
359
+ # `or more` CANNOT FIRE with the shipped constants, and that is
360
+ # deliberate rather than an oversight: 65,536 bytes is at least
361
+ # 16,384 characters even at four bytes each, so the first term is
362
+ # always already true when the byte cap bites. It is here for the
363
+ # day someone raises MAX_CONTENT_CHARS or lowers
364
+ # MAX_CONTENT_BYTES, which is exactly when a silent under-report
365
+ # would appear. A mutation deleting it survived until a test moved
366
+ # the constants to reach it. §51 moved the expression into
367
+ # `_page` as `beyond or more`, where `beyond` is the same question
368
+ # asked of an arbitrary offset instead of only the first page.
369
+ return _page(url, body, more, parsed.offset)
152
370
 
153
371
  return {"error": f"Too many redirects (more than {MAX_REDIRECTS}): {parsed.url}"}