@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -55,6 +55,8 @@ import os
55
55
  import platform
56
56
  import subprocess
57
57
  import sys
58
+ import tempfile
59
+ from typing import Optional
58
60
 
59
61
  import config
60
62
 
@@ -68,7 +70,9 @@ MARKER = "@@VENASTINE_RESULT@@"
68
70
 
69
71
  # Bounded on the CHILD side, where the size is known before anything is
70
72
  # written, and re-checked here. #55's own rule -- bound the read, do not
71
- # slice afterwards -- applied to the mechanism that fixes #55. The value
73
+ # slice afterwards -- applied to the mechanism that fixes #55. That bounds
74
+ # the RESULT; the TRANSPORT carrying it was unbounded until batch 221
75
+ # (#204), and `run_isolated` says how it is bounded now. The value
72
76
  # is generous next to what a math tool can actually return: CPython caps
73
77
  # int->str at 4300 digits, so the realistic ceiling is far below this and
74
78
  # the check is for the pathological case rather than the ordinary one.
@@ -157,6 +161,13 @@ def _resource_limits():
157
161
  def _set_limits():
158
162
  resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds))
159
163
  resource.setrlimit(resource.RLIMIT_AS, (mem_bytes, mem_bytes))
164
+ # The child's stdout and stderr are files since batch 221 (#204),
165
+ # so what it may write to either is capped here: the result bound
166
+ # twice over, which a well-behaved child never approaches. Past it
167
+ # a write fails (CPython ignores SIGXFSZ) and the call reports no
168
+ # usable result. Windows has no rlimit; the wall clock bounds it.
169
+ file_bytes = 2 * MAX_CHILD_OUTPUT_BYTES
170
+ resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes))
160
171
 
161
172
  return _set_limits
162
173
 
@@ -181,17 +192,21 @@ def _limit_message(tool_name: str, timeout: float) -> str:
181
192
 
182
193
 
183
194
  def _interpret(tool_name: str, stdout: str, stderr: str, returncode: int,
184
- timeout: float) -> dict:
195
+ timeout: float, stdout_bytes: Optional[int] = None) -> dict:
185
196
  """Turn what the child said into the dict dispatch expects.
186
197
 
187
198
  Every branch returns rather than raises. dispatch's own containment
188
199
  would convert an exception into an error dict anyway, but it would log
189
200
  a traceback with it -- and an oversized factorial is an ordinary thing
190
201
  for a model to ask for, not a bug in this harness.
202
+
203
+ `stdout_bytes` is the size the child WROTE, which `stdout` no longer
204
+ holds whole since batch 221: the parent reads only the bound plus one.
191
205
  """
192
- if len(stdout) > MAX_CHILD_OUTPUT_BYTES:
206
+ size = len(stdout) if stdout_bytes is None else stdout_bytes
207
+ if size > MAX_CHILD_OUTPUT_BYTES:
193
208
  return {"error": f"{tool_name} produced more output than can be "
194
- f"returned ({len(stdout)} bytes). Ask for a smaller "
209
+ f"returned ({size} bytes). Ask for a smaller "
195
210
  f"result."}
196
211
 
197
212
  for line in stdout.splitlines():
@@ -227,14 +242,23 @@ def _interpret(tool_name: str, stdout: str, stderr: str, returncode: int,
227
242
 
228
243
 
229
244
  def run_isolated(handler, params: dict, timeout: float,
230
- tool_name: str = "") -> dict:
245
+ tool_name: str = "", confine: Optional[bool] = None) -> dict:
231
246
  """Run `handler` in a killable child and return its result dict.
232
247
 
233
248
  `handler` is resolved by module and qualified name in the child rather
234
249
  than pickled: the registry's handlers are plain module-level functions
235
250
  and naming them keeps the child's import surface to the one tool.
236
251
  Both names come from the registry, never from a model.
252
+
253
+ `confine` walls the child in (security/confine.py) once it has loaded
254
+ the tool and before it reads `params`. None means "when this is a
255
+ shared hosted deployment" (§79 HD11), the one place a stranger chooses
256
+ what a math tool evaluates.
237
257
  """
258
+ if confine is None:
259
+ from security import hosted
260
+
261
+ confine = hosted.is_shared()
238
262
  tool_name = tool_name or getattr(handler, "__name__", "tool")
239
263
  module = getattr(handler, "__module__", None)
240
264
  function = getattr(handler, "__name__", None)
@@ -248,40 +272,75 @@ def run_isolated(handler, params: dict, timeout: float,
248
272
 
249
273
  request = json.dumps({"module": module, "function": function,
250
274
  "params": params, "max_bytes":
251
- MAX_CHILD_OUTPUT_BYTES})
275
+ MAX_CHILD_OUTPUT_BYTES, "confine": bool(confine),
276
+ "tool_name": tool_name})
252
277
 
253
278
  popen_kwargs = {}
254
279
  if platform.system() != "Windows":
255
280
  popen_kwargs["preexec_fn"] = _resource_limits()
256
281
 
257
- try:
258
- child = subprocess.run(
259
- [sys.executable, "-m", "tools.isolation"],
260
- input=request,
261
- capture_output=True,
262
- text=True,
263
- encoding="utf-8",
264
- errors="replace",
265
- timeout=timeout,
266
- cwd=_package_root(),
267
- env=_child_env(),
268
- **popen_kwargs,
269
- )
270
- except subprocess.TimeoutExpired:
271
- # subprocess.run has already killed the child and reaped it, which
272
- # is the property that makes this a bound rather than a report.
273
- #
274
- # On Unix the CPU limit usually gets there first and this branch
275
- # never runs -- see _interpret's negative-returncode case, which
276
- # shares this wording rather than repeating it.
277
- return {"error": _limit_message(tool_name, timeout)}
278
- except OSError as e:
279
- logger.warning("Could not start an isolated child for %s: %s",
280
- tool_name, e)
281
- return {"error": f"{tool_name} could not be run: {e}"}
282
-
283
- return _interpret(tool_name, child.stdout, child.stderr,
284
- child.returncode, timeout)
282
+ # THE TRANSPORT IS BOUNDED TOO, since batch 221 (#204). The 4 MB bound
283
+ # is the child's, on the result it prints; `capture_output` then held
284
+ # EVERYTHING the child wrote -- progress spam, a warning flood, a giant
285
+ # traceback, ordinary misbehaviour for the calls this module contains --
286
+ # in the parent's memory until the wall clock fired. Temporary files
287
+ # take the pipes' place, the parent reads at most the bound from one and
288
+ # a tail from the other, and on POSIX `_resource_limits` caps what the
289
+ # child may write to either. No thread reads anything (H2).
290
+ with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
291
+ try:
292
+ child = subprocess.run(
293
+ [sys.executable, "-m", "tools.isolation"],
294
+ input=request.encode("utf-8"),
295
+ stdout=out,
296
+ stderr=err,
297
+ timeout=timeout,
298
+ cwd=_package_root(),
299
+ env=_child_env(),
300
+ **popen_kwargs,
301
+ )
302
+ except subprocess.TimeoutExpired:
303
+ # subprocess.run has already killed the child and reaped it,
304
+ # which is the property that makes this a bound rather than a
305
+ # report.
306
+ #
307
+ # On Unix the CPU limit usually gets there first and this
308
+ # branch never runs -- see _interpret's negative-returncode
309
+ # case, which shares this wording rather than repeating it.
310
+ return {"error": _limit_message(tool_name, timeout)}
311
+ except OSError as e:
312
+ logger.warning("Could not start an isolated child for %s: %s",
313
+ tool_name, e)
314
+ return {"error": f"{tool_name} could not be run: {e}"}
315
+ stdout, stdout_bytes = _head(out, MAX_CHILD_OUTPUT_BYTES + 1)
316
+ stderr = _tail(err, 4 * _STDERR_EXCERPT)
317
+
318
+ return _interpret(tool_name, stdout, stderr, child.returncode, timeout,
319
+ stdout_bytes=stdout_bytes)
320
+
321
+
322
+ def _head(handle, limit: int) -> tuple:
323
+ """(the first `limit` bytes of `handle` as text, its whole size)."""
324
+ size = os.fstat(handle.fileno()).st_size
325
+ handle.seek(0)
326
+ # min(), because read(n) allocates n bytes up front: 4 MB for a result
327
+ # of forty, which the tracemalloc test below measured.
328
+ return _text(handle.read(min(limit, size))), size
329
+
330
+
331
+ def _tail(handle, limit: int) -> str:
332
+ """The last `limit` bytes of `handle` as text: the only part of the
333
+ child's stderr anything reads is the end of it."""
334
+ size = os.fstat(handle.fileno()).st_size
335
+ handle.seek(max(0, size - limit))
336
+ return _text(handle.read())
337
+
338
+
339
+ def _text(raw: bytes) -> str:
340
+ """Decoded as the text-mode pipes decoded it: UTF-8 with replacement,
341
+ and universal newlines, so a Windows child's \\r\\n reads as \\n."""
342
+ return (raw.decode("utf-8", errors="replace")
343
+ .replace("\r\n", "\n").replace("\r", "\n"))
285
344
 
286
345
 
287
346
  def _child_main() -> int:
@@ -295,6 +354,20 @@ def _child_main() -> int:
295
354
 
296
355
  module = importlib.import_module(request["module"])
297
356
  handler = getattr(module, request["function"])
357
+ if request.get("confine"):
358
+ # AFTER the import, which is what needs the filesystem, and BEFORE
359
+ # the params are touched, which is where a visitor's expression is.
360
+ # A wall that will not go up means the call does not run: there is
361
+ # no unconfined fallback to fall to (security/confine.py).
362
+ from security import confine
363
+
364
+ reason = confine.lock_down()
365
+ if reason:
366
+ name = request.get("tool_name") or request["function"]
367
+ sys.stdout.write(MARKER + json.dumps({
368
+ "error": f"{name} was not run: this deployment could not "
369
+ f"confine it ({reason})."}) + "\n")
370
+ return 0
298
371
  result = handler(request["params"])
299
372
 
300
373
  payload = json.dumps(result, default=str)