@yahyahammad/venastine 0.2.0 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -4
- package/CONFIG_ARCHITECTURE.md +2030 -0
- package/LICENSE +201 -201
- package/NOTICE +1 -1
- package/README.md +1034 -895
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -88
- package/agents/builtin/review.md +74 -74
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +68 -2
- package/agents/subagent_tool.py +472 -238
- package/agents/tui_commands.py +238 -151
- package/bin/venastine.mjs +150 -4
- package/bin/venastine.py +71 -0
- package/cli_commands.py +838 -0
- package/config.py +95 -1198
- package/config.yaml +1103 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +971 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -118
- package/core/ansi.py +452 -0
- package/core/approval.py +137 -115
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +2715 -1559
- package/core/commands.py +460 -0
- package/core/compaction.py +1735 -971
- package/core/config_loader.py +1982 -1563
- package/core/connect.py +686 -0
- package/core/events.py +123 -61
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +411 -328
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1490
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +190 -176
- package/core/provider_errors.py +234 -0
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -468
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +1561 -1382
- package/core/reasoning/output_writer.py +345 -286
- package/core/reasoning/payload_validation.py +524 -398
- package/core/reasoning/pipeline_storage.py +88 -22
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +458 -434
- package/core/reasoning/source_corpus.py +433 -327
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -183
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +148 -20
- package/database.py +333 -252
- package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +50 -9
- package/json_store.py +343 -123
- package/logging_setup.py +295 -185
- package/main.py +3433 -1961
- package/mcp_client/client.py +733 -499
- package/mcp_client/config.py +126 -21
- package/mcp_client/registration.py +343 -182
- package/memories/manager.py +147 -132
- package/memories/tui_commands.py +319 -244
- package/package.json +303 -73
- package/project_init/config_files.py +282 -278
- package/project_init/generator.py +728 -564
- package/project_init/manifest.py +18 -6
- package/project_init/tui_commands.py +147 -144
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +2 -0
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -262
- package/providers.json.example +4 -2
- package/pyproject.toml +149 -100
- package/requirements.txt +295 -72
- package/safety/policy_enforcement.py +1082 -746
- package/security/askpass.py +419 -0
- package/security/capability.py +207 -177
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +237 -190
- package/security/posture.py +124 -12
- package/security/protected_paths.py +501 -309
- package/security/sandbox.py +4108 -985
- package/security/secrets.py +855 -0
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -111
- package/storage.py +1338 -1005
- package/tools/base.py +161 -1
- package/tools/builtin/_math_common.py +667 -377
- package/tools/builtin/_net_common.py +225 -19
- package/tools/builtin/arxiv.py +497 -39
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +371 -153
- package/tools/builtin/file_ops.py +720 -412
- package/tools/builtin/geometry.py +48 -12
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +76 -66
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +227 -193
- package/tools/builtin/probability_stats.py +12 -16
- package/tools/builtin/project_docs.py +464 -405
- package/tools/builtin/remember.py +175 -158
- package/tools/builtin/shell.py +980 -461
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +149 -147
- package/tools/builtin/web_search.py +191 -13
- package/tools/context.py +159 -136
- package/tools/isolation.py +107 -34
- package/tools/registry.py +1242 -925
- package/tui/app.py +8318 -4100
- package/tui/app.tcss +747 -556
- package/tui/commands.py +26 -170
- package/tui/diffs.py +34 -13
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -1279
- package/tui/meters.py +364 -268
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -943
- package/tui/themes.py +599 -559
- package/tui/widgets.py +3813 -2332
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/tools/isolation.py
CHANGED
|
@@ -55,6 +55,8 @@ import os
|
|
|
55
55
|
import platform
|
|
56
56
|
import subprocess
|
|
57
57
|
import sys
|
|
58
|
+
import tempfile
|
|
59
|
+
from typing import Optional
|
|
58
60
|
|
|
59
61
|
import config
|
|
60
62
|
|
|
@@ -68,7 +70,9 @@ MARKER = "@@VENASTINE_RESULT@@"
|
|
|
68
70
|
|
|
69
71
|
# Bounded on the CHILD side, where the size is known before anything is
|
|
70
72
|
# written, and re-checked here. #55's own rule -- bound the read, do not
|
|
71
|
-
# slice afterwards -- applied to the mechanism that fixes #55.
|
|
73
|
+
# slice afterwards -- applied to the mechanism that fixes #55. That bounds
|
|
74
|
+
# the RESULT; the TRANSPORT carrying it was unbounded until batch 221
|
|
75
|
+
# (#204), and `run_isolated` says how it is bounded now. The value
|
|
72
76
|
# is generous next to what a math tool can actually return: CPython caps
|
|
73
77
|
# int->str at 4300 digits, so the realistic ceiling is far below this and
|
|
74
78
|
# the check is for the pathological case rather than the ordinary one.
|
|
@@ -157,6 +161,13 @@ def _resource_limits():
|
|
|
157
161
|
def _set_limits():
|
|
158
162
|
resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds))
|
|
159
163
|
resource.setrlimit(resource.RLIMIT_AS, (mem_bytes, mem_bytes))
|
|
164
|
+
# The child's stdout and stderr are files since batch 221 (#204),
|
|
165
|
+
# so what it may write to either is capped here: the result bound
|
|
166
|
+
# twice over, which a well-behaved child never approaches. Past it
|
|
167
|
+
# a write fails (CPython ignores SIGXFSZ) and the call reports no
|
|
168
|
+
# usable result. Windows has no rlimit; the wall clock bounds it.
|
|
169
|
+
file_bytes = 2 * MAX_CHILD_OUTPUT_BYTES
|
|
170
|
+
resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes))
|
|
160
171
|
|
|
161
172
|
return _set_limits
|
|
162
173
|
|
|
@@ -181,17 +192,21 @@ def _limit_message(tool_name: str, timeout: float) -> str:
|
|
|
181
192
|
|
|
182
193
|
|
|
183
194
|
def _interpret(tool_name: str, stdout: str, stderr: str, returncode: int,
|
|
184
|
-
timeout: float) -> dict:
|
|
195
|
+
timeout: float, stdout_bytes: Optional[int] = None) -> dict:
|
|
185
196
|
"""Turn what the child said into the dict dispatch expects.
|
|
186
197
|
|
|
187
198
|
Every branch returns rather than raises. dispatch's own containment
|
|
188
199
|
would convert an exception into an error dict anyway, but it would log
|
|
189
200
|
a traceback with it -- and an oversized factorial is an ordinary thing
|
|
190
201
|
for a model to ask for, not a bug in this harness.
|
|
202
|
+
|
|
203
|
+
`stdout_bytes` is the size the child WROTE, which `stdout` no longer
|
|
204
|
+
holds whole since batch 221: the parent reads only the bound plus one.
|
|
191
205
|
"""
|
|
192
|
-
|
|
206
|
+
size = len(stdout) if stdout_bytes is None else stdout_bytes
|
|
207
|
+
if size > MAX_CHILD_OUTPUT_BYTES:
|
|
193
208
|
return {"error": f"{tool_name} produced more output than can be "
|
|
194
|
-
f"returned ({
|
|
209
|
+
f"returned ({size} bytes). Ask for a smaller "
|
|
195
210
|
f"result."}
|
|
196
211
|
|
|
197
212
|
for line in stdout.splitlines():
|
|
@@ -227,14 +242,23 @@ def _interpret(tool_name: str, stdout: str, stderr: str, returncode: int,
|
|
|
227
242
|
|
|
228
243
|
|
|
229
244
|
def run_isolated(handler, params: dict, timeout: float,
|
|
230
|
-
tool_name: str = "") -> dict:
|
|
245
|
+
tool_name: str = "", confine: Optional[bool] = None) -> dict:
|
|
231
246
|
"""Run `handler` in a killable child and return its result dict.
|
|
232
247
|
|
|
233
248
|
`handler` is resolved by module and qualified name in the child rather
|
|
234
249
|
than pickled: the registry's handlers are plain module-level functions
|
|
235
250
|
and naming them keeps the child's import surface to the one tool.
|
|
236
251
|
Both names come from the registry, never from a model.
|
|
252
|
+
|
|
253
|
+
`confine` walls the child in (security/confine.py) once it has loaded
|
|
254
|
+
the tool and before it reads `params`. None means "when this is a
|
|
255
|
+
shared hosted deployment" (§79 HD11), the one place a stranger chooses
|
|
256
|
+
what a math tool evaluates.
|
|
237
257
|
"""
|
|
258
|
+
if confine is None:
|
|
259
|
+
from security import hosted
|
|
260
|
+
|
|
261
|
+
confine = hosted.is_shared()
|
|
238
262
|
tool_name = tool_name or getattr(handler, "__name__", "tool")
|
|
239
263
|
module = getattr(handler, "__module__", None)
|
|
240
264
|
function = getattr(handler, "__name__", None)
|
|
@@ -248,40 +272,75 @@ def run_isolated(handler, params: dict, timeout: float,
|
|
|
248
272
|
|
|
249
273
|
request = json.dumps({"module": module, "function": function,
|
|
250
274
|
"params": params, "max_bytes":
|
|
251
|
-
MAX_CHILD_OUTPUT_BYTES
|
|
275
|
+
MAX_CHILD_OUTPUT_BYTES, "confine": bool(confine),
|
|
276
|
+
"tool_name": tool_name})
|
|
252
277
|
|
|
253
278
|
popen_kwargs = {}
|
|
254
279
|
if platform.system() != "Windows":
|
|
255
280
|
popen_kwargs["preexec_fn"] = _resource_limits()
|
|
256
281
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
282
|
+
# THE TRANSPORT IS BOUNDED TOO, since batch 221 (#204). The 4 MB bound
|
|
283
|
+
# is the child's, on the result it prints; `capture_output` then held
|
|
284
|
+
# EVERYTHING the child wrote -- progress spam, a warning flood, a giant
|
|
285
|
+
# traceback, ordinary misbehaviour for the calls this module contains --
|
|
286
|
+
# in the parent's memory until the wall clock fired. Temporary files
|
|
287
|
+
# take the pipes' place, the parent reads at most the bound from one and
|
|
288
|
+
# a tail from the other, and on POSIX `_resource_limits` caps what the
|
|
289
|
+
# child may write to either. No thread reads anything (H2).
|
|
290
|
+
with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
|
|
291
|
+
try:
|
|
292
|
+
child = subprocess.run(
|
|
293
|
+
[sys.executable, "-m", "tools.isolation"],
|
|
294
|
+
input=request.encode("utf-8"),
|
|
295
|
+
stdout=out,
|
|
296
|
+
stderr=err,
|
|
297
|
+
timeout=timeout,
|
|
298
|
+
cwd=_package_root(),
|
|
299
|
+
env=_child_env(),
|
|
300
|
+
**popen_kwargs,
|
|
301
|
+
)
|
|
302
|
+
except subprocess.TimeoutExpired:
|
|
303
|
+
# subprocess.run has already killed the child and reaped it,
|
|
304
|
+
# which is the property that makes this a bound rather than a
|
|
305
|
+
# report.
|
|
306
|
+
#
|
|
307
|
+
# On Unix the CPU limit usually gets there first and this
|
|
308
|
+
# branch never runs -- see _interpret's negative-returncode
|
|
309
|
+
# case, which shares this wording rather than repeating it.
|
|
310
|
+
return {"error": _limit_message(tool_name, timeout)}
|
|
311
|
+
except OSError as e:
|
|
312
|
+
logger.warning("Could not start an isolated child for %s: %s",
|
|
313
|
+
tool_name, e)
|
|
314
|
+
return {"error": f"{tool_name} could not be run: {e}"}
|
|
315
|
+
stdout, stdout_bytes = _head(out, MAX_CHILD_OUTPUT_BYTES + 1)
|
|
316
|
+
stderr = _tail(err, 4 * _STDERR_EXCERPT)
|
|
317
|
+
|
|
318
|
+
return _interpret(tool_name, stdout, stderr, child.returncode, timeout,
|
|
319
|
+
stdout_bytes=stdout_bytes)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _head(handle, limit: int) -> tuple:
|
|
323
|
+
"""(the first `limit` bytes of `handle` as text, its whole size)."""
|
|
324
|
+
size = os.fstat(handle.fileno()).st_size
|
|
325
|
+
handle.seek(0)
|
|
326
|
+
# min(), because read(n) allocates n bytes up front: 4 MB for a result
|
|
327
|
+
# of forty, which the tracemalloc test below measured.
|
|
328
|
+
return _text(handle.read(min(limit, size))), size
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _tail(handle, limit: int) -> str:
|
|
332
|
+
"""The last `limit` bytes of `handle` as text: the only part of the
|
|
333
|
+
child's stderr anything reads is the end of it."""
|
|
334
|
+
size = os.fstat(handle.fileno()).st_size
|
|
335
|
+
handle.seek(max(0, size - limit))
|
|
336
|
+
return _text(handle.read())
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _text(raw: bytes) -> str:
|
|
340
|
+
"""Decoded as the text-mode pipes decoded it: UTF-8 with replacement,
|
|
341
|
+
and universal newlines, so a Windows child's \\r\\n reads as \\n."""
|
|
342
|
+
return (raw.decode("utf-8", errors="replace")
|
|
343
|
+
.replace("\r\n", "\n").replace("\r", "\n"))
|
|
285
344
|
|
|
286
345
|
|
|
287
346
|
def _child_main() -> int:
|
|
@@ -295,6 +354,20 @@ def _child_main() -> int:
|
|
|
295
354
|
|
|
296
355
|
module = importlib.import_module(request["module"])
|
|
297
356
|
handler = getattr(module, request["function"])
|
|
357
|
+
if request.get("confine"):
|
|
358
|
+
# AFTER the import, which is what needs the filesystem, and BEFORE
|
|
359
|
+
# the params are touched, which is where a visitor's expression is.
|
|
360
|
+
# A wall that will not go up means the call does not run: there is
|
|
361
|
+
# no unconfined fallback to fall to (security/confine.py).
|
|
362
|
+
from security import confine
|
|
363
|
+
|
|
364
|
+
reason = confine.lock_down()
|
|
365
|
+
if reason:
|
|
366
|
+
name = request.get("tool_name") or request["function"]
|
|
367
|
+
sys.stdout.write(MARKER + json.dumps({
|
|
368
|
+
"error": f"{name} was not run: this deployment could not "
|
|
369
|
+
f"confine it ({reason})."}) + "\n")
|
|
370
|
+
return 0
|
|
298
371
|
result = handler(request["params"])
|
|
299
372
|
|
|
300
373
|
payload = json.dumps(result, default=str)
|