@yahyahammad/venastine 0.3.8 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2139 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1043 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +584 -528
  14. package/bin/venastine.py +73 -71
  15. package/cli_commands.py +848 -835
  16. package/config.yaml +64 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +987 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2756 -2715
  28. package/core/commands.py +464 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +274 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +467 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +114 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +780 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +314 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +1145 -267
  102. package/tools/builtin/_page_text.py +490 -0
  103. package/tools/builtin/_pdf_text.py +113 -0
  104. package/tools/builtin/arxiv.py +685 -681
  105. package/tools/builtin/fetch_url.py +639 -371
  106. package/tools/builtin/file_ops.py +720 -720
  107. package/tools/builtin/geometry.py +184 -184
  108. package/tools/builtin/load_skill.py +76 -76
  109. package/tools/builtin/pin.py +227 -227
  110. package/tools/builtin/probability_stats.py +224 -224
  111. package/tools/builtin/project_docs.py +464 -464
  112. package/tools/builtin/remember.py +175 -175
  113. package/tools/builtin/shell.py +980 -980
  114. package/tools/builtin/todo.py +149 -149
  115. package/tools/builtin/web_search.py +402 -328
  116. package/tools/context.py +159 -159
  117. package/tools/isolation.py +387 -387
  118. package/tools/registry.py +1252 -1242
  119. package/tui/app.py +8372 -8307
  120. package/tui/app.tcss +754 -739
  121. package/tui/commands.py +26 -26
  122. package/tui/diffs.py +286 -286
  123. package/tui/history.py +128 -128
  124. package/tui/hosted_driver.py +102 -102
  125. package/tui/markdown.py +1329 -1329
  126. package/tui/meters.py +364 -364
  127. package/tui/ravens.py +146 -146
  128. package/tui/screens.py +1255 -1197
  129. package/tui/themes.py +599 -599
  130. package/tui/widgets.py +3861 -3796
@@ -1,387 +1,387 @@
1
- """
2
- tools/isolation.py
3
-
4
- ROADMAP_v2 §31 (H2, H3). A wall clock for tool calls that can actually
5
- stop the work.
6
-
7
- WHY A PROCESS AND NOT A THREAD, WHICH IS THE WHOLE DECISION.
8
-
9
- #57 was deferred once on the grounds that `signal.alarm` is main-thread
10
- only and the TUI runs tools in a worker, so a thread-based watchdog was
11
- the shape -- and "abandoning a thread mid-sympy is not clean". That is
12
- true, and it is an argument against the thread rather than against the
13
- bound. A thread cannot be cancelled in CPython: there is no interruption
14
- point inside `sympy.binomial` for anything to check, so a watchdog that
15
- times out returns an error while the core stays busy until the process
16
- exits. The harness would then report a timeout that did not happen, which
17
- is the class of mechanism this project keeps deciding against -- the same
18
- reason `with_catalogs` suppresses a catalog rather than showing one whose
19
- tool cannot be called.
20
-
21
- A subprocess has the interruption point the thread lacks: the OS. Killing
22
- it reclaims the CPU, which
23
- `test_a_timed_out_call_is_STOPPED_not_merely_abandoned` measures directly
24
- -- by evidence the child leaves on disk, not by os.times(), whose
25
- children_user field is always 0.0 on Windows and would make the
26
- assertion vacuous on the platform it usually runs on. That measurement is
27
- the difference between the two designs rather than a detail of this one.
28
-
29
- WHAT IT COSTS. A fresh interpreter plus sympy plus the tool module is
30
- ~0.34s measured, against a harness where every step is a multi-second
31
- model call. No warm pool: a pool holds interpreters that a killed call
32
- would have to replace anyway, and the state a pooled worker accumulates
33
- is exactly what a bound is supposed to discard.
34
-
35
- MODELLED ON mcp_client.client.call_tool, which had this shape first and
36
- wrote down why: an inner bound that stops the work (there,
37
- `read_timeout_seconds`; here, RLIMIT_CPU) with an outer wall clock as the
38
- backstop "for the case the answer is never coming at all". On Windows
39
- there is no rlimit and no cheap equivalent, so the clock runs alone --
40
- H3 records that asymmetry rather than papering over it.
41
-
42
- WHAT CROSSES THE BOUNDARY. Params in as JSON, the result dict out as
43
- JSON, and nothing else. That is not a limitation to work around: it is
44
- the check that a BUDGET_COMPUTE tool really is a pure function of its
45
- params, which `assert_budget_declared` enforces from the other side by
46
- refusing a compute tool that declares an injection.
47
- """
48
-
49
- from __future__ import annotations
50
-
51
- import json
52
- import logging
53
- import math
54
- import os
55
- import platform
56
- import subprocess
57
- import sys
58
- import tempfile
59
- from typing import Optional
60
-
61
- import config
62
-
63
- logger = logging.getLogger(__name__)
64
-
65
- # The child prints one JSON object on this marker line. A marker rather
66
- # than "parse the whole of stdout" because an imported module is entitled
67
- # to write to stdout -- a deprecation notice from a pinned SymPy would
68
- # otherwise be indistinguishable from a malformed result.
69
- MARKER = "@@VENASTINE_RESULT@@"
70
-
71
- # Bounded on the CHILD side, where the size is known before anything is
72
- # written, and re-checked here. #55's own rule -- bound the read, do not
73
- # slice afterwards -- applied to the mechanism that fixes #55. That bounds
74
- # the RESULT; the TRANSPORT carrying it was unbounded until batch 221
75
- # (#204), and `run_isolated` says how it is bounded now. The value
76
- # is generous next to what a math tool can actually return: CPython caps
77
- # int->str at 4300 digits, so the realistic ceiling is far below this and
78
- # the check is for the pathological case rather than the ordinary one.
79
- MAX_CHILD_OUTPUT_BYTES = 4_000_000
80
-
81
- # How much of the child's stderr reaches the error message. Enough to name
82
- # a real failure, bounded because it is unbounded text from a process that
83
- # just misbehaved.
84
- _STDERR_EXCERPT = 400
85
-
86
-
87
- def _package_root() -> str:
88
- """The directory holding the `tools` package.
89
-
90
- Derived from __file__, never from the working directory: dispatch has
91
- no guarantee about cwd -- the TUI runs from wherever it was launched
92
- and a subagent inherits nothing -- and a child that cannot import its
93
- own tool would fail as a transport error on every call.
94
- """
95
- return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
96
-
97
-
98
- def _child_env() -> dict:
99
- """A minimal environment for the child.
100
-
101
- A math tool needs no credentials, so it is handed none. This is
102
- security/sandbox._scrubbed_env's argument applied to the other
103
- subprocess this project spawns, with one difference: the sandbox
104
- scrubs a shell's inherited environment, while there is nothing to
105
- scrub here -- the child's environment is built rather than filtered,
106
- which is the stronger direction.
107
-
108
- PYTHONPATH is built from the PARENT's resolved sys.path rather than
109
- from the package root alone. Guessing which variables reconstruct an
110
- import surface is a losing game -- on Windows the user site-packages
111
- is derived from APPDATA, a virtualenv needs its own prefix, conda
112
- needs another -- and getting it wrong presents as ModuleNotFoundError
113
- for sympy on every math call. Handing the child the paths the parent
114
- actually resolved is deterministic and gives it exactly the parent's
115
- imports and no more.
116
-
117
- SYSTEMROOT is not optional on Windows: without it the interpreter
118
- cannot initialise its socket/crypto machinery and exits before
119
- reaching any tool.
120
- """
121
- root = _package_root()
122
- path = [root] + [p for p in sys.path
123
- if p and p != root and os.path.isdir(p)]
124
- env = {"PYTHONPATH": os.pathsep.join(path),
125
- # No .pyc writes from a process that exists for one call, and
126
- # no first-call penalty for the directory scan.
127
- "PYTHONDONTWRITEBYTECODE": "1",
128
- # Deterministic, and it keeps a child's traceback readable.
129
- "PYTHONIOENCODING": "utf-8"}
130
- for name in ("SYSTEMROOT", "SystemRoot", "WINDIR", "PATH", "TEMP", "TMP",
131
- "LD_LIBRARY_PATH"):
132
- value = os.environ.get(name)
133
- if value:
134
- env[name] = value
135
- return env
136
-
137
-
138
- def _resource_limits():
139
- """H3's inner layer: a preexec_fn setting RLIMIT_CPU and RLIMIT_AS, or
140
- None where the platform has no rlimit.
141
-
142
- Deliberately NOT security.sandbox._unix_resource_limits, though it is
143
- the same three lines and the same idea. That one spends
144
- SANDBOX_CPU_SECONDS (30), which is ABOVE this budget -- reusing it
145
- would install a limit that can never fire and read, to anyone
146
- checking, as though the inner layer were present. The CPU ceiling has
147
- to be the compute budget or it is decoration.
148
-
149
- The MEMORY ceiling IS shared with the sandbox, because that number is
150
- about what one contained subprocess of this harness may hold and
151
- nothing here argues for a different answer.
152
- """
153
- if platform.system() == "Windows":
154
- return None
155
-
156
- import resource
157
-
158
- cpu_seconds = max(1, int(math.ceil(config.TOOL_COMPUTE_TIMEOUT_S)))
159
- mem_bytes = config.SANDBOX_MEMORY_MB * 1024 * 1024
160
-
161
- def _set_limits():
162
- resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds))
163
- resource.setrlimit(resource.RLIMIT_AS, (mem_bytes, mem_bytes))
164
- # The child's stdout and stderr are files since batch 221 (#204),
165
- # so what it may write to either is capped here: the result bound
166
- # twice over, which a well-behaved child never approaches. Past it
167
- # a write fails (CPython ignores SIGXFSZ) and the call reports no
168
- # usable result. Windows has no rlimit; the wall clock bounds it.
169
- file_bytes = 2 * MAX_CHILD_OUTPUT_BYTES
170
- resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes))
171
-
172
- return _set_limits
173
-
174
-
175
- def _limit_message(tool_name: str, timeout: float) -> str:
176
- """What the model is told when a bound stopped the call.
177
-
178
- ONE wording, because H3 gives the child two bounds on Unix and one
179
- on Windows and they do not fail the same way. The wall clock raises
180
- TimeoutExpired in the parent; RLIMIT_CPU kills the child, so
181
- subprocess.run RETURNS -- normally, with a negative returncode --
182
- and the parent never sees a timeout at all.
183
-
184
- They are the same event to the only reader that matters: a model
185
- deciding whether to ask for something smaller. Two wordings would
186
- make the better-protected platform give the worse answer, which is
187
- what CI caught on the first Linux run of this section.
188
- """
189
- return (f"{tool_name} exceeded its {timeout:g}s limit and was stopped. "
190
- f"The inputs given are too large for this operation -- try "
191
- f"smaller values.")
192
-
193
-
194
- def _interpret(tool_name: str, stdout: str, stderr: str, returncode: int,
195
- timeout: float, stdout_bytes: Optional[int] = None) -> dict:
196
- """Turn what the child said into the dict dispatch expects.
197
-
198
- Every branch returns rather than raises. dispatch's own containment
199
- would convert an exception into an error dict anyway, but it would log
200
- a traceback with it -- and an oversized factorial is an ordinary thing
201
- for a model to ask for, not a bug in this harness.
202
-
203
- `stdout_bytes` is the size the child WROTE, which `stdout` no longer
204
- holds whole since batch 221: the parent reads only the bound plus one.
205
- """
206
- size = len(stdout) if stdout_bytes is None else stdout_bytes
207
- if size > MAX_CHILD_OUTPUT_BYTES:
208
- return {"error": f"{tool_name} produced more output than can be "
209
- f"returned ({size} bytes). Ask for a smaller "
210
- f"result."}
211
-
212
- for line in stdout.splitlines():
213
- if line.startswith(MARKER):
214
- try:
215
- return json.loads(line[len(MARKER):])
216
- except ValueError:
217
- break
218
-
219
- # A NEGATIVE returncode means the child was killed by a signal, and
220
- # for a child this module launched that means a bound fired --
221
- # RLIMIT_CPU raises SIGXCPU and then SIGKILL at the hard limit
222
- # (H3's inner layer), which is the ONLY way a compute child dies
223
- # without either answering or timing out in the parent.
224
- #
225
- # Checked before the generic branch, not after, and reported with
226
- # the same sentence the wall clock uses. Windows has no rlimit and
227
- # signals a killed process differently, so this is unreachable
228
- # there -- which is exactly why it went unnoticed until CI ran the
229
- # suite on Linux.
230
- if returncode is not None and returncode < 0:
231
- logger.info("%s was stopped by its resource limit (signal %d).",
232
- tool_name, -returncode)
233
- return {"error": _limit_message(tool_name, timeout)}
234
-
235
- excerpt = (stderr or "").strip().replace("\n", " ")[-_STDERR_EXCERPT:]
236
- # logger.warning, not exception: there is no traceback here to keep,
237
- # and this path is reachable by ordinary input.
238
- logger.warning("%s produced no readable result (exit %s): %s",
239
- tool_name, returncode, excerpt)
240
- return {"error": f"{tool_name} did not return a usable result "
241
- f"(exit {returncode}). {excerpt}".strip()}
242
-
243
-
244
- def run_isolated(handler, params: dict, timeout: float,
245
- tool_name: str = "", confine: Optional[bool] = None) -> dict:
246
- """Run `handler` in a killable child and return its result dict.
247
-
248
- `handler` is resolved by module and qualified name in the child rather
249
- than pickled: the registry's handlers are plain module-level functions
250
- and naming them keeps the child's import surface to the one tool.
251
- Both names come from the registry, never from a model.
252
-
253
- `confine` walls the child in (security/confine.py) once it has loaded
254
- the tool and before it reads `params`. None means "when this is a
255
- shared hosted deployment" (§79 HD11), the one place a stranger chooses
256
- what a math tool evaluates.
257
- """
258
- if confine is None:
259
- from security import hosted
260
-
261
- confine = hosted.is_shared()
262
- tool_name = tool_name or getattr(handler, "__name__", "tool")
263
- module = getattr(handler, "__module__", None)
264
- function = getattr(handler, "__name__", None)
265
- if not module or not function:
266
- # A handler that is not a named module-level function cannot be
267
- # reached from a fresh interpreter. Refuse rather than fall back to
268
- # an in-process call, which would silently drop the bound this
269
- # whole module exists to apply.
270
- return {"error": f"{tool_name} cannot be isolated: its handler is "
271
- f"not a named module-level function."}
272
-
273
- request = json.dumps({"module": module, "function": function,
274
- "params": params, "max_bytes":
275
- MAX_CHILD_OUTPUT_BYTES, "confine": bool(confine),
276
- "tool_name": tool_name})
277
-
278
- popen_kwargs = {}
279
- if platform.system() != "Windows":
280
- popen_kwargs["preexec_fn"] = _resource_limits()
281
-
282
- # THE TRANSPORT IS BOUNDED TOO, since batch 221 (#204). The 4 MB bound
283
- # is the child's, on the result it prints; `capture_output` then held
284
- # EVERYTHING the child wrote -- progress spam, a warning flood, a giant
285
- # traceback, ordinary misbehaviour for the calls this module contains --
286
- # in the parent's memory until the wall clock fired. Temporary files
287
- # take the pipes' place, the parent reads at most the bound from one and
288
- # a tail from the other, and on POSIX `_resource_limits` caps what the
289
- # child may write to either. No thread reads anything (H2).
290
- with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
291
- try:
292
- child = subprocess.run(
293
- [sys.executable, "-m", "tools.isolation"],
294
- input=request.encode("utf-8"),
295
- stdout=out,
296
- stderr=err,
297
- timeout=timeout,
298
- cwd=_package_root(),
299
- env=_child_env(),
300
- **popen_kwargs,
301
- )
302
- except subprocess.TimeoutExpired:
303
- # subprocess.run has already killed the child and reaped it,
304
- # which is the property that makes this a bound rather than a
305
- # report.
306
- #
307
- # On Unix the CPU limit usually gets there first and this
308
- # branch never runs -- see _interpret's negative-returncode
309
- # case, which shares this wording rather than repeating it.
310
- return {"error": _limit_message(tool_name, timeout)}
311
- except OSError as e:
312
- logger.warning("Could not start an isolated child for %s: %s",
313
- tool_name, e)
314
- return {"error": f"{tool_name} could not be run: {e}"}
315
- stdout, stdout_bytes = _head(out, MAX_CHILD_OUTPUT_BYTES + 1)
316
- stderr = _tail(err, 4 * _STDERR_EXCERPT)
317
-
318
- return _interpret(tool_name, stdout, stderr, child.returncode, timeout,
319
- stdout_bytes=stdout_bytes)
320
-
321
-
322
- def _head(handle, limit: int) -> tuple:
323
- """(the first `limit` bytes of `handle` as text, its whole size)."""
324
- size = os.fstat(handle.fileno()).st_size
325
- handle.seek(0)
326
- # min(), because read(n) allocates n bytes up front: 4 MB for a result
327
- # of forty, which the tracemalloc test below measured.
328
- return _text(handle.read(min(limit, size))), size
329
-
330
-
331
- def _tail(handle, limit: int) -> str:
332
- """The last `limit` bytes of `handle` as text: the only part of the
333
- child's stderr anything reads is the end of it."""
334
- size = os.fstat(handle.fileno()).st_size
335
- handle.seek(max(0, size - limit))
336
- return _text(handle.read())
337
-
338
-
339
- def _text(raw: bytes) -> str:
340
- """Decoded as the text-mode pipes decoded it: UTF-8 with replacement,
341
- and universal newlines, so a Windows child's \\r\\n reads as \\n."""
342
- return (raw.decode("utf-8", errors="replace")
343
- .replace("\r\n", "\n").replace("\r", "\n"))
344
-
345
-
346
- def _child_main() -> int:
347
- """The child. Reads one request from stdin, prints one marked result.
348
-
349
- Kept deliberately small: everything it does before the handler runs is
350
- time the wall clock is already counting.
351
- """
352
- request = json.loads(sys.stdin.read())
353
- import importlib
354
-
355
- module = importlib.import_module(request["module"])
356
- handler = getattr(module, request["function"])
357
- if request.get("confine"):
358
- # AFTER the import, which is what needs the filesystem, and BEFORE
359
- # the params are touched, which is where a visitor's expression is.
360
- # A wall that will not go up means the call does not run: there is
361
- # no unconfined fallback to fall to (security/confine.py).
362
- from security import confine
363
-
364
- reason = confine.lock_down()
365
- if reason:
366
- name = request.get("tool_name") or request["function"]
367
- sys.stdout.write(MARKER + json.dumps({
368
- "error": f"{name} was not run: this deployment could not "
369
- f"confine it ({reason})."}) + "\n")
370
- return 0
371
- result = handler(request["params"])
372
-
373
- payload = json.dumps(result, default=str)
374
- limit = request.get("max_bytes", MAX_CHILD_OUTPUT_BYTES)
375
- if len(payload) > limit:
376
- # Refused HERE, where the size is known before anything is
377
- # written. The parent checks too, but this is the producer and
378
- # #55's whole point is that the producer is where a bound belongs.
379
- payload = json.dumps({
380
- "error": "the result is too large to return "
381
- f"({len(payload)} bytes). Ask for a smaller result."})
382
- sys.stdout.write(MARKER + payload + "\n")
383
- return 0
384
-
385
-
386
- if __name__ == "__main__":
387
- sys.exit(_child_main())
1
+ """
2
+ tools/isolation.py
3
+
4
+ ROADMAP_v2 §31 (H2, H3). A wall clock for tool calls that can actually
5
+ stop the work.
6
+
7
+ WHY A PROCESS AND NOT A THREAD, WHICH IS THE WHOLE DECISION.
8
+
9
+ #57 was deferred once on the grounds that `signal.alarm` is main-thread
10
+ only and the TUI runs tools in a worker, so a thread-based watchdog was
11
+ the shape -- and "abandoning a thread mid-sympy is not clean". That is
12
+ true, and it is an argument against the thread rather than against the
13
+ bound. A thread cannot be cancelled in CPython: there is no interruption
14
+ point inside `sympy.binomial` for anything to check, so a watchdog that
15
+ times out returns an error while the core stays busy until the process
16
+ exits. The harness would then report a timeout that did not happen, which
17
+ is the class of mechanism this project keeps deciding against -- the same
18
+ reason `with_catalogs` suppresses a catalog rather than showing one whose
19
+ tool cannot be called.
20
+
21
+ A subprocess has the interruption point the thread lacks: the OS. Killing
22
+ it reclaims the CPU, which
23
+ `test_a_timed_out_call_is_STOPPED_not_merely_abandoned` measures directly
24
+ -- by evidence the child leaves on disk, not by os.times(), whose
25
+ children_user field is always 0.0 on Windows and would make the
26
+ assertion vacuous on the platform it usually runs on. That measurement is
27
+ the difference between the two designs rather than a detail of this one.
28
+
29
+ WHAT IT COSTS. A fresh interpreter plus sympy plus the tool module is
30
+ ~0.34s measured, against a harness where every step is a multi-second
31
+ model call. No warm pool: a pool holds interpreters that a killed call
32
+ would have to replace anyway, and the state a pooled worker accumulates
33
+ is exactly what a bound is supposed to discard.
34
+
35
+ MODELLED ON mcp_client.client.call_tool, which had this shape first and
36
+ wrote down why: an inner bound that stops the work (there,
37
+ `read_timeout_seconds`; here, RLIMIT_CPU) with an outer wall clock as the
38
+ backstop "for the case the answer is never coming at all". On Windows
39
+ there is no rlimit and no cheap equivalent, so the clock runs alone --
40
+ H3 records that asymmetry rather than papering over it.
41
+
42
+ WHAT CROSSES THE BOUNDARY. Params in as JSON, the result dict out as
43
+ JSON, and nothing else. That is not a limitation to work around: it is
44
+ the check that a BUDGET_COMPUTE tool really is a pure function of its
45
+ params, which `assert_budget_declared` enforces from the other side by
46
+ refusing a compute tool that declares an injection.
47
+ """
48
+
49
+ from __future__ import annotations
50
+
51
+ import json
52
+ import logging
53
+ import math
54
+ import os
55
+ import platform
56
+ import subprocess
57
+ import sys
58
+ import tempfile
59
+ from typing import Optional
60
+
61
+ import config
62
+
63
+ logger = logging.getLogger(__name__)
64
+
65
+ # The child prints one JSON object on this marker line. A marker rather
66
+ # than "parse the whole of stdout" because an imported module is entitled
67
+ # to write to stdout -- a deprecation notice from a pinned SymPy would
68
+ # otherwise be indistinguishable from a malformed result.
69
+ MARKER = "@@VENASTINE_RESULT@@"
70
+
71
+ # Bounded on the CHILD side, where the size is known before anything is
72
+ # written, and re-checked here. #55's own rule -- bound the read, do not
73
+ # slice afterwards -- applied to the mechanism that fixes #55. That bounds
74
+ # the RESULT; the TRANSPORT carrying it was unbounded until batch 221
75
+ # (#204), and `run_isolated` says how it is bounded now. The value
76
+ # is generous next to what a math tool can actually return: CPython caps
77
+ # int->str at 4300 digits, so the realistic ceiling is far below this and
78
+ # the check is for the pathological case rather than the ordinary one.
79
+ MAX_CHILD_OUTPUT_BYTES = 4_000_000
80
+
81
+ # How much of the child's stderr reaches the error message. Enough to name
82
+ # a real failure, bounded because it is unbounded text from a process that
83
+ # just misbehaved.
84
+ _STDERR_EXCERPT = 400
85
+
86
+
87
+ def _package_root() -> str:
88
+ """The directory holding the `tools` package.
89
+
90
+ Derived from __file__, never from the working directory: dispatch has
91
+ no guarantee about cwd -- the TUI runs from wherever it was launched
92
+ and a subagent inherits nothing -- and a child that cannot import its
93
+ own tool would fail as a transport error on every call.
94
+ """
95
+ return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
96
+
97
+
98
+ def _child_env() -> dict:
99
+ """A minimal environment for the child.
100
+
101
+ A math tool needs no credentials, so it is handed none. This is
102
+ security/sandbox._scrubbed_env's argument applied to the other
103
+ subprocess this project spawns, with one difference: the sandbox
104
+ scrubs a shell's inherited environment, while there is nothing to
105
+ scrub here -- the child's environment is built rather than filtered,
106
+ which is the stronger direction.
107
+
108
+ PYTHONPATH is built from the PARENT's resolved sys.path rather than
109
+ from the package root alone. Guessing which variables reconstruct an
110
+ import surface is a losing game -- on Windows the user site-packages
111
+ is derived from APPDATA, a virtualenv needs its own prefix, conda
112
+ needs another -- and getting it wrong presents as ModuleNotFoundError
113
+ for sympy on every math call. Handing the child the paths the parent
114
+ actually resolved is deterministic and gives it exactly the parent's
115
+ imports and no more.
116
+
117
+ SYSTEMROOT is not optional on Windows: without it the interpreter
118
+ cannot initialise its socket/crypto machinery and exits before
119
+ reaching any tool.
120
+ """
121
+ root = _package_root()
122
+ path = [root] + [p for p in sys.path
123
+ if p and p != root and os.path.isdir(p)]
124
+ env = {"PYTHONPATH": os.pathsep.join(path),
125
+ # No .pyc writes from a process that exists for one call, and
126
+ # no first-call penalty for the directory scan.
127
+ "PYTHONDONTWRITEBYTECODE": "1",
128
+ # Deterministic, and it keeps a child's traceback readable.
129
+ "PYTHONIOENCODING": "utf-8"}
130
+ for name in ("SYSTEMROOT", "SystemRoot", "WINDIR", "PATH", "TEMP", "TMP",
131
+ "LD_LIBRARY_PATH"):
132
+ value = os.environ.get(name)
133
+ if value:
134
+ env[name] = value
135
+ return env
136
+
137
+
138
+ def _resource_limits():
139
+ """H3's inner layer: a preexec_fn setting RLIMIT_CPU and RLIMIT_AS, or
140
+ None where the platform has no rlimit.
141
+
142
+ Deliberately NOT security.sandbox._unix_resource_limits, though it is
143
+ the same three lines and the same idea. That one spends
144
+ SANDBOX_CPU_SECONDS (30), which is ABOVE this budget -- reusing it
145
+ would install a limit that can never fire and read, to anyone
146
+ checking, as though the inner layer were present. The CPU ceiling has
147
+ to be the compute budget or it is decoration.
148
+
149
+ The MEMORY ceiling IS shared with the sandbox, because that number is
150
+ about what one contained subprocess of this harness may hold and
151
+ nothing here argues for a different answer.
152
+ """
153
+ if platform.system() == "Windows":
154
+ return None
155
+
156
+ import resource
157
+
158
+ cpu_seconds = max(1, int(math.ceil(config.TOOL_COMPUTE_TIMEOUT_S)))
159
+ mem_bytes = config.SANDBOX_MEMORY_MB * 1024 * 1024
160
+
161
+ def _set_limits():
162
+ resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds))
163
+ resource.setrlimit(resource.RLIMIT_AS, (mem_bytes, mem_bytes))
164
+ # The child's stdout and stderr are files since batch 221 (#204),
165
+ # so what it may write to either is capped here: the result bound
166
+ # twice over, which a well-behaved child never approaches. Past it
167
+ # a write fails (CPython ignores SIGXFSZ) and the call reports no
168
+ # usable result. Windows has no rlimit; the wall clock bounds it.
169
+ file_bytes = 2 * MAX_CHILD_OUTPUT_BYTES
170
+ resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes))
171
+
172
+ return _set_limits
173
+
174
+
175
+ def _limit_message(tool_name: str, timeout: float) -> str:
176
+ """What the model is told when a bound stopped the call.
177
+
178
+ ONE wording, because H3 gives the child two bounds on Unix and one
179
+ on Windows and they do not fail the same way. The wall clock raises
180
+ TimeoutExpired in the parent; RLIMIT_CPU kills the child, so
181
+ subprocess.run RETURNS -- normally, with a negative returncode --
182
+ and the parent never sees a timeout at all.
183
+
184
+ They are the same event to the only reader that matters: a model
185
+ deciding whether to ask for something smaller. Two wordings would
186
+ make the better-protected platform give the worse answer, which is
187
+ what CI caught on the first Linux run of this section.
188
+ """
189
+ return (f"{tool_name} exceeded its {timeout:g}s limit and was stopped. "
190
+ f"The inputs given are too large for this operation -- try "
191
+ f"smaller values.")
192
+
193
+
194
+ def _interpret(tool_name: str, stdout: str, stderr: str, returncode: int,
195
+ timeout: float, stdout_bytes: Optional[int] = None) -> dict:
196
+ """Turn what the child said into the dict dispatch expects.
197
+
198
+ Every branch returns rather than raises. dispatch's own containment
199
+ would convert an exception into an error dict anyway, but it would log
200
+ a traceback with it -- and an oversized factorial is an ordinary thing
201
+ for a model to ask for, not a bug in this harness.
202
+
203
+ `stdout_bytes` is the size the child WROTE, which `stdout` no longer
204
+ holds whole since batch 221: the parent reads only the bound plus one.
205
+ """
206
+ size = len(stdout) if stdout_bytes is None else stdout_bytes
207
+ if size > MAX_CHILD_OUTPUT_BYTES:
208
+ return {"error": f"{tool_name} produced more output than can be "
209
+ f"returned ({size} bytes). Ask for a smaller "
210
+ f"result."}
211
+
212
+ for line in stdout.splitlines():
213
+ if line.startswith(MARKER):
214
+ try:
215
+ return json.loads(line[len(MARKER):])
216
+ except ValueError:
217
+ break
218
+
219
+ # A NEGATIVE returncode means the child was killed by a signal, and
220
+ # for a child this module launched that means a bound fired --
221
+ # RLIMIT_CPU raises SIGXCPU and then SIGKILL at the hard limit
222
+ # (H3's inner layer), which is the ONLY way a compute child dies
223
+ # without either answering or timing out in the parent.
224
+ #
225
+ # Checked before the generic branch, not after, and reported with
226
+ # the same sentence the wall clock uses. Windows has no rlimit and
227
+ # signals a killed process differently, so this is unreachable
228
+ # there -- which is exactly why it went unnoticed until CI ran the
229
+ # suite on Linux.
230
+ if returncode is not None and returncode < 0:
231
+ logger.info("%s was stopped by its resource limit (signal %d).",
232
+ tool_name, -returncode)
233
+ return {"error": _limit_message(tool_name, timeout)}
234
+
235
+ excerpt = (stderr or "").strip().replace("\n", " ")[-_STDERR_EXCERPT:]
236
+ # logger.warning, not exception: there is no traceback here to keep,
237
+ # and this path is reachable by ordinary input.
238
+ logger.warning("%s produced no readable result (exit %s): %s",
239
+ tool_name, returncode, excerpt)
240
+ return {"error": f"{tool_name} did not return a usable result "
241
+ f"(exit {returncode}). {excerpt}".strip()}
242
+
243
+
244
+ def run_isolated(handler, params: dict, timeout: float,
245
+ tool_name: str = "", confine: Optional[bool] = None) -> dict:
246
+ """Run `handler` in a killable child and return its result dict.
247
+
248
+ `handler` is resolved by module and qualified name in the child rather
249
+ than pickled: the registry's handlers are plain module-level functions
250
+ and naming them keeps the child's import surface to the one tool.
251
+ Both names come from the registry, never from a model.
252
+
253
+ `confine` walls the child in (security/confine.py) once it has loaded
254
+ the tool and before it reads `params`. None means "when this is a
255
+ shared hosted deployment" (§79 HD11), the one place a stranger chooses
256
+ what a math tool evaluates.
257
+ """
258
+ if confine is None:
259
+ from security import hosted
260
+
261
+ confine = hosted.is_shared()
262
+ tool_name = tool_name or getattr(handler, "__name__", "tool")
263
+ module = getattr(handler, "__module__", None)
264
+ function = getattr(handler, "__name__", None)
265
+ if not module or not function:
266
+ # A handler that is not a named module-level function cannot be
267
+ # reached from a fresh interpreter. Refuse rather than fall back to
268
+ # an in-process call, which would silently drop the bound this
269
+ # whole module exists to apply.
270
+ return {"error": f"{tool_name} cannot be isolated: its handler is "
271
+ f"not a named module-level function."}
272
+
273
+ request = json.dumps({"module": module, "function": function,
274
+ "params": params, "max_bytes":
275
+ MAX_CHILD_OUTPUT_BYTES, "confine": bool(confine),
276
+ "tool_name": tool_name})
277
+
278
+ popen_kwargs = {}
279
+ if platform.system() != "Windows":
280
+ popen_kwargs["preexec_fn"] = _resource_limits()
281
+
282
+ # THE TRANSPORT IS BOUNDED TOO, since batch 221 (#204). The 4 MB bound
283
+ # is the child's, on the result it prints; `capture_output` then held
284
+ # EVERYTHING the child wrote -- progress spam, a warning flood, a giant
285
+ # traceback, ordinary misbehaviour for the calls this module contains --
286
+ # in the parent's memory until the wall clock fired. Temporary files
287
+ # take the pipes' place, the parent reads at most the bound from one and
288
+ # a tail from the other, and on POSIX `_resource_limits` caps what the
289
+ # child may write to either. No thread reads anything (H2).
290
+ with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
291
+ try:
292
+ child = subprocess.run(
293
+ [sys.executable, "-m", "tools.isolation"],
294
+ input=request.encode("utf-8"),
295
+ stdout=out,
296
+ stderr=err,
297
+ timeout=timeout,
298
+ cwd=_package_root(),
299
+ env=_child_env(),
300
+ **popen_kwargs,
301
+ )
302
+ except subprocess.TimeoutExpired:
303
+ # subprocess.run has already killed the child and reaped it,
304
+ # which is the property that makes this a bound rather than a
305
+ # report.
306
+ #
307
+ # On Unix the CPU limit usually gets there first and this
308
+ # branch never runs -- see _interpret's negative-returncode
309
+ # case, which shares this wording rather than repeating it.
310
+ return {"error": _limit_message(tool_name, timeout)}
311
+ except OSError as e:
312
+ logger.warning("Could not start an isolated child for %s: %s",
313
+ tool_name, e)
314
+ return {"error": f"{tool_name} could not be run: {e}"}
315
+ stdout, stdout_bytes = _head(out, MAX_CHILD_OUTPUT_BYTES + 1)
316
+ stderr = _tail(err, 4 * _STDERR_EXCERPT)
317
+
318
+ return _interpret(tool_name, stdout, stderr, child.returncode, timeout,
319
+ stdout_bytes=stdout_bytes)
320
+
321
+
322
+ def _head(handle, limit: int) -> tuple:
323
+ """(the first `limit` bytes of `handle` as text, its whole size)."""
324
+ size = os.fstat(handle.fileno()).st_size
325
+ handle.seek(0)
326
+ # min(), because read(n) allocates n bytes up front: 4 MB for a result
327
+ # of forty, which the tracemalloc test below measured.
328
+ return _text(handle.read(min(limit, size))), size
329
+
330
+
331
+ def _tail(handle, limit: int) -> str:
332
+ """The last `limit` bytes of `handle` as text: the only part of the
333
+ child's stderr anything reads is the end of it."""
334
+ size = os.fstat(handle.fileno()).st_size
335
+ handle.seek(max(0, size - limit))
336
+ return _text(handle.read())
337
+
338
+
339
+ def _text(raw: bytes) -> str:
340
+ """Decoded as the text-mode pipes decoded it: UTF-8 with replacement,
341
+ and universal newlines, so a Windows child's \\r\\n reads as \\n."""
342
+ return (raw.decode("utf-8", errors="replace")
343
+ .replace("\r\n", "\n").replace("\r", "\n"))
344
+
345
+
346
+ def _child_main() -> int:
347
+ """The child. Reads one request from stdin, prints one marked result.
348
+
349
+ Kept deliberately small: everything it does before the handler runs is
350
+ time the wall clock is already counting.
351
+ """
352
+ request = json.loads(sys.stdin.read())
353
+ import importlib
354
+
355
+ module = importlib.import_module(request["module"])
356
+ handler = getattr(module, request["function"])
357
+ if request.get("confine"):
358
+ # AFTER the import, which is what needs the filesystem, and BEFORE
359
+ # the params are touched, which is where a visitor's expression is.
360
+ # A wall that will not go up means the call does not run: there is
361
+ # no unconfined fallback to fall to (security/confine.py).
362
+ from security import confine
363
+
364
+ reason = confine.lock_down()
365
+ if reason:
366
+ name = request.get("tool_name") or request["function"]
367
+ sys.stdout.write(MARKER + json.dumps({
368
+ "error": f"{name} was not run: this deployment could not "
369
+ f"confine it ({reason})."}) + "\n")
370
+ return 0
371
+ result = handler(request["params"])
372
+
373
+ payload = json.dumps(result, default=str)
374
+ limit = request.get("max_bytes", MAX_CHILD_OUTPUT_BYTES)
375
+ if len(payload) > limit:
376
+ # Refused HERE, where the size is known before anything is
377
+ # written. The parent checks too, but this is the producer and
378
+ # #55's whole point is that the producer is where a bound belongs.
379
+ payload = json.dumps({
380
+ "error": "the result is too large to return "
381
+ f"({len(payload)} bytes). Ask for a smaller result."})
382
+ sys.stdout.write(MARKER + payload + "\n")
383
+ return 0
384
+
385
+
386
+ if __name__ == "__main__":
387
+ sys.exit(_child_main())