@yahyahammad/venastine 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2030 -2002
- package/LICENSE +201 -201
- package/README.md +1034 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +528 -528
- package/bin/venastine.py +71 -71
- package/cli_commands.py +838 -835
- package/config.yaml +13 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +971 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2715 -2715
- package/core/commands.py +460 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +234 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +458 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +108 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +733 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +295 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +267 -267
- package/tools/builtin/arxiv.py +681 -681
- package/tools/builtin/fetch_url.py +371 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +328 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1242 -1242
- package/tui/app.py +8318 -8307
- package/tui/app.tcss +747 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3813 -3796
package/tools/isolation.py
CHANGED
|
@@ -1,387 +1,387 @@
|
|
|
1
|
-
"""
|
|
2
|
-
tools/isolation.py
|
|
3
|
-
|
|
4
|
-
ROADMAP_v2 §31 (H2, H3). A wall clock for tool calls that can actually
|
|
5
|
-
stop the work.
|
|
6
|
-
|
|
7
|
-
WHY A PROCESS AND NOT A THREAD, WHICH IS THE WHOLE DECISION.
|
|
8
|
-
|
|
9
|
-
#57 was deferred once on the grounds that `signal.alarm` is main-thread
|
|
10
|
-
only and the TUI runs tools in a worker, so a thread-based watchdog was
|
|
11
|
-
the shape -- and "abandoning a thread mid-sympy is not clean". That is
|
|
12
|
-
true, and it is an argument against the thread rather than against the
|
|
13
|
-
bound. A thread cannot be cancelled in CPython: there is no interruption
|
|
14
|
-
point inside `sympy.binomial` for anything to check, so a watchdog that
|
|
15
|
-
times out returns an error while the core stays busy until the process
|
|
16
|
-
exits. The harness would then report a timeout that did not happen, which
|
|
17
|
-
is the class of mechanism this project keeps deciding against -- the same
|
|
18
|
-
reason `with_catalogs` suppresses a catalog rather than showing one whose
|
|
19
|
-
tool cannot be called.
|
|
20
|
-
|
|
21
|
-
A subprocess has the interruption point the thread lacks: the OS. Killing
|
|
22
|
-
it reclaims the CPU, which
|
|
23
|
-
`test_a_timed_out_call_is_STOPPED_not_merely_abandoned` measures directly
|
|
24
|
-
-- by evidence the child leaves on disk, not by os.times(), whose
|
|
25
|
-
children_user field is always 0.0 on Windows and would make the
|
|
26
|
-
assertion vacuous on the platform it usually runs on. That measurement is
|
|
27
|
-
the difference between the two designs rather than a detail of this one.
|
|
28
|
-
|
|
29
|
-
WHAT IT COSTS. A fresh interpreter plus sympy plus the tool module is
|
|
30
|
-
~0.34s measured, against a harness where every step is a multi-second
|
|
31
|
-
model call. No warm pool: a pool holds interpreters that a killed call
|
|
32
|
-
would have to replace anyway, and the state a pooled worker accumulates
|
|
33
|
-
is exactly what a bound is supposed to discard.
|
|
34
|
-
|
|
35
|
-
MODELLED ON mcp_client.client.call_tool, which had this shape first and
|
|
36
|
-
wrote down why: an inner bound that stops the work (there,
|
|
37
|
-
`read_timeout_seconds`; here, RLIMIT_CPU) with an outer wall clock as the
|
|
38
|
-
backstop "for the case the answer is never coming at all". On Windows
|
|
39
|
-
there is no rlimit and no cheap equivalent, so the clock runs alone --
|
|
40
|
-
H3 records that asymmetry rather than papering over it.
|
|
41
|
-
|
|
42
|
-
WHAT CROSSES THE BOUNDARY. Params in as JSON, the result dict out as
|
|
43
|
-
JSON, and nothing else. That is not a limitation to work around: it is
|
|
44
|
-
the check that a BUDGET_COMPUTE tool really is a pure function of its
|
|
45
|
-
params, which `assert_budget_declared` enforces from the other side by
|
|
46
|
-
refusing a compute tool that declares an injection.
|
|
47
|
-
"""
|
|
48
|
-
|
|
49
|
-
from __future__ import annotations
|
|
50
|
-
|
|
51
|
-
import json
|
|
52
|
-
import logging
|
|
53
|
-
import math
|
|
54
|
-
import os
|
|
55
|
-
import platform
|
|
56
|
-
import subprocess
|
|
57
|
-
import sys
|
|
58
|
-
import tempfile
|
|
59
|
-
from typing import Optional
|
|
60
|
-
|
|
61
|
-
import config
|
|
62
|
-
|
|
63
|
-
logger = logging.getLogger(__name__)
|
|
64
|
-
|
|
65
|
-
# The child prints one JSON object on this marker line. A marker rather
|
|
66
|
-
# than "parse the whole of stdout" because an imported module is entitled
|
|
67
|
-
# to write to stdout -- a deprecation notice from a pinned SymPy would
|
|
68
|
-
# otherwise be indistinguishable from a malformed result.
|
|
69
|
-
MARKER = "@@VENASTINE_RESULT@@"
|
|
70
|
-
|
|
71
|
-
# Bounded on the CHILD side, where the size is known before anything is
|
|
72
|
-
# written, and re-checked here. #55's own rule -- bound the read, do not
|
|
73
|
-
# slice afterwards -- applied to the mechanism that fixes #55. That bounds
|
|
74
|
-
# the RESULT; the TRANSPORT carrying it was unbounded until batch 221
|
|
75
|
-
# (#204), and `run_isolated` says how it is bounded now. The value
|
|
76
|
-
# is generous next to what a math tool can actually return: CPython caps
|
|
77
|
-
# int->str at 4300 digits, so the realistic ceiling is far below this and
|
|
78
|
-
# the check is for the pathological case rather than the ordinary one.
|
|
79
|
-
MAX_CHILD_OUTPUT_BYTES = 4_000_000
|
|
80
|
-
|
|
81
|
-
# How much of the child's stderr reaches the error message. Enough to name
|
|
82
|
-
# a real failure, bounded because it is unbounded text from a process that
|
|
83
|
-
# just misbehaved.
|
|
84
|
-
_STDERR_EXCERPT = 400
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
def _package_root() -> str:
|
|
88
|
-
"""The directory holding the `tools` package.
|
|
89
|
-
|
|
90
|
-
Derived from __file__, never from the working directory: dispatch has
|
|
91
|
-
no guarantee about cwd -- the TUI runs from wherever it was launched
|
|
92
|
-
and a subagent inherits nothing -- and a child that cannot import its
|
|
93
|
-
own tool would fail as a transport error on every call.
|
|
94
|
-
"""
|
|
95
|
-
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def _child_env() -> dict:
|
|
99
|
-
"""A minimal environment for the child.
|
|
100
|
-
|
|
101
|
-
A math tool needs no credentials, so it is handed none. This is
|
|
102
|
-
security/sandbox._scrubbed_env's argument applied to the other
|
|
103
|
-
subprocess this project spawns, with one difference: the sandbox
|
|
104
|
-
scrubs a shell's inherited environment, while there is nothing to
|
|
105
|
-
scrub here -- the child's environment is built rather than filtered,
|
|
106
|
-
which is the stronger direction.
|
|
107
|
-
|
|
108
|
-
PYTHONPATH is built from the PARENT's resolved sys.path rather than
|
|
109
|
-
from the package root alone. Guessing which variables reconstruct an
|
|
110
|
-
import surface is a losing game -- on Windows the user site-packages
|
|
111
|
-
is derived from APPDATA, a virtualenv needs its own prefix, conda
|
|
112
|
-
needs another -- and getting it wrong presents as ModuleNotFoundError
|
|
113
|
-
for sympy on every math call. Handing the child the paths the parent
|
|
114
|
-
actually resolved is deterministic and gives it exactly the parent's
|
|
115
|
-
imports and no more.
|
|
116
|
-
|
|
117
|
-
SYSTEMROOT is not optional on Windows: without it the interpreter
|
|
118
|
-
cannot initialise its socket/crypto machinery and exits before
|
|
119
|
-
reaching any tool.
|
|
120
|
-
"""
|
|
121
|
-
root = _package_root()
|
|
122
|
-
path = [root] + [p for p in sys.path
|
|
123
|
-
if p and p != root and os.path.isdir(p)]
|
|
124
|
-
env = {"PYTHONPATH": os.pathsep.join(path),
|
|
125
|
-
# No .pyc writes from a process that exists for one call, and
|
|
126
|
-
# no first-call penalty for the directory scan.
|
|
127
|
-
"PYTHONDONTWRITEBYTECODE": "1",
|
|
128
|
-
# Deterministic, and it keeps a child's traceback readable.
|
|
129
|
-
"PYTHONIOENCODING": "utf-8"}
|
|
130
|
-
for name in ("SYSTEMROOT", "SystemRoot", "WINDIR", "PATH", "TEMP", "TMP",
|
|
131
|
-
"LD_LIBRARY_PATH"):
|
|
132
|
-
value = os.environ.get(name)
|
|
133
|
-
if value:
|
|
134
|
-
env[name] = value
|
|
135
|
-
return env
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
def _resource_limits():
|
|
139
|
-
"""H3's inner layer: a preexec_fn setting RLIMIT_CPU and RLIMIT_AS, or
|
|
140
|
-
None where the platform has no rlimit.
|
|
141
|
-
|
|
142
|
-
Deliberately NOT security.sandbox._unix_resource_limits, though it is
|
|
143
|
-
the same three lines and the same idea. That one spends
|
|
144
|
-
SANDBOX_CPU_SECONDS (30), which is ABOVE this budget -- reusing it
|
|
145
|
-
would install a limit that can never fire and read, to anyone
|
|
146
|
-
checking, as though the inner layer were present. The CPU ceiling has
|
|
147
|
-
to be the compute budget or it is decoration.
|
|
148
|
-
|
|
149
|
-
The MEMORY ceiling IS shared with the sandbox, because that number is
|
|
150
|
-
about what one contained subprocess of this harness may hold and
|
|
151
|
-
nothing here argues for a different answer.
|
|
152
|
-
"""
|
|
153
|
-
if platform.system() == "Windows":
|
|
154
|
-
return None
|
|
155
|
-
|
|
156
|
-
import resource
|
|
157
|
-
|
|
158
|
-
cpu_seconds = max(1, int(math.ceil(config.TOOL_COMPUTE_TIMEOUT_S)))
|
|
159
|
-
mem_bytes = config.SANDBOX_MEMORY_MB * 1024 * 1024
|
|
160
|
-
|
|
161
|
-
def _set_limits():
|
|
162
|
-
resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds))
|
|
163
|
-
resource.setrlimit(resource.RLIMIT_AS, (mem_bytes, mem_bytes))
|
|
164
|
-
# The child's stdout and stderr are files since batch 221 (#204),
|
|
165
|
-
# so what it may write to either is capped here: the result bound
|
|
166
|
-
# twice over, which a well-behaved child never approaches. Past it
|
|
167
|
-
# a write fails (CPython ignores SIGXFSZ) and the call reports no
|
|
168
|
-
# usable result. Windows has no rlimit; the wall clock bounds it.
|
|
169
|
-
file_bytes = 2 * MAX_CHILD_OUTPUT_BYTES
|
|
170
|
-
resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes))
|
|
171
|
-
|
|
172
|
-
return _set_limits
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
def _limit_message(tool_name: str, timeout: float) -> str:
|
|
176
|
-
"""What the model is told when a bound stopped the call.
|
|
177
|
-
|
|
178
|
-
ONE wording, because H3 gives the child two bounds on Unix and one
|
|
179
|
-
on Windows and they do not fail the same way. The wall clock raises
|
|
180
|
-
TimeoutExpired in the parent; RLIMIT_CPU kills the child, so
|
|
181
|
-
subprocess.run RETURNS -- normally, with a negative returncode --
|
|
182
|
-
and the parent never sees a timeout at all.
|
|
183
|
-
|
|
184
|
-
They are the same event to the only reader that matters: a model
|
|
185
|
-
deciding whether to ask for something smaller. Two wordings would
|
|
186
|
-
make the better-protected platform give the worse answer, which is
|
|
187
|
-
what CI caught on the first Linux run of this section.
|
|
188
|
-
"""
|
|
189
|
-
return (f"{tool_name} exceeded its {timeout:g}s limit and was stopped. "
|
|
190
|
-
f"The inputs given are too large for this operation -- try "
|
|
191
|
-
f"smaller values.")
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
def _interpret(tool_name: str, stdout: str, stderr: str, returncode: int,
|
|
195
|
-
timeout: float, stdout_bytes: Optional[int] = None) -> dict:
|
|
196
|
-
"""Turn what the child said into the dict dispatch expects.
|
|
197
|
-
|
|
198
|
-
Every branch returns rather than raises. dispatch's own containment
|
|
199
|
-
would convert an exception into an error dict anyway, but it would log
|
|
200
|
-
a traceback with it -- and an oversized factorial is an ordinary thing
|
|
201
|
-
for a model to ask for, not a bug in this harness.
|
|
202
|
-
|
|
203
|
-
`stdout_bytes` is the size the child WROTE, which `stdout` no longer
|
|
204
|
-
holds whole since batch 221: the parent reads only the bound plus one.
|
|
205
|
-
"""
|
|
206
|
-
size = len(stdout) if stdout_bytes is None else stdout_bytes
|
|
207
|
-
if size > MAX_CHILD_OUTPUT_BYTES:
|
|
208
|
-
return {"error": f"{tool_name} produced more output than can be "
|
|
209
|
-
f"returned ({size} bytes). Ask for a smaller "
|
|
210
|
-
f"result."}
|
|
211
|
-
|
|
212
|
-
for line in stdout.splitlines():
|
|
213
|
-
if line.startswith(MARKER):
|
|
214
|
-
try:
|
|
215
|
-
return json.loads(line[len(MARKER):])
|
|
216
|
-
except ValueError:
|
|
217
|
-
break
|
|
218
|
-
|
|
219
|
-
# A NEGATIVE returncode means the child was killed by a signal, and
|
|
220
|
-
# for a child this module launched that means a bound fired --
|
|
221
|
-
# RLIMIT_CPU raises SIGXCPU and then SIGKILL at the hard limit
|
|
222
|
-
# (H3's inner layer), which is the ONLY way a compute child dies
|
|
223
|
-
# without either answering or timing out in the parent.
|
|
224
|
-
#
|
|
225
|
-
# Checked before the generic branch, not after, and reported with
|
|
226
|
-
# the same sentence the wall clock uses. Windows has no rlimit and
|
|
227
|
-
# signals a killed process differently, so this is unreachable
|
|
228
|
-
# there -- which is exactly why it went unnoticed until CI ran the
|
|
229
|
-
# suite on Linux.
|
|
230
|
-
if returncode is not None and returncode < 0:
|
|
231
|
-
logger.info("%s was stopped by its resource limit (signal %d).",
|
|
232
|
-
tool_name, -returncode)
|
|
233
|
-
return {"error": _limit_message(tool_name, timeout)}
|
|
234
|
-
|
|
235
|
-
excerpt = (stderr or "").strip().replace("\n", " ")[-_STDERR_EXCERPT:]
|
|
236
|
-
# logger.warning, not exception: there is no traceback here to keep,
|
|
237
|
-
# and this path is reachable by ordinary input.
|
|
238
|
-
logger.warning("%s produced no readable result (exit %s): %s",
|
|
239
|
-
tool_name, returncode, excerpt)
|
|
240
|
-
return {"error": f"{tool_name} did not return a usable result "
|
|
241
|
-
f"(exit {returncode}). {excerpt}".strip()}
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
def run_isolated(handler, params: dict, timeout: float,
|
|
245
|
-
tool_name: str = "", confine: Optional[bool] = None) -> dict:
|
|
246
|
-
"""Run `handler` in a killable child and return its result dict.
|
|
247
|
-
|
|
248
|
-
`handler` is resolved by module and qualified name in the child rather
|
|
249
|
-
than pickled: the registry's handlers are plain module-level functions
|
|
250
|
-
and naming them keeps the child's import surface to the one tool.
|
|
251
|
-
Both names come from the registry, never from a model.
|
|
252
|
-
|
|
253
|
-
`confine` walls the child in (security/confine.py) once it has loaded
|
|
254
|
-
the tool and before it reads `params`. None means "when this is a
|
|
255
|
-
shared hosted deployment" (§79 HD11), the one place a stranger chooses
|
|
256
|
-
what a math tool evaluates.
|
|
257
|
-
"""
|
|
258
|
-
if confine is None:
|
|
259
|
-
from security import hosted
|
|
260
|
-
|
|
261
|
-
confine = hosted.is_shared()
|
|
262
|
-
tool_name = tool_name or getattr(handler, "__name__", "tool")
|
|
263
|
-
module = getattr(handler, "__module__", None)
|
|
264
|
-
function = getattr(handler, "__name__", None)
|
|
265
|
-
if not module or not function:
|
|
266
|
-
# A handler that is not a named module-level function cannot be
|
|
267
|
-
# reached from a fresh interpreter. Refuse rather than fall back to
|
|
268
|
-
# an in-process call, which would silently drop the bound this
|
|
269
|
-
# whole module exists to apply.
|
|
270
|
-
return {"error": f"{tool_name} cannot be isolated: its handler is "
|
|
271
|
-
f"not a named module-level function."}
|
|
272
|
-
|
|
273
|
-
request = json.dumps({"module": module, "function": function,
|
|
274
|
-
"params": params, "max_bytes":
|
|
275
|
-
MAX_CHILD_OUTPUT_BYTES, "confine": bool(confine),
|
|
276
|
-
"tool_name": tool_name})
|
|
277
|
-
|
|
278
|
-
popen_kwargs = {}
|
|
279
|
-
if platform.system() != "Windows":
|
|
280
|
-
popen_kwargs["preexec_fn"] = _resource_limits()
|
|
281
|
-
|
|
282
|
-
# THE TRANSPORT IS BOUNDED TOO, since batch 221 (#204). The 4 MB bound
|
|
283
|
-
# is the child's, on the result it prints; `capture_output` then held
|
|
284
|
-
# EVERYTHING the child wrote -- progress spam, a warning flood, a giant
|
|
285
|
-
# traceback, ordinary misbehaviour for the calls this module contains --
|
|
286
|
-
# in the parent's memory until the wall clock fired. Temporary files
|
|
287
|
-
# take the pipes' place, the parent reads at most the bound from one and
|
|
288
|
-
# a tail from the other, and on POSIX `_resource_limits` caps what the
|
|
289
|
-
# child may write to either. No thread reads anything (H2).
|
|
290
|
-
with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
|
|
291
|
-
try:
|
|
292
|
-
child = subprocess.run(
|
|
293
|
-
[sys.executable, "-m", "tools.isolation"],
|
|
294
|
-
input=request.encode("utf-8"),
|
|
295
|
-
stdout=out,
|
|
296
|
-
stderr=err,
|
|
297
|
-
timeout=timeout,
|
|
298
|
-
cwd=_package_root(),
|
|
299
|
-
env=_child_env(),
|
|
300
|
-
**popen_kwargs,
|
|
301
|
-
)
|
|
302
|
-
except subprocess.TimeoutExpired:
|
|
303
|
-
# subprocess.run has already killed the child and reaped it,
|
|
304
|
-
# which is the property that makes this a bound rather than a
|
|
305
|
-
# report.
|
|
306
|
-
#
|
|
307
|
-
# On Unix the CPU limit usually gets there first and this
|
|
308
|
-
# branch never runs -- see _interpret's negative-returncode
|
|
309
|
-
# case, which shares this wording rather than repeating it.
|
|
310
|
-
return {"error": _limit_message(tool_name, timeout)}
|
|
311
|
-
except OSError as e:
|
|
312
|
-
logger.warning("Could not start an isolated child for %s: %s",
|
|
313
|
-
tool_name, e)
|
|
314
|
-
return {"error": f"{tool_name} could not be run: {e}"}
|
|
315
|
-
stdout, stdout_bytes = _head(out, MAX_CHILD_OUTPUT_BYTES + 1)
|
|
316
|
-
stderr = _tail(err, 4 * _STDERR_EXCERPT)
|
|
317
|
-
|
|
318
|
-
return _interpret(tool_name, stdout, stderr, child.returncode, timeout,
|
|
319
|
-
stdout_bytes=stdout_bytes)
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
def _head(handle, limit: int) -> tuple:
|
|
323
|
-
"""(the first `limit` bytes of `handle` as text, its whole size)."""
|
|
324
|
-
size = os.fstat(handle.fileno()).st_size
|
|
325
|
-
handle.seek(0)
|
|
326
|
-
# min(), because read(n) allocates n bytes up front: 4 MB for a result
|
|
327
|
-
# of forty, which the tracemalloc test below measured.
|
|
328
|
-
return _text(handle.read(min(limit, size))), size
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
def _tail(handle, limit: int) -> str:
|
|
332
|
-
"""The last `limit` bytes of `handle` as text: the only part of the
|
|
333
|
-
child's stderr anything reads is the end of it."""
|
|
334
|
-
size = os.fstat(handle.fileno()).st_size
|
|
335
|
-
handle.seek(max(0, size - limit))
|
|
336
|
-
return _text(handle.read())
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
def _text(raw: bytes) -> str:
|
|
340
|
-
"""Decoded as the text-mode pipes decoded it: UTF-8 with replacement,
|
|
341
|
-
and universal newlines, so a Windows child's \\r\\n reads as \\n."""
|
|
342
|
-
return (raw.decode("utf-8", errors="replace")
|
|
343
|
-
.replace("\r\n", "\n").replace("\r", "\n"))
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
def _child_main() -> int:
|
|
347
|
-
"""The child. Reads one request from stdin, prints one marked result.
|
|
348
|
-
|
|
349
|
-
Kept deliberately small: everything it does before the handler runs is
|
|
350
|
-
time the wall clock is already counting.
|
|
351
|
-
"""
|
|
352
|
-
request = json.loads(sys.stdin.read())
|
|
353
|
-
import importlib
|
|
354
|
-
|
|
355
|
-
module = importlib.import_module(request["module"])
|
|
356
|
-
handler = getattr(module, request["function"])
|
|
357
|
-
if request.get("confine"):
|
|
358
|
-
# AFTER the import, which is what needs the filesystem, and BEFORE
|
|
359
|
-
# the params are touched, which is where a visitor's expression is.
|
|
360
|
-
# A wall that will not go up means the call does not run: there is
|
|
361
|
-
# no unconfined fallback to fall to (security/confine.py).
|
|
362
|
-
from security import confine
|
|
363
|
-
|
|
364
|
-
reason = confine.lock_down()
|
|
365
|
-
if reason:
|
|
366
|
-
name = request.get("tool_name") or request["function"]
|
|
367
|
-
sys.stdout.write(MARKER + json.dumps({
|
|
368
|
-
"error": f"{name} was not run: this deployment could not "
|
|
369
|
-
f"confine it ({reason})."}) + "\n")
|
|
370
|
-
return 0
|
|
371
|
-
result = handler(request["params"])
|
|
372
|
-
|
|
373
|
-
payload = json.dumps(result, default=str)
|
|
374
|
-
limit = request.get("max_bytes", MAX_CHILD_OUTPUT_BYTES)
|
|
375
|
-
if len(payload) > limit:
|
|
376
|
-
# Refused HERE, where the size is known before anything is
|
|
377
|
-
# written. The parent checks too, but this is the producer and
|
|
378
|
-
# #55's whole point is that the producer is where a bound belongs.
|
|
379
|
-
payload = json.dumps({
|
|
380
|
-
"error": "the result is too large to return "
|
|
381
|
-
f"({len(payload)} bytes). Ask for a smaller result."})
|
|
382
|
-
sys.stdout.write(MARKER + payload + "\n")
|
|
383
|
-
return 0
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
if __name__ == "__main__":
|
|
387
|
-
sys.exit(_child_main())
|
|
1
|
+
"""
|
|
2
|
+
tools/isolation.py
|
|
3
|
+
|
|
4
|
+
ROADMAP_v2 §31 (H2, H3). A wall clock for tool calls that can actually
|
|
5
|
+
stop the work.
|
|
6
|
+
|
|
7
|
+
WHY A PROCESS AND NOT A THREAD, WHICH IS THE WHOLE DECISION.
|
|
8
|
+
|
|
9
|
+
#57 was deferred once on the grounds that `signal.alarm` is main-thread
|
|
10
|
+
only and the TUI runs tools in a worker, so a thread-based watchdog was
|
|
11
|
+
the shape -- and "abandoning a thread mid-sympy is not clean". That is
|
|
12
|
+
true, and it is an argument against the thread rather than against the
|
|
13
|
+
bound. A thread cannot be cancelled in CPython: there is no interruption
|
|
14
|
+
point inside `sympy.binomial` for anything to check, so a watchdog that
|
|
15
|
+
times out returns an error while the core stays busy until the process
|
|
16
|
+
exits. The harness would then report a timeout that did not happen, which
|
|
17
|
+
is the class of mechanism this project keeps deciding against -- the same
|
|
18
|
+
reason `with_catalogs` suppresses a catalog rather than showing one whose
|
|
19
|
+
tool cannot be called.
|
|
20
|
+
|
|
21
|
+
A subprocess has the interruption point the thread lacks: the OS. Killing
|
|
22
|
+
it reclaims the CPU, which
|
|
23
|
+
`test_a_timed_out_call_is_STOPPED_not_merely_abandoned` measures directly
|
|
24
|
+
-- by evidence the child leaves on disk, not by os.times(), whose
|
|
25
|
+
children_user field is always 0.0 on Windows and would make the
|
|
26
|
+
assertion vacuous on the platform it usually runs on. That measurement is
|
|
27
|
+
the difference between the two designs rather than a detail of this one.
|
|
28
|
+
|
|
29
|
+
WHAT IT COSTS. A fresh interpreter plus sympy plus the tool module is
|
|
30
|
+
~0.34s measured, against a harness where every step is a multi-second
|
|
31
|
+
model call. No warm pool: a pool holds interpreters that a killed call
|
|
32
|
+
would have to replace anyway, and the state a pooled worker accumulates
|
|
33
|
+
is exactly what a bound is supposed to discard.
|
|
34
|
+
|
|
35
|
+
MODELLED ON mcp_client.client.call_tool, which had this shape first and
|
|
36
|
+
wrote down why: an inner bound that stops the work (there,
|
|
37
|
+
`read_timeout_seconds`; here, RLIMIT_CPU) with an outer wall clock as the
|
|
38
|
+
backstop "for the case the answer is never coming at all". On Windows
|
|
39
|
+
there is no rlimit and no cheap equivalent, so the clock runs alone --
|
|
40
|
+
H3 records that asymmetry rather than papering over it.
|
|
41
|
+
|
|
42
|
+
WHAT CROSSES THE BOUNDARY. Params in as JSON, the result dict out as
|
|
43
|
+
JSON, and nothing else. That is not a limitation to work around: it is
|
|
44
|
+
the check that a BUDGET_COMPUTE tool really is a pure function of its
|
|
45
|
+
params, which `assert_budget_declared` enforces from the other side by
|
|
46
|
+
refusing a compute tool that declares an injection.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
from __future__ import annotations
|
|
50
|
+
|
|
51
|
+
import json
|
|
52
|
+
import logging
|
|
53
|
+
import math
|
|
54
|
+
import os
|
|
55
|
+
import platform
|
|
56
|
+
import subprocess
|
|
57
|
+
import sys
|
|
58
|
+
import tempfile
|
|
59
|
+
from typing import Optional
|
|
60
|
+
|
|
61
|
+
import config
|
|
62
|
+
|
|
63
|
+
logger = logging.getLogger(__name__)
|
|
64
|
+
|
|
65
|
+
# The child prints one JSON object on this marker line. A marker rather
|
|
66
|
+
# than "parse the whole of stdout" because an imported module is entitled
|
|
67
|
+
# to write to stdout -- a deprecation notice from a pinned SymPy would
|
|
68
|
+
# otherwise be indistinguishable from a malformed result.
|
|
69
|
+
MARKER = "@@VENASTINE_RESULT@@"
|
|
70
|
+
|
|
71
|
+
# Bounded on the CHILD side, where the size is known before anything is
|
|
72
|
+
# written, and re-checked here. #55's own rule -- bound the read, do not
|
|
73
|
+
# slice afterwards -- applied to the mechanism that fixes #55. That bounds
|
|
74
|
+
# the RESULT; the TRANSPORT carrying it was unbounded until batch 221
|
|
75
|
+
# (#204), and `run_isolated` says how it is bounded now. The value
|
|
76
|
+
# is generous next to what a math tool can actually return: CPython caps
|
|
77
|
+
# int->str at 4300 digits, so the realistic ceiling is far below this and
|
|
78
|
+
# the check is for the pathological case rather than the ordinary one.
|
|
79
|
+
MAX_CHILD_OUTPUT_BYTES = 4_000_000
|
|
80
|
+
|
|
81
|
+
# How much of the child's stderr reaches the error message. Enough to name
|
|
82
|
+
# a real failure, bounded because it is unbounded text from a process that
|
|
83
|
+
# just misbehaved.
|
|
84
|
+
_STDERR_EXCERPT = 400
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _package_root() -> str:
|
|
88
|
+
"""The directory holding the `tools` package.
|
|
89
|
+
|
|
90
|
+
Derived from __file__, never from the working directory: dispatch has
|
|
91
|
+
no guarantee about cwd -- the TUI runs from wherever it was launched
|
|
92
|
+
and a subagent inherits nothing -- and a child that cannot import its
|
|
93
|
+
own tool would fail as a transport error on every call.
|
|
94
|
+
"""
|
|
95
|
+
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _child_env() -> dict:
|
|
99
|
+
"""A minimal environment for the child.
|
|
100
|
+
|
|
101
|
+
A math tool needs no credentials, so it is handed none. This is
|
|
102
|
+
security/sandbox._scrubbed_env's argument applied to the other
|
|
103
|
+
subprocess this project spawns, with one difference: the sandbox
|
|
104
|
+
scrubs a shell's inherited environment, while there is nothing to
|
|
105
|
+
scrub here -- the child's environment is built rather than filtered,
|
|
106
|
+
which is the stronger direction.
|
|
107
|
+
|
|
108
|
+
PYTHONPATH is built from the PARENT's resolved sys.path rather than
|
|
109
|
+
from the package root alone. Guessing which variables reconstruct an
|
|
110
|
+
import surface is a losing game -- on Windows the user site-packages
|
|
111
|
+
is derived from APPDATA, a virtualenv needs its own prefix, conda
|
|
112
|
+
needs another -- and getting it wrong presents as ModuleNotFoundError
|
|
113
|
+
for sympy on every math call. Handing the child the paths the parent
|
|
114
|
+
actually resolved is deterministic and gives it exactly the parent's
|
|
115
|
+
imports and no more.
|
|
116
|
+
|
|
117
|
+
SYSTEMROOT is not optional on Windows: without it the interpreter
|
|
118
|
+
cannot initialise its socket/crypto machinery and exits before
|
|
119
|
+
reaching any tool.
|
|
120
|
+
"""
|
|
121
|
+
root = _package_root()
|
|
122
|
+
path = [root] + [p for p in sys.path
|
|
123
|
+
if p and p != root and os.path.isdir(p)]
|
|
124
|
+
env = {"PYTHONPATH": os.pathsep.join(path),
|
|
125
|
+
# No .pyc writes from a process that exists for one call, and
|
|
126
|
+
# no first-call penalty for the directory scan.
|
|
127
|
+
"PYTHONDONTWRITEBYTECODE": "1",
|
|
128
|
+
# Deterministic, and it keeps a child's traceback readable.
|
|
129
|
+
"PYTHONIOENCODING": "utf-8"}
|
|
130
|
+
for name in ("SYSTEMROOT", "SystemRoot", "WINDIR", "PATH", "TEMP", "TMP",
|
|
131
|
+
"LD_LIBRARY_PATH"):
|
|
132
|
+
value = os.environ.get(name)
|
|
133
|
+
if value:
|
|
134
|
+
env[name] = value
|
|
135
|
+
return env
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _resource_limits():
|
|
139
|
+
"""H3's inner layer: a preexec_fn setting RLIMIT_CPU and RLIMIT_AS, or
|
|
140
|
+
None where the platform has no rlimit.
|
|
141
|
+
|
|
142
|
+
Deliberately NOT security.sandbox._unix_resource_limits, though it is
|
|
143
|
+
the same three lines and the same idea. That one spends
|
|
144
|
+
SANDBOX_CPU_SECONDS (30), which is ABOVE this budget -- reusing it
|
|
145
|
+
would install a limit that can never fire and read, to anyone
|
|
146
|
+
checking, as though the inner layer were present. The CPU ceiling has
|
|
147
|
+
to be the compute budget or it is decoration.
|
|
148
|
+
|
|
149
|
+
The MEMORY ceiling IS shared with the sandbox, because that number is
|
|
150
|
+
about what one contained subprocess of this harness may hold and
|
|
151
|
+
nothing here argues for a different answer.
|
|
152
|
+
"""
|
|
153
|
+
if platform.system() == "Windows":
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
import resource
|
|
157
|
+
|
|
158
|
+
cpu_seconds = max(1, int(math.ceil(config.TOOL_COMPUTE_TIMEOUT_S)))
|
|
159
|
+
mem_bytes = config.SANDBOX_MEMORY_MB * 1024 * 1024
|
|
160
|
+
|
|
161
|
+
def _set_limits():
|
|
162
|
+
resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds))
|
|
163
|
+
resource.setrlimit(resource.RLIMIT_AS, (mem_bytes, mem_bytes))
|
|
164
|
+
# The child's stdout and stderr are files since batch 221 (#204),
|
|
165
|
+
# so what it may write to either is capped here: the result bound
|
|
166
|
+
# twice over, which a well-behaved child never approaches. Past it
|
|
167
|
+
# a write fails (CPython ignores SIGXFSZ) and the call reports no
|
|
168
|
+
# usable result. Windows has no rlimit; the wall clock bounds it.
|
|
169
|
+
file_bytes = 2 * MAX_CHILD_OUTPUT_BYTES
|
|
170
|
+
resource.setrlimit(resource.RLIMIT_FSIZE, (file_bytes, file_bytes))
|
|
171
|
+
|
|
172
|
+
return _set_limits
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _limit_message(tool_name: str, timeout: float) -> str:
|
|
176
|
+
"""What the model is told when a bound stopped the call.
|
|
177
|
+
|
|
178
|
+
ONE wording, because H3 gives the child two bounds on Unix and one
|
|
179
|
+
on Windows and they do not fail the same way. The wall clock raises
|
|
180
|
+
TimeoutExpired in the parent; RLIMIT_CPU kills the child, so
|
|
181
|
+
subprocess.run RETURNS -- normally, with a negative returncode --
|
|
182
|
+
and the parent never sees a timeout at all.
|
|
183
|
+
|
|
184
|
+
They are the same event to the only reader that matters: a model
|
|
185
|
+
deciding whether to ask for something smaller. Two wordings would
|
|
186
|
+
make the better-protected platform give the worse answer, which is
|
|
187
|
+
what CI caught on the first Linux run of this section.
|
|
188
|
+
"""
|
|
189
|
+
return (f"{tool_name} exceeded its {timeout:g}s limit and was stopped. "
|
|
190
|
+
f"The inputs given are too large for this operation -- try "
|
|
191
|
+
f"smaller values.")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _interpret(tool_name: str, stdout: str, stderr: str, returncode: int,
|
|
195
|
+
timeout: float, stdout_bytes: Optional[int] = None) -> dict:
|
|
196
|
+
"""Turn what the child said into the dict dispatch expects.
|
|
197
|
+
|
|
198
|
+
Every branch returns rather than raises. dispatch's own containment
|
|
199
|
+
would convert an exception into an error dict anyway, but it would log
|
|
200
|
+
a traceback with it -- and an oversized factorial is an ordinary thing
|
|
201
|
+
for a model to ask for, not a bug in this harness.
|
|
202
|
+
|
|
203
|
+
`stdout_bytes` is the size the child WROTE, which `stdout` no longer
|
|
204
|
+
holds whole since batch 221: the parent reads only the bound plus one.
|
|
205
|
+
"""
|
|
206
|
+
size = len(stdout) if stdout_bytes is None else stdout_bytes
|
|
207
|
+
if size > MAX_CHILD_OUTPUT_BYTES:
|
|
208
|
+
return {"error": f"{tool_name} produced more output than can be "
|
|
209
|
+
f"returned ({size} bytes). Ask for a smaller "
|
|
210
|
+
f"result."}
|
|
211
|
+
|
|
212
|
+
for line in stdout.splitlines():
|
|
213
|
+
if line.startswith(MARKER):
|
|
214
|
+
try:
|
|
215
|
+
return json.loads(line[len(MARKER):])
|
|
216
|
+
except ValueError:
|
|
217
|
+
break
|
|
218
|
+
|
|
219
|
+
# A NEGATIVE returncode means the child was killed by a signal, and
|
|
220
|
+
# for a child this module launched that means a bound fired --
|
|
221
|
+
# RLIMIT_CPU raises SIGXCPU and then SIGKILL at the hard limit
|
|
222
|
+
# (H3's inner layer), which is the ONLY way a compute child dies
|
|
223
|
+
# without either answering or timing out in the parent.
|
|
224
|
+
#
|
|
225
|
+
# Checked before the generic branch, not after, and reported with
|
|
226
|
+
# the same sentence the wall clock uses. Windows has no rlimit and
|
|
227
|
+
# signals a killed process differently, so this is unreachable
|
|
228
|
+
# there -- which is exactly why it went unnoticed until CI ran the
|
|
229
|
+
# suite on Linux.
|
|
230
|
+
if returncode is not None and returncode < 0:
|
|
231
|
+
logger.info("%s was stopped by its resource limit (signal %d).",
|
|
232
|
+
tool_name, -returncode)
|
|
233
|
+
return {"error": _limit_message(tool_name, timeout)}
|
|
234
|
+
|
|
235
|
+
excerpt = (stderr or "").strip().replace("\n", " ")[-_STDERR_EXCERPT:]
|
|
236
|
+
# logger.warning, not exception: there is no traceback here to keep,
|
|
237
|
+
# and this path is reachable by ordinary input.
|
|
238
|
+
logger.warning("%s produced no readable result (exit %s): %s",
|
|
239
|
+
tool_name, returncode, excerpt)
|
|
240
|
+
return {"error": f"{tool_name} did not return a usable result "
|
|
241
|
+
f"(exit {returncode}). {excerpt}".strip()}
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def run_isolated(handler, params: dict, timeout: float,
|
|
245
|
+
tool_name: str = "", confine: Optional[bool] = None) -> dict:
|
|
246
|
+
"""Run `handler` in a killable child and return its result dict.
|
|
247
|
+
|
|
248
|
+
`handler` is resolved by module and qualified name in the child rather
|
|
249
|
+
than pickled: the registry's handlers are plain module-level functions
|
|
250
|
+
and naming them keeps the child's import surface to the one tool.
|
|
251
|
+
Both names come from the registry, never from a model.
|
|
252
|
+
|
|
253
|
+
`confine` walls the child in (security/confine.py) once it has loaded
|
|
254
|
+
the tool and before it reads `params`. None means "when this is a
|
|
255
|
+
shared hosted deployment" (§79 HD11), the one place a stranger chooses
|
|
256
|
+
what a math tool evaluates.
|
|
257
|
+
"""
|
|
258
|
+
if confine is None:
|
|
259
|
+
from security import hosted
|
|
260
|
+
|
|
261
|
+
confine = hosted.is_shared()
|
|
262
|
+
tool_name = tool_name or getattr(handler, "__name__", "tool")
|
|
263
|
+
module = getattr(handler, "__module__", None)
|
|
264
|
+
function = getattr(handler, "__name__", None)
|
|
265
|
+
if not module or not function:
|
|
266
|
+
# A handler that is not a named module-level function cannot be
|
|
267
|
+
# reached from a fresh interpreter. Refuse rather than fall back to
|
|
268
|
+
# an in-process call, which would silently drop the bound this
|
|
269
|
+
# whole module exists to apply.
|
|
270
|
+
return {"error": f"{tool_name} cannot be isolated: its handler is "
|
|
271
|
+
f"not a named module-level function."}
|
|
272
|
+
|
|
273
|
+
request = json.dumps({"module": module, "function": function,
|
|
274
|
+
"params": params, "max_bytes":
|
|
275
|
+
MAX_CHILD_OUTPUT_BYTES, "confine": bool(confine),
|
|
276
|
+
"tool_name": tool_name})
|
|
277
|
+
|
|
278
|
+
popen_kwargs = {}
|
|
279
|
+
if platform.system() != "Windows":
|
|
280
|
+
popen_kwargs["preexec_fn"] = _resource_limits()
|
|
281
|
+
|
|
282
|
+
# THE TRANSPORT IS BOUNDED TOO, since batch 221 (#204). The 4 MB bound
|
|
283
|
+
# is the child's, on the result it prints; `capture_output` then held
|
|
284
|
+
# EVERYTHING the child wrote -- progress spam, a warning flood, a giant
|
|
285
|
+
# traceback, ordinary misbehaviour for the calls this module contains --
|
|
286
|
+
# in the parent's memory until the wall clock fired. Temporary files
|
|
287
|
+
# take the pipes' place, the parent reads at most the bound from one and
|
|
288
|
+
# a tail from the other, and on POSIX `_resource_limits` caps what the
|
|
289
|
+
# child may write to either. No thread reads anything (H2).
|
|
290
|
+
with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
|
|
291
|
+
try:
|
|
292
|
+
child = subprocess.run(
|
|
293
|
+
[sys.executable, "-m", "tools.isolation"],
|
|
294
|
+
input=request.encode("utf-8"),
|
|
295
|
+
stdout=out,
|
|
296
|
+
stderr=err,
|
|
297
|
+
timeout=timeout,
|
|
298
|
+
cwd=_package_root(),
|
|
299
|
+
env=_child_env(),
|
|
300
|
+
**popen_kwargs,
|
|
301
|
+
)
|
|
302
|
+
except subprocess.TimeoutExpired:
|
|
303
|
+
# subprocess.run has already killed the child and reaped it,
|
|
304
|
+
# which is the property that makes this a bound rather than a
|
|
305
|
+
# report.
|
|
306
|
+
#
|
|
307
|
+
# On Unix the CPU limit usually gets there first and this
|
|
308
|
+
# branch never runs -- see _interpret's negative-returncode
|
|
309
|
+
# case, which shares this wording rather than repeating it.
|
|
310
|
+
return {"error": _limit_message(tool_name, timeout)}
|
|
311
|
+
except OSError as e:
|
|
312
|
+
logger.warning("Could not start an isolated child for %s: %s",
|
|
313
|
+
tool_name, e)
|
|
314
|
+
return {"error": f"{tool_name} could not be run: {e}"}
|
|
315
|
+
stdout, stdout_bytes = _head(out, MAX_CHILD_OUTPUT_BYTES + 1)
|
|
316
|
+
stderr = _tail(err, 4 * _STDERR_EXCERPT)
|
|
317
|
+
|
|
318
|
+
return _interpret(tool_name, stdout, stderr, child.returncode, timeout,
|
|
319
|
+
stdout_bytes=stdout_bytes)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _head(handle, limit: int) -> tuple:
|
|
323
|
+
"""(the first `limit` bytes of `handle` as text, its whole size)."""
|
|
324
|
+
size = os.fstat(handle.fileno()).st_size
|
|
325
|
+
handle.seek(0)
|
|
326
|
+
# min(), because read(n) allocates n bytes up front: 4 MB for a result
|
|
327
|
+
# of forty, which the tracemalloc test below measured.
|
|
328
|
+
return _text(handle.read(min(limit, size))), size
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _tail(handle, limit: int) -> str:
|
|
332
|
+
"""The last `limit` bytes of `handle` as text: the only part of the
|
|
333
|
+
child's stderr anything reads is the end of it."""
|
|
334
|
+
size = os.fstat(handle.fileno()).st_size
|
|
335
|
+
handle.seek(max(0, size - limit))
|
|
336
|
+
return _text(handle.read())
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _text(raw: bytes) -> str:
|
|
340
|
+
"""Decoded as the text-mode pipes decoded it: UTF-8 with replacement,
|
|
341
|
+
and universal newlines, so a Windows child's \\r\\n reads as \\n."""
|
|
342
|
+
return (raw.decode("utf-8", errors="replace")
|
|
343
|
+
.replace("\r\n", "\n").replace("\r", "\n"))
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _child_main() -> int:
|
|
347
|
+
"""The child. Reads one request from stdin, prints one marked result.
|
|
348
|
+
|
|
349
|
+
Kept deliberately small: everything it does before the handler runs is
|
|
350
|
+
time the wall clock is already counting.
|
|
351
|
+
"""
|
|
352
|
+
request = json.loads(sys.stdin.read())
|
|
353
|
+
import importlib
|
|
354
|
+
|
|
355
|
+
module = importlib.import_module(request["module"])
|
|
356
|
+
handler = getattr(module, request["function"])
|
|
357
|
+
if request.get("confine"):
|
|
358
|
+
# AFTER the import, which is what needs the filesystem, and BEFORE
|
|
359
|
+
# the params are touched, which is where a visitor's expression is.
|
|
360
|
+
# A wall that will not go up means the call does not run: there is
|
|
361
|
+
# no unconfined fallback to fall to (security/confine.py).
|
|
362
|
+
from security import confine
|
|
363
|
+
|
|
364
|
+
reason = confine.lock_down()
|
|
365
|
+
if reason:
|
|
366
|
+
name = request.get("tool_name") or request["function"]
|
|
367
|
+
sys.stdout.write(MARKER + json.dumps({
|
|
368
|
+
"error": f"{name} was not run: this deployment could not "
|
|
369
|
+
f"confine it ({reason})."}) + "\n")
|
|
370
|
+
return 0
|
|
371
|
+
result = handler(request["params"])
|
|
372
|
+
|
|
373
|
+
payload = json.dumps(result, default=str)
|
|
374
|
+
limit = request.get("max_bytes", MAX_CHILD_OUTPUT_BYTES)
|
|
375
|
+
if len(payload) > limit:
|
|
376
|
+
# Refused HERE, where the size is known before anything is
|
|
377
|
+
# written. The parent checks too, but this is the producer and
|
|
378
|
+
# #55's whole point is that the producer is where a bound belongs.
|
|
379
|
+
payload = json.dumps({
|
|
380
|
+
"error": "the result is too large to return "
|
|
381
|
+
f"({len(payload)} bytes). Ask for a smaller result."})
|
|
382
|
+
sys.stdout.write(MARKER + payload + "\n")
|
|
383
|
+
return 0
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
if __name__ == "__main__":
|
|
387
|
+
sys.exit(_child_main())
|