@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
package/tools/base.py ADDED
@@ -0,0 +1,305 @@
1
+ from dataclasses import dataclass
2
+ from typing import Callable, Iterable, Optional
3
+
4
+
5
+ # ROADMAP_v2 §25 (R13). How far approving a tool BY NAME goes -- the
6
+ # question asked before any call exists, by the §18 subagent sign-off and
7
+ # by the §25 pipeline grant.
8
+ #
9
+ # Three values rather than two booleans. Booleans can express "an
10
+ # unattended pipeline run may pre-grant this, but a human answering in the
11
+ # moment may not", which is incoherent -- the pipeline is strictly
12
+ # stricter than the sign-off, so the answers form an order.
13
+ GRANT_ANYWHERE = "anywhere"
14
+ GRANT_SIGNOFF_ONLY = "signoff_only"
15
+ GRANT_NEVER = "never"
16
+
17
+ GRANT_POLICIES = frozenset({GRANT_ANYWHERE, GRANT_SIGNOFF_ONLY, GRANT_NEVER})
18
+
19
+
20
+ # ROADMAP_v2 §31 (H1). What a call to this tool is allowed to COST, and
21
+ # therefore which mechanism bounds it. Asked once, at registration, for
22
+ # the same reason grant_policy is: the answer is a property of the tool,
23
+ # and a layer that infers it from a name or a module is a second source
24
+ # of truth that drifts the moment a tool is added.
25
+ #
26
+ # BUDGET_COMPUTE A pure function of its params. Nothing bounds it from
27
+ # the INSIDE -- sympy has no interruption points -- so
28
+ # dispatch runs it in a killable subprocess under
29
+ # config.TOOL_COMPUTE_TIMEOUT_S (H2).
30
+ # BUDGET_IO Carries its own bound already: a request timeout, a
31
+ # size check before opening, a sandbox, or an operation
32
+ # bounded by construction. NOT wrapped by dispatch,
33
+ # because an outer clock over a blocking read can only
34
+ # stop WAITING -- mcp_client says so about its own
35
+ # backstop, and a mechanism that reports a timeout while
36
+ # the work continues is worse than none (H7).
37
+ # BUDGET_HUMAN Blocks on a person, or on a sub-run metered
38
+ # separately. ask_user waits up to
39
+ # ATTENDED_APPROVAL_TIMEOUT_S; spawn_subagent is bounded
40
+ # by the child's max_steps and the token budget. A
41
+ # compute clock over either would be a bug, not a
42
+ # tightening (H8).
43
+ BUDGET_COMPUTE = "compute"
44
+ BUDGET_IO = "io"
45
+ BUDGET_HUMAN = "human"
46
+
47
+ BUDGETS = frozenset({BUDGET_COMPUTE, BUDGET_IO, BUDGET_HUMAN})
48
+
49
+
50
+ @dataclass
51
+ class ToolSpec:
52
+ name: str # Tool name
53
+ schema: dict # Tool schema
54
+ handler: Callable[[dict], dict] # The actual function the tool runs
55
+ # Optional path-dependent approval policy. When present, its result is
56
+ # OR'd with the generic requires_approval() lookup by
57
+ # registry.approval_needed(). Signature: (tool_name, params) -> bool.
58
+ # Used by file_ops to auto-approve within WORKSPACE_DIR and require
59
+ # user approval outside it, and by shell for Docker-availability.
60
+ #
61
+ # ROADMAP_v2 §15 changed this from a REPLACEMENT for requires_approval
62
+ # to one term of an OR. Before, defining an approval_check made a tool
63
+ # invisible to config/context-level approval settings entirely, which
64
+ # would have silently voided agent approval_overrides for read/write/
65
+ # edit/shell -- the four tools where per-agent tightening matters most.
66
+ approval_check: Optional[Callable[[str, dict], bool]] = None
67
+ # Optional "is this tool meaningful right now?" predicate, consulted by
68
+ # registry.schemas() when deciding what to advertise to the model.
69
+ # Signature: () -> bool. Distinct from permissions: the tool is allowed,
70
+ # it just has nothing to act on yet (load_skill with an empty catalog).
71
+ # Not consulted by dispatch() -- a tool declaring itself unavailable is
72
+ # expected to return a clean error if called anyway, and a second denial
73
+ # path would only add a worse message.
74
+ available_check: Optional[Callable[[], bool]] = None
75
+ # Optional "once approved in this run, don't ask again" marker. Only
76
+ # "run" is defined; None (the default) means every call is asked about
77
+ # independently, which is the existing behaviour for every builtin.
78
+ #
79
+ # Exists so §18's subagent sign-off is a property of spawn_subagent
80
+ # rather than a tool name hard-coded into core/loop.py -- the loop asks
81
+ # the registry what a tool's grant scope is, the same way it asks
82
+ # whether approval is needed at all. §23's response channel reuses it.
83
+ grant_scope: Optional[str] = None
84
+ # ROADMAP_v2 §23. Which KIND of question approving this tool asks.
85
+ # "approval" (the default) is a yes/no; spawn_subagent sets
86
+ # "subagent_signoff", whose answer is the SUBSET of tools the child may
87
+ # then use without asking again.
88
+ #
89
+ # A property of the tool rather than a name hard-coded into
90
+ # core/loop.py, for exactly the reason grant_scope is: the loop asks
91
+ # the registry what shape of question a tool needs, the same way it
92
+ # asks whether it needs one at all.
93
+ request_kind: str = "approval"
94
+ # Optional extra fields for that question's payload, when the answer
95
+ # depends on something only the tool knows. Signature:
96
+ # (params, context) -> dict. spawn_subagent uses it to carry the
97
+ # candidate tool list, which is agents/manager.py's knowledge and must
98
+ # not become the loop's. Mirrors approval_notice exactly.
99
+ request_payload: Optional[Callable[[dict, object], dict]] = None
100
+ # ROADMAP_v2 §32 (A7). Optional "I would refuse this call anyway"
101
+ # pre-flight. Signature: (params, context) -> Optional[str], the
102
+ # refusal reason or None. Returning a reason does NOT deny the
103
+ # call -- the handler still runs and still refuses; it tells the
104
+ # loop not to ASK first.
105
+ #
106
+ # core/loop.py already establishes this ordering for the case it
107
+ # can see on its own: "a context-excluded tool reports the context
108
+ # denial instead of prompting and then being denied anyway". That
109
+ # reasoning is not specific to contexts, but the loop cannot apply
110
+ # it to a condition only the tool knows -- spawn_subagent's
111
+ # unknown-agent and depth-limit checks live inside run(), so both
112
+ # prompted first and errored second. An unknown name produced a
113
+ # sign-off modal with an EMPTY notice and an EMPTY candidate list:
114
+ # a question with nothing in it.
115
+ #
116
+ # A property of the tool rather than a name hard-coded into the
117
+ # loop, for exactly the reason grant_scope and request_kind are.
118
+ refusal_check: Optional[Callable[[dict, object], Optional[str]]] = None
119
+ # Optional extra text shown in the approval prompt, above the params.
120
+ # Signature: (params, context) -> str. Lets a tool explain what
121
+ # approving actually authorises when the params alone don't say --
122
+ # spawn_subagent uses it to list the tools the subagent would then be
123
+ # able to run without asking again. Keeps that knowledge in the tool
124
+ # instead of teaching the TUI about agents.
125
+ approval_notice: Optional[Callable[[dict, object], str]] = None
126
+ # ROADMAP_v2 §25 (R13). One of GRANT_ANYWHERE / GRANT_SIGNOFF_ONLY /
127
+ # GRANT_NEVER, above.
128
+ #
129
+ # DISTINCT from registry.grantable(), which is mechanical: does this
130
+ # tool decide approval from its PARAMS, in which case a name-level
131
+ # answer was never about the call that happens. This field is policy:
132
+ # given that consent by name is meaningful, is ONE TICK BEFORE ANY
133
+ # CALL EXISTS an acceptable answer, and does that depend on whether a
134
+ # human is present? Both callers check both.
135
+ #
136
+ # None means UNDECLARED, not a default. Omission has to be a
137
+ # detectable mistake rather than an inherited answer -- this field
138
+ # exists because the previous shape was a denylist in ONE consumer's
139
+ # module, so a tool registered after it was written was offered for a
140
+ # grant without anybody being asked. That is what happened to
141
+ # write_project_doc between §24 and #133.
142
+ #
143
+ # assert_grant_policy_declared() below makes omission fatal at import,
144
+ # the same trade D24 made for permissions.
145
+ grant_policy: Optional[str] = None
146
+ # ROADMAP_v2 §31 (H1). One of BUDGET_COMPUTE / BUDGET_IO /
147
+ # BUDGET_HUMAN, above.
148
+ #
149
+ # None means UNDECLARED, not a default -- grant_policy's rule one
150
+ # question over, for grant_policy's reason. Before §31 every bound in
151
+ # the tool layer was per-tool and voluntary: three network tools each
152
+ # set their own REQUEST_TIMEOUT_S, `shell` inherited the sandbox's,
153
+ # and MCP tools got a two-layer clock in mcp_client -- while the six
154
+ # math tools, the only ones that can burn a core indefinitely, had
155
+ # none at all (#57). A default here would recreate exactly that, and
156
+ # silently: an unwrapped tool looks identical to a correctly bounded
157
+ # one from every layer above.
158
+ #
159
+ # assert_budget_declared() below makes omission fatal at import.
160
+ budget: Optional[str] = None
161
+ # ROADMAP_v2 §42 (RA2). Which parameter, if any, carries the
162
+ # AGENT'S OWN STATED REASON for making this call. `shell` sets
163
+ # "rationale"; every other tool leaves it None.
164
+ #
165
+ # DISPLAY ONLY, and that is the whole contract. The named param is
166
+ # rendered on the approval prompt and skipped by the transcript
167
+ # digest, and it reaches NOTHING else: not
168
+ # `_shell_approval_check`, not `classify_command`, not
169
+ # `run_sandboxed`. A self-reported reason is unverifiable by
170
+ # construction -- a hijacked agent writes a reassuring one -- so
171
+ # letting it touch a decision would be strictly worse than not
172
+ # having it. Its value is longitudinal: a thread's worth of stated
173
+ # reasons, archived beside the calls, is a drift signal no single
174
+ # prompt can give. test_rationale.py pins the invariance by
175
+ # asserting on the arguments the classifier RECEIVES, not on the
176
+ # decision it returns -- an outcome check would pass for a tool
177
+ # that read the field and happened to agree.
178
+ #
179
+ # NO IMPORT-TIME GUARD, unlike grant_policy and budget. For those,
180
+ # None is an unanswered question and silence is the failure; here
181
+ # None is the correct answer for every tool but one, so a
182
+ # declaration requirement would be noise on twenty-two of them.
183
+ rationale_param: Optional[str] = None
184
+ # ROADMAP_v2 §46 (EP2). Which parameter, if any, is the SUBJECT of
185
+ # the approval question -- the one thing a person must read before
186
+ # answering. `shell` sets "command"; every other tool leaves it
187
+ # None and its modal is unchanged.
188
+ #
189
+ # Mechanism, not policy, in the shape rationale_param established
190
+ # directly above: the consumer asks the registry rather than testing
191
+ # for the literal name `shell`, so no tool name enters core/loop.py
192
+ # or tui/screens.py. Two consumers need it -- the TUI modal, which
193
+ # PINS it outside the scroll region, and the CLI prompt, which
194
+ # leads with it.
195
+ #
196
+ # DISPLAY ONLY, exactly like rationale_param, and for a different
197
+ # reason: this value is ALSO in `params`, which the prompt renders
198
+ # whole. Nothing may read it to decide anything, because the thing
199
+ # that decides is `classify_command`, and a second reader of the
200
+ # same field is how the two came to disagree in #157.
201
+ #
202
+ # Why it is not just `params["command"]` at the render site: a
203
+ # screen that knows a parameter name knows a tool, and the modal is
204
+ # shared by web_search, spawn_subagent, the MCP tools and /init's
205
+ # confirm. EP1 pins the SUBJECT of the question, whatever the tool
206
+ # says that is.
207
+ #
208
+ # NO IMPORT-TIME GUARD, for rationale_param's reason: None is the
209
+ # correct answer for every tool but one.
210
+ headline_param: Optional[str] = None
211
+
212
+
213
+ def assert_grant_policy_declared(tools: Iterable[str], specs=None) -> None:
214
+ """R13: every statically registered tool must declare a grant policy,
215
+ and it must be one of the three defined values.
216
+
217
+ Raises rather than warning, for D24's reason: the failure this guards
218
+ against is invisible at runtime. A tool with no declaration resolves
219
+ to GRANT_NEVER (registry.grant_policy), so forgetting the field would
220
+ silently drop a tool out of both grant paths with nothing logged --
221
+ and a MISSPELLED value would do the same while looking answered,
222
+ which is worse. Both are caught here instead.
223
+
224
+ The precedent is `fetch_url` under D24: registered, documented as
225
+ working, and denied on every call for its entire life because nobody
226
+ added a field. The equivalent mistake here is quieter still, because
227
+ a tool that is never offered for a grant simply prompts per call --
228
+ correct-looking behaviour that hides an unanswered policy question.
229
+
230
+ Dynamically-named `mcp__*` tools are exempt by design; they are named
231
+ at connection time and can never carry a static declaration.
232
+ `mcp_client/registration.py` passes one explicitly anyway, so the
233
+ fallback is documentation rather than the live path.
234
+ """
235
+ specs = specs if specs is not None else tools
236
+ undeclared, invalid = [], []
237
+ for name in tools:
238
+ if name.startswith("mcp__"):
239
+ continue
240
+ spec = specs.get(name) if hasattr(specs, "get") else None
241
+ policy = getattr(spec, "grant_policy", None)
242
+ if policy is None:
243
+ undeclared.append(name)
244
+ elif policy not in GRANT_POLICIES:
245
+ invalid.append(f"{name}={policy!r}")
246
+ if undeclared or invalid:
247
+ raise RuntimeError(
248
+ "Tools are registered without a usable grant policy, so the "
249
+ "question of whether one tick before any call exists may "
250
+ "authorise them was never answered: "
251
+ f"undeclared={sorted(undeclared)} invalid={sorted(invalid)}. "
252
+ f"Set ToolSpec.grant_policy to one of {sorted(GRANT_POLICIES)} "
253
+ "(ROADMAP_v2 §25, R13)."
254
+ )
255
+
256
+
257
+ def assert_budget_declared(tools: Iterable[str], specs=None,
258
+ injectable=None) -> None:
259
+ """H1: every statically registered tool must declare a cost class, it
260
+ must be one of the three defined values, and a BUDGET_COMPUTE tool
261
+ must not also declare an injection.
262
+
263
+ Raises rather than warning, and the argument is the one above it,
264
+ unchanged: the failure is invisible at runtime. An undeclared tool is
265
+ simply not wrapped, which from every layer above looks exactly like a
266
+ tool that is correctly bounded somewhere else -- and that is the state
267
+ #57 found, where dispatch called every handler directly and nothing
268
+ anywhere recorded whether that was intended.
269
+
270
+ The third check is not a style rule. BUDGET_COMPUTE is not merely a
271
+ policy: it routes the call through a PROCESS boundary, and nothing in
272
+ _INJECTABLE_PARAMS survives one. A live ConversationMemory or a
273
+ response_channel cannot be pickled, so a tool declaring both would
274
+ fail at its first call rather than here. The two fields are checked
275
+ together because neither is wrong on its own.
276
+
277
+ Dynamically-named `mcp__*` tools are exempt, as they are for
278
+ grant_policy and for the same reason -- they are named at connection
279
+ time. Their bound is mcp_client's own two-layer clock, and
280
+ registration.py passes BUDGET_IO explicitly to say so.
281
+ """
282
+ specs = specs if specs is not None else tools
283
+ injectable = injectable or {}
284
+ undeclared, invalid, contradictory = [], [], []
285
+ for name in tools:
286
+ if name.startswith("mcp__"):
287
+ continue
288
+ spec = specs.get(name) if hasattr(specs, "get") else None
289
+ budget = getattr(spec, "budget", None)
290
+ if budget is None:
291
+ undeclared.append(name)
292
+ elif budget not in BUDGETS:
293
+ invalid.append(f"{name}={budget!r}")
294
+ elif budget == BUDGET_COMPUTE and injectable.get(name):
295
+ contradictory.append(
296
+ f"{name} injects {sorted(injectable[name])}")
297
+ if undeclared or invalid or contradictory:
298
+ raise RuntimeError(
299
+ "Tools are registered without a usable cost class, so what a "
300
+ "call to them is allowed to cost was never answered: "
301
+ f"undeclared={sorted(undeclared)} invalid={sorted(invalid)} "
302
+ f"compute-with-injection={sorted(contradictory)}. "
303
+ f"Set ToolSpec.budget to one of {sorted(BUDGETS)} "
304
+ "(ROADMAP_v2 §31, H1)."
305
+ )
File without changes
@@ -0,0 +1,377 @@
1
+ """
2
+ tools/builtin/_math_common.py
3
+
4
+ Shared safe-parsing foundation for every math tool. Prefixed with an
5
+ underscore since this isn't itself a registered tool -- it's imported by
6
+ the six math tools that are.
7
+
8
+ SAFETY NOTE: SymPy's parser (sympify/parse_expr) tokenizes and then calls
9
+ Python's eval() on the result, so an expression string is code. Every one
10
+ of them arrives from the model, which during the grounding passes has been
11
+ reading pages it did not author, and all six math tools are
12
+ `permission=True, approval=False` -- advertised and callable with no
13
+ prompt in plain chat AND in all ten research passes.
14
+
15
+ BLANKING `__builtins__` IS NOT SUFFICIENT, AND CANNOT BE (#52). That was
16
+ this module's original premise and it was wrong in two independent ways:
17
+
18
+ 1. `_SAFE_GLOBALS` is built from `dir(sympy)`, so SymPy's OWN parser
19
+ entry points are in the namespace by construction -- `sympify`,
20
+ `parse_expr`, `var`, `init_printing`, `S`, `lambdify`. `sympify`'s
21
+ inner `parse_expr` builds its own globals via
22
+ `exec('from sympy import *', global_dict)`, and Python inserts REAL
23
+ `__builtins__` into any globals dict handed to exec/eval that lacks
24
+ it. The outer blanking never applies to the inner parse.
25
+ 2. `().__class__.__bases__[0].__subclasses__()` needs no builtins at
26
+ all -- only attribute access on a literal -- and yields hundreds of
27
+ live classes. No namespace pruning can stop that.
28
+
29
+ A THIRD mechanism killed the first fix, and is why the guard looks the
30
+ way it does. That fix filtered NAME *tokens* against an allowlist. It was
31
+ bypassed within the hour:
32
+
33
+ f"{sympify('__import__(chr(111)+chr(115)).getpid()')}" -> the pid
34
+
35
+ On Python 3.11 an f-string is a SINGLE `STRING` token -- PEP 701 only
36
+ split it into `FSTRING_*` tokens in 3.12 -- so a NAME-token filter is
37
+ structurally blind to everything inside one, while the code inside is
38
+ evaluated with `_SAFE_GLOBALS` in scope, `sympify` included. `str.format`
39
+ reaches dunder attributes the same way, through its own mini-language.
40
+
41
+ The lesson generalises past f-strings: **a token stream is not the
42
+ language**, and which constructs the tokenizer hides is a property of the
43
+ Python version rather than of this code.
44
+
45
+ So the guard is `_validate_ast`, an allowlist over AST NODE TYPES applied
46
+ to exactly the string that will be evaluated -- see the long comment
47
+ above it. `parse_expr` is `stringify_expr` followed by `eval_expr`, and
48
+ the check sits between them.
49
+
50
+ One thing NOT to rely on: today the 26 submodule objects in
51
+ `_SAFE_GLOBALS` (`external.gmpy.os`, `printing.gtk.subprocess`,
52
+ `parsing.sympy_parser.builtins`, ...) happen to be unreachable, because
53
+ `auto_symbol` leaves a name alone only when it is a Basic, a type, or
54
+ callable -- and a module is none of the three, so it is rewritten to
55
+ `Symbol('external')`. That is an undocumented SymPy implementation detail
56
+ protecting a real hole. Attribute access is refused outright and the name
57
+ allowlist excludes every module, so nothing depends on it.
58
+ """
59
+
60
+ from __future__ import annotations
61
+
62
+ import ast
63
+ import types
64
+ from typing import Any, Optional
65
+
66
+ import sympy
67
+ from sympy.parsing.sympy_parser import (
68
+ eval_expr,
69
+ standard_transformations,
70
+ stringify_expr,
71
+ implicit_multiplication_application,
72
+ convert_equals_signs,
73
+ )
74
+
75
+ # SymPy's own public namespace (sin, cos, sqrt, pi, Symbol, Matrix, oo, ...)
76
+ # with __builtins__ explicitly neutered. Built once at import time.
77
+ #
78
+ # The blanking is kept -- it is not what makes this safe (see the module
79
+ # docstring), but removing it would re-open the one payload shape the
80
+ # original test covered, for no gain.
81
+ _SAFE_GLOBALS: dict[str, Any] = {
82
+ name: getattr(sympy, name) for name in dir(sympy) if not name.startswith("_")
83
+ }
84
+ _SAFE_GLOBALS["__builtins__"] = {}
85
+
86
+
87
+ class MathParseError(ValueError):
88
+ """Raised when an expression string can't be safely parsed."""
89
+
90
+
91
+ # What a math EXPRESSION may name. Grouped by category rather than listed
92
+ # flat, so that widening it is a visible decision about a KIND of thing
93
+ # rather than a name appended to a pile.
94
+ #
95
+ # The rule for admitting a name: it computes over SymPy objects. Anything
96
+ # that parses, evaluates, compiles, renders or executes stays out --
97
+ # `sympify`, `parse_expr`, `var`, `S`, `lambdify`, `init_printing`,
98
+ # `preview`, and the 26 submodule objects, all of which are reachable
99
+ # names in `_SAFE_GLOBALS` and none of which a math expression needs.
100
+ #
101
+ # Being CLOSED is the property that matters. A future SymPy that adds a
102
+ # new evaluating callable, or that stops rewriting bare module names into
103
+ # Symbols, is refused by default rather than newly exposed. That is the
104
+ # whole reason this is an allowlist and not a denylist of known-bad names.
105
+ _ALLOWED_NAMES = frozenset("""
106
+ sin cos tan cot sec csc sinc asin acos atan atan2 acot asec acsc
107
+ sinh cosh tanh coth sech csch asinh acosh atanh acoth asech acsch
108
+ exp log ln sqrt cbrt root Abs sign floor ceiling frac Mod
109
+ factorial factorial2 subfactorial binomial gamma lowergamma uppergamma
110
+ loggamma digamma trigamma polygamma beta
111
+ erf erfc erfi erfinv erf2 zeta dirichlet_eta lerchphi polylog
112
+ LambertW Ei li Li Si Ci Shi Chi expint
113
+ besselj bessely besseli besselk hankel1 hankel2 airyai airybi
114
+ jacobi gegenbauer chebyshevt chebyshevu legendre assoc_legendre
115
+ hermite laguerre assoc_laguerre hyper meijerg elliptic_k elliptic_e
116
+ conjugate re im arg Add Mul Pow
117
+ fibonacci lucas tribonacci catalan harmonic bernoulli euler
118
+ primepi prime isprime nextprime prevprime factorint divisors totient
119
+ gcd lcm igcd ilcm mod_inverse
120
+ pi E I oo zoo nan GoldenRatio EulerGamma Catalan TribonacciConstant
121
+ true false Integer Float Rational Symbol Tuple Dict
122
+ Matrix ImmutableMatrix eye zeros ones diag
123
+ Interval FiniteSet Union Intersection Complement EmptySet
124
+ Range ImageSet ConditionSet ProductSet Naturals Integers Reals
125
+ Eq Ne Lt Le Gt Ge StrictLessThan StrictGreaterThan
126
+ And Or Not Xor Nand Nor Implies Equivalent ITE
127
+ diff integrate limit Sum Product Derivative Integral Limit Order
128
+ simplify expand factor together apart cancel collect
129
+ trigsimp radsimp powsimp logcombine expand_trig expand_log
130
+ solve solveset roots nsolve dsolve linsolve nonlinsolve
131
+ Poly degree LC LT rem quo div
132
+ transpose det trace adjugate
133
+ Function Lambda Piecewise Max Min Heaviside DiracDelta KroneckerDelta
134
+ Point Point2D Point3D Line Segment Ray Circle Ellipse Polygon Triangle
135
+ N nsimplify
136
+ """.split())
137
+
138
+
139
+ # implicit_multiplication_application lets "2x" parse as "2*x".
140
+ # convert_equals_signs lets "x**2 - 4 = 0" parse into an Eq(...) object
141
+ # instead of raising a syntax error on the bare "=".
142
+ _TRANSFORMATIONS = standard_transformations + (
143
+ implicit_multiplication_application,
144
+ convert_equals_signs,
145
+ )
146
+
147
+
148
+ # ---------------------------------------------------------------------------
149
+ # ---- The guard: an AST allowlist over the code that will be EVALUATED ------
150
+ # ---------------------------------------------------------------------------
151
+ #
152
+ # WHY NOT A TOKEN FILTER. The first attempt at #52 rejected dunder and
153
+ # non-allowlisted NAME *tokens*. It was bypassed within the hour:
154
+ #
155
+ # f"{sympify('__import__(chr(111)+chr(115)).getpid()')}" -> the pid
156
+ #
157
+ # On Python 3.11 an f-string is a SINGLE `STRING` token (PEP 701 only
158
+ # split it into `FSTRING_*` tokens in 3.12), so a NAME-token filter is
159
+ # structurally blind to everything inside one -- and the code inside is
160
+ # evaluated with `_SAFE_GLOBALS` in scope, `sympify` included.
161
+ # `"{0.__class__.__bases__}".format(pi)` reaches dunder attributes the
162
+ # same way, through `str.format`'s own mini-language.
163
+ #
164
+ # The lesson generalises past f-strings: a token stream is not the
165
+ # language. Any construct whose contents the tokenizer does not expose as
166
+ # tokens is invisible to a token filter, and "which constructs are those"
167
+ # is a property of the Python version, not of this code. That is not
168
+ # something a filter can be argued sound against.
169
+ #
170
+ # So the guard validates the ABSTRACT SYNTAX TREE of exactly the string
171
+ # that gets evaluated. `parse_expr` is `stringify_expr` (apply the
172
+ # transformations, emit code) followed by `eval_expr` (eval it), so
173
+ # splitting it open puts the check between those two steps -- after
174
+ # "2x" has become `Integer(2)*Symbol('x')`, and before anything runs.
175
+ #
176
+ # What that buys, and it is the whole point: the allowed set is CLOSED
177
+ # over node types. A construct nobody thought of -- an f-string, a walrus,
178
+ # a comprehension, a lambda, a generator, whatever a future Python adds --
179
+ # is refused because it is not on the list, not because someone
180
+ # remembered it.
181
+
182
+ _ALLOWED_NODES = (
183
+ ast.Expression,
184
+ ast.BinOp, ast.UnaryOp, ast.BoolOp, ast.Compare,
185
+ ast.Call, ast.keyword,
186
+ ast.Name, ast.Load,
187
+ ast.Constant,
188
+ ast.Tuple, ast.List,
189
+ ast.Subscript, ast.Slice,
190
+ # operators
191
+ ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow,
192
+ ast.LShift, ast.RShift, ast.BitOr, ast.BitXor, ast.BitAnd, ast.MatMult,
193
+ ast.UAdd, ast.USub, ast.Not, ast.Invert,
194
+ ast.And, ast.Or,
195
+ ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE,
196
+ )
197
+
198
+ # `ast.Attribute` is deliberately ABSENT, and it is the single most
199
+ # load-bearing omission. Attribute access IS #52's mechanism 2 --
200
+ # `().__class__.__bases__[0].__subclasses__()` needs no name lookup at
201
+ # all -- and it is also how `str.format` and every bound-method route is
202
+ # reached. No transformation emits an attribute for a legitimate
203
+ # expression, so nothing needs it: the functional forms (`diff(f, x)`
204
+ # rather than `f.diff(x)`) are what the tools take anyway.
205
+
206
+
207
+ # The only two calls `stringify_expr` gives a string argument to. Kept
208
+ # minimal on purpose: each entry is a callable that must be argued not to
209
+ # sympify what it is handed.
210
+ _STRING_ARG_CONSTRUCTORS = frozenset({"Symbol", "Float"})
211
+
212
+
213
+ def _validate_ast(code: str, local_dict: dict) -> None:
214
+ """Refuse anything outside the closed node allowlist. Raises
215
+ MathParseError; returns None when the expression is acceptable."""
216
+ try:
217
+ tree = ast.parse(code, mode="eval")
218
+ except SyntaxError as e:
219
+ raise MathParseError(f"Could not parse expression: {e}") from e
220
+
221
+ for node in ast.walk(tree):
222
+ if not isinstance(node, _ALLOWED_NODES):
223
+ raise MathParseError(
224
+ f"{type(node).__name__} is not allowed in a math "
225
+ f"expression")
226
+
227
+ # Every name is resolved against _SAFE_GLOBALS at eval time, so
228
+ # every name must be one we chose. Unknown names never reach here
229
+ # -- auto_symbol has already rewritten them to Symbol('name') --
230
+ # except the ones the CALLER declared via `symbols=`, which is
231
+ # what local_dict holds.
232
+ if isinstance(node, ast.Name):
233
+ if node.id not in _ALLOWED_NAMES and node.id not in local_dict:
234
+ raise MathParseError(
235
+ f"'{node.id}' is not a permitted function or "
236
+ f"constant. Use a mathematical function, or a plain "
237
+ f"name for a variable.")
238
+
239
+ # A STRING may appear only where the transformations put one.
240
+ #
241
+ # Seven allowlisted functions call `sympify()` on their own arguments
242
+ # -- `factor`, `cancel`, `together`, `apart`, `roots`, `degree`,
243
+ # `nsolve` -- which was found by auditing the allowlist's source
244
+ # rather than by guessing. Handing any of them a string literal
245
+ # re-enters the parser with SymPy's own default globals, real
246
+ # builtins included, and that is #52's mechanism 1 through a side
247
+ # door:
248
+ #
249
+ # factor(" __import__(chr(111)+chr(115)).getpid() ") -> the pid
250
+ #
251
+ # The leading space matters: an earlier version of this guard only
252
+ # tripped on a string that *started or ended* with a dunder, so one
253
+ # space defeated it. Prefix matching on a payload is the same
254
+ # category of mistake as `chr(111)+chr(115)` defeating a search for
255
+ # the literal "os".
256
+ #
257
+ # `stringify_expr` emits string arguments in exactly two places --
258
+ # `Symbol('x')` for every free variable and `Float('1.5')` for every
259
+ # non-integer literal -- so licensing those two and refusing every
260
+ # other string keeps the property closed rather than filtered. This
261
+ # also removes `"%s" % pi` and every other string-valued construct
262
+ # without needing a rule per construct.
263
+ licensed = set()
264
+ for node in ast.walk(tree):
265
+ if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
266
+ and node.func.id in _STRING_ARG_CONSTRUCTORS):
267
+ for arg in node.args:
268
+ if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
269
+ licensed.add(id(arg))
270
+
271
+ for node in ast.walk(tree):
272
+ if (isinstance(node, ast.Constant) and isinstance(node.value, str)
273
+ and id(node) not in licensed):
274
+ raise MathParseError(
275
+ "string literals are not allowed in a math expression")
276
+
277
+
278
+ def _reject_escaped_value(value: Any, _depth: int = 0) -> None:
279
+ """Backstop: refuse a RESULT that is not a mathematical value.
280
+
281
+ Every guard above this one is an allowlist over the *input*. This is
282
+ the one check on the *output*, and it is deliberately a DENY of the
283
+ escape signature rather than an allowlist: its job is to catch an
284
+ escape nobody anticipated, at the moment it succeeds, and an
285
+ allowlist of return types can only catch escapes whose shape was
286
+ already imagined.
287
+
288
+ The signature is simple. A math expression evaluates to a VALUE -- a
289
+ number, an expression tree, a matrix, a list of solutions. It never
290
+ evaluates to a module, to a plain function, or to a class from
291
+ outside SymPy. Those are what every escape in #52 produced or reached
292
+ for: `<module 'os'>`, `__subclasses__`, a bound method's `__globals__`.
293
+
294
+ SymPy's own classes ARE allowed: `sin` is a `FunctionClass`, and
295
+ `Function('f')` legitimately evaluates to a class. Both are subclasses
296
+ of `Basic`, which is what separates them from `type` or `os`.
297
+ """
298
+ if _depth > 6:
299
+ return # deep enough; containers are shallow here
300
+
301
+ if isinstance(value, types.ModuleType):
302
+ raise MathParseError(
303
+ "expression evaluated to a module, which is never a "
304
+ "mathematical value")
305
+
306
+ if isinstance(value, type):
307
+ if not issubclass(value, (sympy.Basic, sympy.matrices.MatrixBase)):
308
+ raise MathParseError(
309
+ f"expression evaluated to the class {value.__name__!r}, "
310
+ f"which is not a SymPy type")
311
+ return
312
+
313
+ # No types.ModuleType here: the branch above already raised for one,
314
+ # so a second listing is unreachable and reads as if this arm were the
315
+ # module guard.
316
+ if isinstance(value, (types.FunctionType, types.BuiltinFunctionType,
317
+ types.MethodType, types.BuiltinMethodType,
318
+ types.MethodWrapperType)):
319
+ raise MathParseError(
320
+ "expression evaluated to a function or method, which is "
321
+ "never a mathematical value")
322
+
323
+ if isinstance(value, dict):
324
+ for k, v in value.items():
325
+ _reject_escaped_value(k, _depth + 1)
326
+ _reject_escaped_value(v, _depth + 1)
327
+ elif isinstance(value, (list, tuple, set, frozenset)):
328
+ for item in value:
329
+ _reject_escaped_value(item, _depth + 1)
330
+
331
+
332
+ def safe_parse(expr_str: str, symbols: Optional[list[str]] = None):
333
+ """
334
+ Parses a math expression string into a SymPy object without exposing
335
+ Python builtins. `symbols` is optional -- SymPy's standard
336
+ transformations already auto-convert undefined names into Symbols,
337
+ but passing them explicitly keeps variable identity consistent across
338
+ multiple calls (e.g. the same 'x' in an expression and a separate
339
+ bound for a definite integral).
340
+ """
341
+ local_dict = {name: sympy.Symbol(name) for name in (symbols or [])}
342
+ try:
343
+ # parse_expr() split open, so the guard can sit between the two
344
+ # halves -- see _validate_ast. `evaluate=False` is not used
345
+ # anywhere here, which is the only other thing parse_expr does.
346
+ code = stringify_expr(expr_str, local_dict, _SAFE_GLOBALS,
347
+ _TRANSFORMATIONS)
348
+ _validate_ast(code, local_dict)
349
+ result = eval_expr(code, local_dict, _SAFE_GLOBALS)
350
+ _reject_escaped_value(result)
351
+ return result
352
+ except MathParseError:
353
+ # A refusal from _validate_ast already says which name or node was
354
+ # refused and what to do instead. Re-wrapping it in "Could not
355
+ # parse expression '<the whole payload>'" buries that under the
356
+ # string the model just sent, and echoes the payload back into
357
+ # the transcript for no reason.
358
+ raise
359
+ except Exception as e:
360
+ raise MathParseError(f"Could not parse expression '{expr_str}': {e}") from e
361
+
362
+
363
+ def serialize(value: Any) -> Any:
364
+ """
365
+ Generic SymPy-object -> JSON-safe converter, used across all six math
366
+ tools. Matrices become nested lists of strings, dicts/lists recurse,
367
+ everything else falls back to str().
368
+ """
369
+ if isinstance(value, sympy.Matrix):
370
+ return [[str(cell) for cell in row] for row in value.tolist()]
371
+ if isinstance(value, dict):
372
+ return {str(k): serialize(v) for k, v in value.items()}
373
+ if isinstance(value, (list, tuple)):
374
+ return [serialize(v) for v in value]
375
+ if isinstance(value, bool):
376
+ return value
377
+ return str(value)