@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,377 +1,667 @@
1
- """
2
- tools/builtin/_math_common.py
3
-
4
- Shared safe-parsing foundation for every math tool. Prefixed with an
5
- underscore since this isn't itself a registered tool -- it's imported by
6
- the six math tools that are.
7
-
8
- SAFETY NOTE: SymPy's parser (sympify/parse_expr) tokenizes and then calls
9
- Python's eval() on the result, so an expression string is code. Every one
10
- of them arrives from the model, which during the grounding passes has been
11
- reading pages it did not author, and all six math tools are
12
- `permission=True, approval=False` -- advertised and callable with no
13
- prompt in plain chat AND in all ten research passes.
14
-
15
- BLANKING `__builtins__` IS NOT SUFFICIENT, AND CANNOT BE (#52). That was
16
- this module's original premise and it was wrong in two independent ways:
17
-
18
- 1. `_SAFE_GLOBALS` is built from `dir(sympy)`, so SymPy's OWN parser
19
- entry points are in the namespace by construction -- `sympify`,
20
- `parse_expr`, `var`, `init_printing`, `S`, `lambdify`. `sympify`'s
21
- inner `parse_expr` builds its own globals via
22
- `exec('from sympy import *', global_dict)`, and Python inserts REAL
23
- `__builtins__` into any globals dict handed to exec/eval that lacks
24
- it. The outer blanking never applies to the inner parse.
25
- 2. `().__class__.__bases__[0].__subclasses__()` needs no builtins at
26
- all -- only attribute access on a literal -- and yields hundreds of
27
- live classes. No namespace pruning can stop that.
28
-
29
- A THIRD mechanism killed the first fix, and is why the guard looks the
30
- way it does. That fix filtered NAME *tokens* against an allowlist. It was
31
- bypassed within the hour:
32
-
33
- f"{sympify('__import__(chr(111)+chr(115)).getpid()')}" -> the pid
34
-
35
- On Python 3.11 an f-string is a SINGLE `STRING` token -- PEP 701 only
36
- split it into `FSTRING_*` tokens in 3.12 -- so a NAME-token filter is
37
- structurally blind to everything inside one, while the code inside is
38
- evaluated with `_SAFE_GLOBALS` in scope, `sympify` included. `str.format`
39
- reaches dunder attributes the same way, through its own mini-language.
40
-
41
- The lesson generalises past f-strings: **a token stream is not the
42
- language**, and which constructs the tokenizer hides is a property of the
43
- Python version rather than of this code.
44
-
45
- So the guard is `_validate_ast`, an allowlist over AST NODE TYPES applied
46
- to exactly the string that will be evaluated -- see the long comment
47
- above it. `parse_expr` is `stringify_expr` followed by `eval_expr`, and
48
- the check sits between them.
49
-
50
- One thing NOT to rely on: today the 26 submodule objects in
51
- `_SAFE_GLOBALS` (`external.gmpy.os`, `printing.gtk.subprocess`,
52
- `parsing.sympy_parser.builtins`, ...) happen to be unreachable, because
53
- `auto_symbol` leaves a name alone only when it is a Basic, a type, or
54
- callable -- and a module is none of the three, so it is rewritten to
55
- `Symbol('external')`. That is an undocumented SymPy implementation detail
56
- protecting a real hole. Attribute access is refused outright and the name
57
- allowlist excludes every module, so nothing depends on it.
58
- """
59
-
60
- from __future__ import annotations
61
-
62
- import ast
63
- import types
64
- from typing import Any, Optional
65
-
66
- import sympy
67
- from sympy.parsing.sympy_parser import (
68
- eval_expr,
69
- standard_transformations,
70
- stringify_expr,
71
- implicit_multiplication_application,
72
- convert_equals_signs,
73
- )
74
-
75
- # SymPy's own public namespace (sin, cos, sqrt, pi, Symbol, Matrix, oo, ...)
76
- # with __builtins__ explicitly neutered. Built once at import time.
77
- #
78
- # The blanking is kept -- it is not what makes this safe (see the module
79
- # docstring), but removing it would re-open the one payload shape the
80
- # original test covered, for no gain.
81
- _SAFE_GLOBALS: dict[str, Any] = {
82
- name: getattr(sympy, name) for name in dir(sympy) if not name.startswith("_")
83
- }
84
- _SAFE_GLOBALS["__builtins__"] = {}
85
-
86
-
87
- class MathParseError(ValueError):
88
- """Raised when an expression string can't be safely parsed."""
89
-
90
-
91
- # What a math EXPRESSION may name. Grouped by category rather than listed
92
- # flat, so that widening it is a visible decision about a KIND of thing
93
- # rather than a name appended to a pile.
94
- #
95
- # The rule for admitting a name: it computes over SymPy objects. Anything
96
- # that parses, evaluates, compiles, renders or executes stays out --
97
- # `sympify`, `parse_expr`, `var`, `S`, `lambdify`, `init_printing`,
98
- # `preview`, and the 26 submodule objects, all of which are reachable
99
- # names in `_SAFE_GLOBALS` and none of which a math expression needs.
100
- #
101
- # Being CLOSED is the property that matters. A future SymPy that adds a
102
- # new evaluating callable, or that stops rewriting bare module names into
103
- # Symbols, is refused by default rather than newly exposed. That is the
104
- # whole reason this is an allowlist and not a denylist of known-bad names.
105
- _ALLOWED_NAMES = frozenset("""
106
- sin cos tan cot sec csc sinc asin acos atan atan2 acot asec acsc
107
- sinh cosh tanh coth sech csch asinh acosh atanh acoth asech acsch
108
- exp log ln sqrt cbrt root Abs sign floor ceiling frac Mod
109
- factorial factorial2 subfactorial binomial gamma lowergamma uppergamma
110
- loggamma digamma trigamma polygamma beta
111
- erf erfc erfi erfinv erf2 zeta dirichlet_eta lerchphi polylog
112
- LambertW Ei li Li Si Ci Shi Chi expint
113
- besselj bessely besseli besselk hankel1 hankel2 airyai airybi
114
- jacobi gegenbauer chebyshevt chebyshevu legendre assoc_legendre
115
- hermite laguerre assoc_laguerre hyper meijerg elliptic_k elliptic_e
116
- conjugate re im arg Add Mul Pow
117
- fibonacci lucas tribonacci catalan harmonic bernoulli euler
118
- primepi prime isprime nextprime prevprime factorint divisors totient
119
- gcd lcm igcd ilcm mod_inverse
120
- pi E I oo zoo nan GoldenRatio EulerGamma Catalan TribonacciConstant
121
- true false Integer Float Rational Symbol Tuple Dict
122
- Matrix ImmutableMatrix eye zeros ones diag
123
- Interval FiniteSet Union Intersection Complement EmptySet
124
- Range ImageSet ConditionSet ProductSet Naturals Integers Reals
125
- Eq Ne Lt Le Gt Ge StrictLessThan StrictGreaterThan
126
- And Or Not Xor Nand Nor Implies Equivalent ITE
127
- diff integrate limit Sum Product Derivative Integral Limit Order
128
- simplify expand factor together apart cancel collect
129
- trigsimp radsimp powsimp logcombine expand_trig expand_log
130
- solve solveset roots nsolve dsolve linsolve nonlinsolve
131
- Poly degree LC LT rem quo div
132
- transpose det trace adjugate
133
- Function Lambda Piecewise Max Min Heaviside DiracDelta KroneckerDelta
134
- Point Point2D Point3D Line Segment Ray Circle Ellipse Polygon Triangle
135
- N nsimplify
136
- """.split())
137
-
138
-
139
- # implicit_multiplication_application lets "2x" parse as "2*x".
140
- # convert_equals_signs lets "x**2 - 4 = 0" parse into an Eq(...) object
141
- # instead of raising a syntax error on the bare "=".
142
- _TRANSFORMATIONS = standard_transformations + (
143
- implicit_multiplication_application,
144
- convert_equals_signs,
145
- )
146
-
147
-
148
- # ---------------------------------------------------------------------------
149
- # ---- The guard: an AST allowlist over the code that will be EVALUATED ------
150
- # ---------------------------------------------------------------------------
151
- #
152
- # WHY NOT A TOKEN FILTER. The first attempt at #52 rejected dunder and
153
- # non-allowlisted NAME *tokens*. It was bypassed within the hour:
154
- #
155
- # f"{sympify('__import__(chr(111)+chr(115)).getpid()')}" -> the pid
156
- #
157
- # On Python 3.11 an f-string is a SINGLE `STRING` token (PEP 701 only
158
- # split it into `FSTRING_*` tokens in 3.12), so a NAME-token filter is
159
- # structurally blind to everything inside one -- and the code inside is
160
- # evaluated with `_SAFE_GLOBALS` in scope, `sympify` included.
161
- # `"{0.__class__.__bases__}".format(pi)` reaches dunder attributes the
162
- # same way, through `str.format`'s own mini-language.
163
- #
164
- # The lesson generalises past f-strings: a token stream is not the
165
- # language. Any construct whose contents the tokenizer does not expose as
166
- # tokens is invisible to a token filter, and "which constructs are those"
167
- # is a property of the Python version, not of this code. That is not
168
- # something a filter can be argued sound against.
169
- #
170
- # So the guard validates the ABSTRACT SYNTAX TREE of exactly the string
171
- # that gets evaluated. `parse_expr` is `stringify_expr` (apply the
172
- # transformations, emit code) followed by `eval_expr` (eval it), so
173
- # splitting it open puts the check between those two steps -- after
174
- # "2x" has become `Integer(2)*Symbol('x')`, and before anything runs.
175
- #
176
- # What that buys, and it is the whole point: the allowed set is CLOSED
177
- # over node types. A construct nobody thought of -- an f-string, a walrus,
178
- # a comprehension, a lambda, a generator, whatever a future Python adds --
179
- # is refused because it is not on the list, not because someone
180
- # remembered it.
181
-
182
- _ALLOWED_NODES = (
183
- ast.Expression,
184
- ast.BinOp, ast.UnaryOp, ast.BoolOp, ast.Compare,
185
- ast.Call, ast.keyword,
186
- ast.Name, ast.Load,
187
- ast.Constant,
188
- ast.Tuple, ast.List,
189
- ast.Subscript, ast.Slice,
190
- # operators
191
- ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow,
192
- ast.LShift, ast.RShift, ast.BitOr, ast.BitXor, ast.BitAnd, ast.MatMult,
193
- ast.UAdd, ast.USub, ast.Not, ast.Invert,
194
- ast.And, ast.Or,
195
- ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE,
196
- )
197
-
198
- # `ast.Attribute` is deliberately ABSENT, and it is the single most
199
- # load-bearing omission. Attribute access IS #52's mechanism 2 --
200
- # `().__class__.__bases__[0].__subclasses__()` needs no name lookup at
201
- # all -- and it is also how `str.format` and every bound-method route is
202
- # reached. No transformation emits an attribute for a legitimate
203
- # expression, so nothing needs it: the functional forms (`diff(f, x)`
204
- # rather than `f.diff(x)`) are what the tools take anyway.
205
-
206
-
207
- # The only two calls `stringify_expr` gives a string argument to. Kept
208
- # minimal on purpose: each entry is a callable that must be argued not to
209
- # sympify what it is handed.
210
- _STRING_ARG_CONSTRUCTORS = frozenset({"Symbol", "Float"})
211
-
212
-
213
- def _validate_ast(code: str, local_dict: dict) -> None:
214
- """Refuse anything outside the closed node allowlist. Raises
215
- MathParseError; returns None when the expression is acceptable."""
216
- try:
217
- tree = ast.parse(code, mode="eval")
218
- except SyntaxError as e:
219
- raise MathParseError(f"Could not parse expression: {e}") from e
220
-
221
- for node in ast.walk(tree):
222
- if not isinstance(node, _ALLOWED_NODES):
223
- raise MathParseError(
224
- f"{type(node).__name__} is not allowed in a math "
225
- f"expression")
226
-
227
- # Every name is resolved against _SAFE_GLOBALS at eval time, so
228
- # every name must be one we chose. Unknown names never reach here
229
- # -- auto_symbol has already rewritten them to Symbol('name') --
230
- # except the ones the CALLER declared via `symbols=`, which is
231
- # what local_dict holds.
232
- if isinstance(node, ast.Name):
233
- if node.id not in _ALLOWED_NAMES and node.id not in local_dict:
234
- raise MathParseError(
235
- f"'{node.id}' is not a permitted function or "
236
- f"constant. Use a mathematical function, or a plain "
237
- f"name for a variable.")
238
-
239
- # A STRING may appear only where the transformations put one.
240
- #
241
- # Seven allowlisted functions call `sympify()` on their own arguments
242
- # -- `factor`, `cancel`, `together`, `apart`, `roots`, `degree`,
243
- # `nsolve` -- which was found by auditing the allowlist's source
244
- # rather than by guessing. Handing any of them a string literal
245
- # re-enters the parser with SymPy's own default globals, real
246
- # builtins included, and that is #52's mechanism 1 through a side
247
- # door:
248
- #
249
- # factor(" __import__(chr(111)+chr(115)).getpid() ") -> the pid
250
- #
251
- # The leading space matters: an earlier version of this guard only
252
- # tripped on a string that *started or ended* with a dunder, so one
253
- # space defeated it. Prefix matching on a payload is the same
254
- # category of mistake as `chr(111)+chr(115)` defeating a search for
255
- # the literal "os".
256
- #
257
- # `stringify_expr` emits string arguments in exactly two places --
258
- # `Symbol('x')` for every free variable and `Float('1.5')` for every
259
- # non-integer literal -- so licensing those two and refusing every
260
- # other string keeps the property closed rather than filtered. This
261
- # also removes `"%s" % pi` and every other string-valued construct
262
- # without needing a rule per construct.
263
- licensed = set()
264
- for node in ast.walk(tree):
265
- if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
266
- and node.func.id in _STRING_ARG_CONSTRUCTORS):
267
- for arg in node.args:
268
- if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
269
- licensed.add(id(arg))
270
-
271
- for node in ast.walk(tree):
272
- if (isinstance(node, ast.Constant) and isinstance(node.value, str)
273
- and id(node) not in licensed):
274
- raise MathParseError(
275
- "string literals are not allowed in a math expression")
276
-
277
-
278
- def _reject_escaped_value(value: Any, _depth: int = 0) -> None:
279
- """Backstop: refuse a RESULT that is not a mathematical value.
280
-
281
- Every guard above this one is an allowlist over the *input*. This is
282
- the one check on the *output*, and it is deliberately a DENY of the
283
- escape signature rather than an allowlist: its job is to catch an
284
- escape nobody anticipated, at the moment it succeeds, and an
285
- allowlist of return types can only catch escapes whose shape was
286
- already imagined.
287
-
288
- The signature is simple. A math expression evaluates to a VALUE -- a
289
- number, an expression tree, a matrix, a list of solutions. It never
290
- evaluates to a module, to a plain function, or to a class from
291
- outside SymPy. Those are what every escape in #52 produced or reached
292
- for: `<module 'os'>`, `__subclasses__`, a bound method's `__globals__`.
293
-
294
- SymPy's own classes ARE allowed: `sin` is a `FunctionClass`, and
295
- `Function('f')` legitimately evaluates to a class. Both are subclasses
296
- of `Basic`, which is what separates them from `type` or `os`.
297
- """
298
- if _depth > 6:
299
- return # deep enough; containers are shallow here
300
-
301
- if isinstance(value, types.ModuleType):
302
- raise MathParseError(
303
- "expression evaluated to a module, which is never a "
304
- "mathematical value")
305
-
306
- if isinstance(value, type):
307
- if not issubclass(value, (sympy.Basic, sympy.matrices.MatrixBase)):
308
- raise MathParseError(
309
- f"expression evaluated to the class {value.__name__!r}, "
310
- f"which is not a SymPy type")
311
- return
312
-
313
- # No types.ModuleType here: the branch above already raised for one,
314
- # so a second listing is unreachable and reads as if this arm were the
315
- # module guard.
316
- if isinstance(value, (types.FunctionType, types.BuiltinFunctionType,
317
- types.MethodType, types.BuiltinMethodType,
318
- types.MethodWrapperType)):
319
- raise MathParseError(
320
- "expression evaluated to a function or method, which is "
321
- "never a mathematical value")
322
-
323
- if isinstance(value, dict):
324
- for k, v in value.items():
325
- _reject_escaped_value(k, _depth + 1)
326
- _reject_escaped_value(v, _depth + 1)
327
- elif isinstance(value, (list, tuple, set, frozenset)):
328
- for item in value:
329
- _reject_escaped_value(item, _depth + 1)
330
-
331
-
332
- def safe_parse(expr_str: str, symbols: Optional[list[str]] = None):
333
- """
334
- Parses a math expression string into a SymPy object without exposing
335
- Python builtins. `symbols` is optional -- SymPy's standard
336
- transformations already auto-convert undefined names into Symbols,
337
- but passing them explicitly keeps variable identity consistent across
338
- multiple calls (e.g. the same 'x' in an expression and a separate
339
- bound for a definite integral).
340
- """
341
- local_dict = {name: sympy.Symbol(name) for name in (symbols or [])}
342
- try:
343
- # parse_expr() split open, so the guard can sit between the two
344
- # halves -- see _validate_ast. `evaluate=False` is not used
345
- # anywhere here, which is the only other thing parse_expr does.
346
- code = stringify_expr(expr_str, local_dict, _SAFE_GLOBALS,
347
- _TRANSFORMATIONS)
348
- _validate_ast(code, local_dict)
349
- result = eval_expr(code, local_dict, _SAFE_GLOBALS)
350
- _reject_escaped_value(result)
351
- return result
352
- except MathParseError:
353
- # A refusal from _validate_ast already says which name or node was
354
- # refused and what to do instead. Re-wrapping it in "Could not
355
- # parse expression '<the whole payload>'" buries that under the
356
- # string the model just sent, and echoes the payload back into
357
- # the transcript for no reason.
358
- raise
359
- except Exception as e:
360
- raise MathParseError(f"Could not parse expression '{expr_str}': {e}") from e
361
-
362
-
363
- def serialize(value: Any) -> Any:
364
- """
365
- Generic SymPy-object -> JSON-safe converter, used across all six math
366
- tools. Matrices become nested lists of strings, dicts/lists recurse,
367
- everything else falls back to str().
368
- """
369
- if isinstance(value, sympy.Matrix):
370
- return [[str(cell) for cell in row] for row in value.tolist()]
371
- if isinstance(value, dict):
372
- return {str(k): serialize(v) for k, v in value.items()}
373
- if isinstance(value, (list, tuple)):
374
- return [serialize(v) for v in value]
375
- if isinstance(value, bool):
376
- return value
377
- return str(value)
1
+ """
2
+ tools/builtin/_math_common.py
3
+
4
+ Shared safe-parsing foundation for every math tool. Prefixed with an
5
+ underscore since this isn't itself a registered tool -- it's imported by
6
+ the six math tools that are.
7
+
8
+ SAFETY NOTE: SymPy's parser (sympify/parse_expr) tokenizes and then calls
9
+ Python's eval() on the result, so an expression string is code. Every one
10
+ of them arrives from the model, which during the grounding passes has been
11
+ reading pages it did not author, and all six math tools are
12
+ `permission=True, approval=False` -- advertised and callable with no
13
+ prompt in plain chat AND in all ten research passes.
14
+
15
+ BLANKING `__builtins__` IS NOT SUFFICIENT, AND CANNOT BE (#52). That was
16
+ this module's original premise and it was wrong in two independent ways:
17
+
18
+ 1. `_SAFE_GLOBALS` is built from `dir(sympy)`, so SymPy's OWN parser
19
+ entry points are in the namespace by construction -- `sympify`,
20
+ `parse_expr`, `var`, `init_printing`, `S`, `lambdify`. `sympify`'s
21
+ inner `parse_expr` builds its own globals via
22
+ `exec('from sympy import *', global_dict)`, and Python inserts REAL
23
+ `__builtins__` into any globals dict handed to exec/eval that lacks
24
+ it. The outer blanking never applies to the inner parse.
25
+ 2. `().__class__.__bases__[0].__subclasses__()` needs no builtins at
26
+ all -- only attribute access on a literal -- and yields hundreds of
27
+ live classes. No namespace pruning can stop that.
28
+
29
+ A THIRD mechanism killed the first fix, and is why the guard looks the
30
+ way it does. That fix filtered NAME *tokens* against an allowlist. It was
31
+ bypassed within the hour:
32
+
33
+ f"{sympify('__import__(chr(111)+chr(115)).getpid()')}" -> the pid
34
+
35
+ On Python 3.11 an f-string is a SINGLE `STRING` token -- PEP 701 only
36
+ split it into `FSTRING_*` tokens in 3.12 -- so a NAME-token filter is
37
+ structurally blind to everything inside one, while the code inside is
38
+ evaluated with `_SAFE_GLOBALS` in scope, `sympify` included. `str.format`
39
+ reaches dunder attributes the same way, through its own mini-language.
40
+
41
+ The lesson generalises past f-strings: **a token stream is not the
42
+ language**, and which constructs the tokenizer hides is a property of the
43
+ Python version rather than of this code.
44
+
45
+ So the guard is `_validate_ast`, an allowlist over AST NODE TYPES applied
46
+ to exactly the string that will be evaluated -- see the long comment
47
+ above it. `parse_expr` is `stringify_expr` followed by `eval_expr`, and
48
+ the check sits between them.
49
+
50
+ One thing NOT to rely on: today the 26 submodule objects in
51
+ `_SAFE_GLOBALS` (`external.gmpy.os`, `printing.gtk.subprocess`,
52
+ `parsing.sympy_parser.builtins`, ...) happen to be unreachable, because
53
+ `auto_symbol` leaves a name alone only when it is a Basic, a type, or
54
+ callable -- and a module is none of the three, so it is rewritten to
55
+ `Symbol('external')`. That is an undocumented SymPy implementation detail
56
+ protecting a real hole. Attribute access is refused outright and the name
57
+ allowlist excludes every module, so nothing depends on it.
58
+ """
59
+
60
+ from __future__ import annotations
61
+
62
+ import ast
63
+ import json
64
+ import re
65
+ import types
66
+ from typing import Any, Optional
67
+
68
+ import sympy
69
+ from pydantic import ValidationError
70
+ from sympy.parsing.sympy_parser import (
71
+ convert_equals_signs,
72
+ eval_expr,
73
+ implicit_multiplication_application,
74
+ standard_transformations,
75
+ stringify_expr,
76
+ )
77
+
78
+ import config
79
+
80
+ # SymPy's own public namespace (sin, cos, sqrt, pi, Symbol, Matrix, oo, ...)
81
+ # with __builtins__ explicitly neutered. Built once at import time.
82
+ #
83
+ # The blanking is kept -- it is not what makes this safe (see the module
84
+ # docstring), but removing it would re-open the one payload shape the
85
+ # original test covered, for no gain.
86
+ _SAFE_GLOBALS: dict[str, Any] = {
87
+ name: getattr(sympy, name) for name in dir(sympy) if not name.startswith("_")
88
+ }
89
+ _SAFE_GLOBALS["__builtins__"] = {}
90
+
91
+
92
+ class MathParseError(ValueError):
93
+ """Raised when an expression string can't be safely parsed.
94
+
95
+ ALSO THE BASE OF THE TWO BELOW, neither of which is a parse failure.
96
+ They are in this family for one mechanical reason: every tool already
97
+ has exactly one `except MathParseError as e: return {"error": str(e)}`
98
+ arm, so a subclass is caught by a handler that is already there and
99
+ already tested. Two new `except` arms across six tools would be twelve
100
+ places to add and twelve chances to miss one, which is the shape #96
101
+ and EP6 are both about (batch 143).
102
+ """
103
+
104
+
105
+ class MathResultTooLarge(MathParseError):
106
+ """A result that was computed CORRECTLY and cannot be returned.
107
+
108
+ Two causes, one message family, because the model's remedy is the same
109
+ for both: CPython caps `int -> str` at 4,300 digits, and a math result
110
+ is bounded at `config.MAX_READ_CHARS`. See `serialize`.
111
+ """
112
+
113
+
114
+ class MathParamsError(MathParseError):
115
+ """The tool's own params did not validate. See `parse_params`."""
116
+
117
+
118
+ # What a math EXPRESSION may name. Grouped by category rather than listed
119
+ # flat, so that widening it is a visible decision about a KIND of thing
120
+ # rather than a name appended to a pile.
121
+ #
122
+ # The rule for admitting a name: it computes over SymPy objects. Anything
123
+ # that parses, evaluates, compiles, renders or executes stays out --
124
+ # `sympify`, `parse_expr`, `var`, `S`, `lambdify`, `init_printing`,
125
+ # `preview`, and the 26 submodule objects, all of which are reachable
126
+ # names in `_SAFE_GLOBALS` and none of which a math expression needs.
127
+ #
128
+ # Being CLOSED is the property that matters. A future SymPy that adds a
129
+ # new evaluating callable, or that stops rewriting bare module names into
130
+ # Symbols, is refused by default rather than newly exposed. That is the
131
+ # whole reason this is an allowlist and not a denylist of known-bad names.
132
+ _ALLOWED_NAMES = frozenset("""
133
+ sin cos tan cot sec csc sinc asin acos atan atan2 acot asec acsc
134
+ sinh cosh tanh coth sech csch asinh acosh atanh acoth asech acsch
135
+ exp log ln sqrt cbrt root Abs sign floor ceiling frac Mod
136
+ factorial factorial2 subfactorial binomial gamma lowergamma uppergamma
137
+ loggamma digamma trigamma polygamma beta
138
+ erf erfc erfi erfinv erf2 zeta dirichlet_eta lerchphi polylog
139
+ LambertW Ei li Li Si Ci Shi Chi expint
140
+ besselj bessely besseli besselk hankel1 hankel2 airyai airybi
141
+ jacobi gegenbauer chebyshevt chebyshevu legendre assoc_legendre
142
+ hermite laguerre assoc_laguerre hyper meijerg elliptic_k elliptic_e
143
+ conjugate re im arg Add Mul Pow
144
+ fibonacci lucas tribonacci catalan harmonic bernoulli euler
145
+ primepi prime isprime nextprime prevprime factorint divisors totient
146
+ gcd lcm igcd ilcm mod_inverse
147
+ pi E I oo zoo nan GoldenRatio EulerGamma Catalan TribonacciConstant
148
+ true false Integer Float Rational Symbol Tuple Dict
149
+ Matrix ImmutableMatrix eye zeros ones diag
150
+ Interval FiniteSet Union Intersection Complement EmptySet
151
+ Range ImageSet ConditionSet ProductSet Naturals Integers Reals
152
+ Eq Ne Lt Le Gt Ge StrictLessThan StrictGreaterThan
153
+ And Or Not Xor Nand Nor Implies Equivalent ITE
154
+ diff integrate limit Sum Product Derivative Integral Limit Order
155
+ simplify expand factor together apart cancel collect
156
+ trigsimp radsimp powsimp logcombine expand_trig expand_log
157
+ solve solveset roots nsolve dsolve linsolve nonlinsolve
158
+ Poly degree LC LT rem quo div
159
+ transpose det trace adjugate
160
+ Function Lambda Piecewise Max Min Heaviside DiracDelta KroneckerDelta
161
+ Point Point2D Point3D Line Segment Ray Circle Ellipse Polygon Triangle
162
+ N nsimplify
163
+ """.split())
164
+
165
+
166
+ # implicit_multiplication_application lets "2x" parse as "2*x".
167
+ # convert_equals_signs lets "x**2 - 4 = 0" parse into an Eq(...) object
168
+ # instead of raising a syntax error on the bare "=".
169
+ _TRANSFORMATIONS = standard_transformations + (
170
+ implicit_multiplication_application,
171
+ convert_equals_signs,
172
+ )
173
+
174
+
175
+ # ---------------------------------------------------------------------------
176
+ # ---- The guard: an AST allowlist over the code that will be EVALUATED ------
177
+ # ---------------------------------------------------------------------------
178
+ #
179
+ # WHY NOT A TOKEN FILTER. The first attempt at #52 rejected dunder and
180
+ # non-allowlisted NAME *tokens*. It was bypassed within the hour:
181
+ #
182
+ # f"{sympify('__import__(chr(111)+chr(115)).getpid()')}" -> the pid
183
+ #
184
+ # On Python 3.11 an f-string is a SINGLE `STRING` token (PEP 701 only
185
+ # split it into `FSTRING_*` tokens in 3.12), so a NAME-token filter is
186
+ # structurally blind to everything inside one -- and the code inside is
187
+ # evaluated with `_SAFE_GLOBALS` in scope, `sympify` included.
188
+ # `"{0.__class__.__bases__}".format(pi)` reaches dunder attributes the
189
+ # same way, through `str.format`'s own mini-language.
190
+ #
191
+ # The lesson generalises past f-strings: a token stream is not the
192
+ # language. Any construct whose contents the tokenizer does not expose as
193
+ # tokens is invisible to a token filter, and "which constructs are those"
194
+ # is a property of the Python version, not of this code. That is not
195
+ # something a filter can be argued sound against.
196
+ #
197
+ # So the guard validates the ABSTRACT SYNTAX TREE of exactly the string
198
+ # that gets evaluated. `parse_expr` is `stringify_expr` (apply the
199
+ # transformations, emit code) followed by `eval_expr` (eval it), so
200
+ # splitting it open puts the check between those two steps -- after
201
+ # "2x" has become `Integer(2)*Symbol('x')`, and before anything runs.
202
+ #
203
+ # What that buys, and it is the whole point: the allowed set is CLOSED
204
+ # over node types. A construct nobody thought of -- an f-string, a walrus,
205
+ # a comprehension, a lambda, a generator, whatever a future Python adds --
206
+ # is refused because it is not on the list, not because someone
207
+ # remembered it.
208
+
209
+ _ALLOWED_NODES = (
210
+ ast.Expression,
211
+ ast.BinOp, ast.UnaryOp, ast.BoolOp, ast.Compare,
212
+ ast.Call, ast.keyword,
213
+ ast.Name, ast.Load,
214
+ ast.Constant,
215
+ ast.Tuple, ast.List,
216
+ ast.Subscript, ast.Slice,
217
+ # operators
218
+ ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow,
219
+ ast.LShift, ast.RShift, ast.BitOr, ast.BitXor, ast.BitAnd, ast.MatMult,
220
+ ast.UAdd, ast.USub, ast.Not, ast.Invert,
221
+ ast.And, ast.Or,
222
+ ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE,
223
+ )
224
+
225
+ # `ast.Attribute` is deliberately ABSENT, and it is the single most
226
+ # load-bearing omission. Attribute access IS #52's mechanism 2 --
227
+ # `().__class__.__bases__[0].__subclasses__()` needs no name lookup at
228
+ # all -- and it is also how `str.format` and every bound-method route is
229
+ # reached. No transformation emits an attribute for a legitimate
230
+ # expression, so nothing needs it: the functional forms (`diff(f, x)`
231
+ # rather than `f.diff(x)`) are what the tools take anyway.
232
+
233
+
234
+ # The only two calls `stringify_expr` gives a string argument to. Kept
235
+ # minimal on purpose: each entry is a callable that must be argued not to
236
+ # sympify what it is handed.
237
+ _STRING_ARG_CONSTRUCTORS = frozenset({"Symbol", "Float"})
238
+
239
+
240
+ def _validate_ast(code: str, local_dict: dict) -> None:
241
+ """Refuse anything outside the closed node allowlist. Raises
242
+ MathParseError; returns None when the expression is acceptable."""
243
+ try:
244
+ tree = ast.parse(code, mode="eval")
245
+ except SyntaxError as e:
246
+ raise MathParseError(f"Could not parse expression: {e}") from e
247
+
248
+ for node in ast.walk(tree):
249
+ if not isinstance(node, _ALLOWED_NODES):
250
+ raise MathParseError(
251
+ f"{type(node).__name__} is not allowed in a math "
252
+ f"expression")
253
+
254
+ # `**` UNPACKING IS THE ONE CONSTRUCT A LISTED NODE SPELLS
255
+ # (TECHNICAL_DEBT 55, batch 160). `f(*args)` is `ast.Starred`,
256
+ # which is unlisted. `f(**mapping)` is `ast.keyword(arg=None)`, and
257
+ # `keyword` is listed for ordinary keyword arguments, so the second
258
+ # form walked past the node check and was refused only later, by
259
+ # SymPy ("sin takes exactly 1 argument"). It was inert, because no
260
+ # str-keyed mapping is reachable here, but inert because of what is
261
+ # unreachable is not what this allowlist claims. No legitimate
262
+ # expression needs it: measured, even ordinary keyword arguments do
263
+ # not survive `auto_symbol` (`prec=20` is refused as a name).
264
+ if isinstance(node, ast.keyword) and node.arg is None:
265
+ raise MathParseError(
266
+ "** unpacking is not allowed in a math expression")
267
+
268
+ # Every name is resolved against _SAFE_GLOBALS at eval time, so
269
+ # every name must be one we chose. Unknown names never reach here
270
+ # -- auto_symbol has already rewritten them to Symbol('name') --
271
+ # except the ones the CALLER declared via `symbols=`, which is
272
+ # what local_dict holds.
273
+ if isinstance(node, ast.Name):
274
+ if node.id not in _ALLOWED_NAMES and node.id not in local_dict:
275
+ raise MathParseError(
276
+ f"'{node.id}' is not a permitted function or "
277
+ f"constant. Use a mathematical function, or a plain "
278
+ f"name for a variable.")
279
+
280
+ # A STRING may appear only where the transformations put one.
281
+ #
282
+ # Seven allowlisted functions call `sympify()` on their own arguments
283
+ # -- `factor`, `cancel`, `together`, `apart`, `roots`, `degree`,
284
+ # `nsolve` -- which was found by auditing the allowlist's source
285
+ # rather than by guessing. Handing any of them a string literal
286
+ # re-enters the parser with SymPy's own default globals, real
287
+ # builtins included, and that is #52's mechanism 1 through a side
288
+ # door:
289
+ #
290
+ # factor(" __import__(chr(111)+chr(115)).getpid() ") -> the pid
291
+ #
292
+ # The leading space matters: an earlier version of this guard only
293
+ # tripped on a string that *started or ended* with a dunder, so one
294
+ # space defeated it. Prefix matching on a payload is the same
295
+ # category of mistake as `chr(111)+chr(115)` defeating a search for
296
+ # the literal "os".
297
+ #
298
+ # `stringify_expr` emits string arguments in exactly two places --
299
+ # `Symbol('x')` for every free variable and `Float('1.5')` for every
300
+ # non-integer literal -- so licensing those two and refusing every
301
+ # other string keeps the property closed rather than filtered. This
302
+ # also removes `"%s" % pi` and every other string-valued construct
303
+ # without needing a rule per construct.
304
+ #
305
+ # A `bytes` LITERAL IS REFUSED IN THE SAME BREATH, and that is the one
306
+ # non-`str` type it has to be: `eval()` and `compile()` both accept
307
+ # bytes where they reject every other non-string object, so a future
308
+ # SymPy callable that sympifies what it is handed would land there.
309
+ # The rule here used to name `str` alone, and `factor(b"1+1")`,
310
+ # `b"abc"` and `roots(b"x**2-1")` were all accepted -- measured, with
311
+ # the `str` spelling of each refused as the control. No expression
312
+ # needs one: `stringify_expr` never emits a bytes argument, so the
313
+ # licensing pass below deliberately licenses `str` only, which refuses
314
+ # even `Symbol(b"x")` (batch 143).
315
+ #
316
+ # Naming one of the two string types is the same category of mistake
317
+ # as the dunder tripwire above: a near-miss on a type check rather
318
+ # than on a payload.
319
+ licensed = set()
320
+ for node in ast.walk(tree):
321
+ if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
322
+ and node.func.id in _STRING_ARG_CONSTRUCTORS):
323
+ for arg in node.args:
324
+ if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
325
+ licensed.add(id(arg))
326
+
327
+ for node in ast.walk(tree):
328
+ if (isinstance(node, ast.Constant)
329
+ and isinstance(node.value, (str, bytes))
330
+ and id(node) not in licensed):
331
+ raise MathParseError(
332
+ "string literals are not allowed in a math expression")
333
+
334
+
335
+ def _reject_escaped_value(value: Any, _depth: int = 0) -> None:
336
+ """Backstop: refuse a RESULT that is not a mathematical value.
337
+
338
+ Every guard above this one is an allowlist over the *input*. This is
339
+ the one check on the *output*, and it is deliberately a DENY of the
340
+ escape signature rather than an allowlist: its job is to catch an
341
+ escape nobody anticipated, at the moment it succeeds, and an
342
+ allowlist of return types can only catch escapes whose shape was
343
+ already imagined.
344
+
345
+ The signature is simple. A math expression evaluates to a VALUE -- a
346
+ number, an expression tree, a matrix, a list of solutions. It never
347
+ evaluates to a module, to a plain function, or to a class from
348
+ outside SymPy. Those are what every escape in #52 produced or reached
349
+ for: `<module 'os'>`, `__subclasses__`, a bound method's `__globals__`.
350
+
351
+ SymPy's own classes ARE allowed: `sin` is a `FunctionClass`, and
352
+ `Function('f')` legitimately evaluates to a class. Both are subclasses
353
+ of `Basic`, which is what separates them from `type` or `os`.
354
+ """
355
+ if _depth > 6:
356
+ # FAILS CLOSED. This used to `return`, which gave the one check
357
+ # written for an escape nobody anticipated an anticipated DEPTH:
358
+ # `[[[os]]]` was refused and `[[[[[[[[os]]]]]]]]` was accepted
359
+ # uninspected -- measured, with the depth-3 case as the control
360
+ # proving the function was live. The comment that stood here
361
+ # ("containers are shallow here") was an assertion about today's
362
+ # return shapes inside the one function whose stated job is the
363
+ # opposite of relying on them.
364
+ #
365
+ # The cap's job is to bound the WALK, which it still does. The
366
+ # deepest legitimate return shape in the battery is `eigenvects()`
367
+ # at 4, and `test_the_backstop_allows_every_legitimate_return_shape`
368
+ # is what goes red if that stops being true (batch 143).
369
+ raise MathParseError(
370
+ "expression evaluated to a value nested more than 6 deep, "
371
+ "which is not a mathematical value")
372
+
373
+ if isinstance(value, types.ModuleType):
374
+ raise MathParseError(
375
+ "expression evaluated to a module, which is never a "
376
+ "mathematical value")
377
+
378
+ if isinstance(value, type):
379
+ if not issubclass(value, (sympy.Basic, sympy.matrices.MatrixBase)):
380
+ raise MathParseError(
381
+ f"expression evaluated to the class {value.__name__!r}, "
382
+ f"which is not a SymPy type")
383
+ return
384
+
385
+ # No types.ModuleType here: the branch above already raised for one,
386
+ # so a second listing is unreachable and reads as if this arm were the
387
+ # module guard.
388
+ if isinstance(value, (types.FunctionType, types.BuiltinFunctionType,
389
+ types.MethodType, types.BuiltinMethodType,
390
+ types.MethodWrapperType)):
391
+ raise MathParseError(
392
+ "expression evaluated to a function or method, which is "
393
+ "never a mathematical value")
394
+
395
+ # Raw bytes are not a mathematical value either, and they are listed
396
+ # here for the same reason the AST rule names them: `b'1+1'`,
397
+ # `[b'x']` and `{"k": b'x'}` all walked past this function, because
398
+ # bytes is not a module, a type, a function, or one of the four
399
+ # containers below (batch 143).
400
+ if isinstance(value, (bytes, bytearray, memoryview)):
401
+ raise MathParseError(
402
+ "expression evaluated to raw bytes, which is never a "
403
+ "mathematical value")
404
+
405
+ if isinstance(value, dict):
406
+ for k, v in value.items():
407
+ _reject_escaped_value(k, _depth + 1)
408
+ _reject_escaped_value(v, _depth + 1)
409
+ elif isinstance(value, (list, tuple, set, frozenset)):
410
+ for item in value:
411
+ _reject_escaped_value(item, _depth + 1)
412
+
413
+
414
+ # A name token, ignoring one preceded by a word character or a dot, so the
415
+ # `e5` inside `1e5` is not read as a name the caller wrote.
416
+ _NAME_TOKEN = re.compile(r"(?<![\w.])[A-Za-z_][A-Za-z0-9_]*")
417
+
418
+
419
+ def _rewritten_names(expr_str: str, result: Any, local_dict: dict) -> set:
420
+ """The multi-character names the caller wrote that did NOT survive the
421
+ parse as themselves.
422
+
423
+ `_TRANSFORMATIONS` carries `split_symbols`, so a multi-character name
424
+ that is not in `local_dict` is rewritten into a PRODUCT of single-letter
425
+ Symbols before anything is computed: `mass` becomes `a*m*s**2`, and
426
+ `determinant([[mass,0],[0,mass]])` then answers `a**2*m**2*s**4` as a
427
+ successful result naming a quantity nobody asked about (batch 131).
428
+
429
+ Decided by comparing what was written against what came back rather than
430
+ by reimplementing SymPy's `_token_splittable`: whatever that predicate
431
+ does with a given spelling, a name is reported here exactly when it is
432
+ not among the result's free symbols. Three exclusions, each for its own
433
+ reason:
434
+
435
+ * a ONE-character name cannot be split, so it is never interesting;
436
+ * a name in `_ALLOWED_NAMES` is a SymPy binding the caller may well
437
+ have meant -- `E` is Euler's number and `sin` is a function, and
438
+ deciding that those were mistakes is not this function's business
439
+ (the `symbols=` route is how a caller overrides one);
440
+ * a DECLARED name is in `local_dict` and therefore already protected,
441
+ which is the whole point of declaring it.
442
+ """
443
+ try:
444
+ survived = {str(s) for s in result.free_symbols}
445
+ except AttributeError:
446
+ # A plain bool/int/str result has no free symbols; nothing survived,
447
+ # which is the right answer for a name that should have.
448
+ survived = set()
449
+ found = set()
450
+ for token in _NAME_TOKEN.findall(expr_str):
451
+ if len(token) < 2 or token in _ALLOWED_NAMES or token in local_dict:
452
+ continue
453
+ if token not in survived:
454
+ found.add(token)
455
+ return found
456
+
457
+
458
+ def _split_warnings(rewritten: set, field: str = "symbols") -> list[str]:
459
+ """One sentence per split name, for a tool's `warnings` list.
460
+
461
+ ONE copy, shared by every tool that reports this: four lists once
462
+ answered "what counts as a project file" and no two agreed (#96), and
463
+ four spellings of this sentence would be the same defect. The text names
464
+ the remedy because the model is the only caller there is, and nothing in
465
+ any schema told it the rule until batch 131.
466
+
467
+ `field` EXISTS BECAUSE THE REMEDY IS NOT THE SAME WORD EVERYWHERE
468
+ (batch 143). Three tools call the declaring parameter `symbols`;
469
+ `logic` calls it `variables`. One sentence with the field named by its
470
+ caller is one copy; a sentence that hardcodes `symbols=` is one copy
471
+ that is wrong for a quarter of its callers, which is worse than four.
472
+ """
473
+ return [
474
+ "'%s' was read as a product of single letters, not as one symbol. "
475
+ "Pass %s=['%s'] to keep it whole." % (name, field, name)
476
+ for name in sorted(rewritten)
477
+ ]
478
+
479
+
480
+ def parse_params(model, params: dict):
481
+ """Validate one tool's params, or refuse in a sentence the model can act on.
482
+
483
+ EVERY math tool built `Model(**params)` on the line ABOVE its `try`, so
484
+ a field constraint that fired left `run()` as a bare
485
+ `pydantic.ValidationError`. Measured through the real transport, with
486
+ `operation="nope"` -- the most ordinary mistake a caller can make:
487
+
488
+ {"error": "symbolic_math did not return a usable result (exit 1).
489
+ ... line 263, in __init__ ... pydantic_core._pydantic_core.
490
+ ValidationError: 1 validation error for SymbolicMathParams ..."}
491
+
492
+ plus the same traceback at WARNING, which the TUI's
493
+ `TranscriptLogHandler` draws on screen. That is the branch
494
+ `test_a_crashed_child_is_NOT_reported_as_a_limit` reserves for a
495
+ harness BUG, and it is H6's defect one line higher up the same
496
+ function: a routine, input-shaped outcome taking the path that means
497
+ "this tool is broken". Three of the three constraints that already
498
+ existed -- `precision` (le=50), `order` (ge=1) and the `operation`
499
+ `Literal` itself -- reached it.
500
+
501
+ It is also why each tool's `else: return {"error": "Unknown
502
+ operation"}` arm has never run: the `Literal` refuses first, so the
503
+ readable message that arm was written to give was unreachable. This is
504
+ where it gets given.
505
+
506
+ Not a pydantic `ValidationError` passed through verbatim: that names
507
+ a file, a line and a pydantic.dev URL, none of which a model can act
508
+ on (batch 143).
509
+ """
510
+ try:
511
+ return model(**params)
512
+ except ValidationError as exc:
513
+ raise MathParamsError(_params_message(exc)) from None
514
+
515
+
516
+ def _params_message(exc: ValidationError) -> str:
517
+ """One sentence per rejected field, in the model's own vocabulary.
518
+
519
+ `err["msg"]` is pydantic's own wording, which for a `Literal` lists the
520
+ operations verbatim -- exactly what the caller needs and the one thing
521
+ the generic transport error cannot say.
522
+ """
523
+ parts = []
524
+ for err in exc.errors():
525
+ field = ".".join(str(p) for p in err.get("loc", ())) or "parameters"
526
+ parts.append("'%s' %s" % (field, err.get("msg", "is not valid")))
527
+ return ("The parameters were not accepted: %s. Correct the parameter "
528
+ "and call again." % "; ".join(parts))
529
+
530
+
531
+ def safe_parse(expr_str: str, symbols: Optional[list[str]] = None,
532
+ rewritten: Optional[set] = None):
533
+ """
534
+ Parses a math expression string into a SymPy object without exposing
535
+ Python builtins.
536
+
537
+ `symbols` IS THE ONLY WAY TO KEEP A MULTI-CHARACTER NAME WHOLE, and
538
+ that is the load-bearing half of this parameter rather than the
539
+ convenience it used to be documented as. `_TRANSFORMATIONS` carries
540
+ `split_symbols`, so a name absent from `local_dict` is rewritten into a
541
+ product of single letters before anything is computed -- `mass` parses
542
+ as `a*m*s**2`. Declaring it binds the name to a Symbol first, so it
543
+ survives. (It also keeps variable identity consistent across calls, e.g.
544
+ the same 'x' in an expression and a separate bound for a definite
545
+ integral, which is what this docstring said when `symbols=` was
546
+ reachable from only two of the four parsing tools.)
547
+
548
+ `rewritten`, when a set is passed, collects the names that were split
549
+ anyway -- a caller that cannot know in advance what the model will send
550
+ reports those rather than returning a confident answer about a
551
+ different expression. See `_rewritten_names`. The tools put them in a
552
+ `warnings` list, which is `probability_stats`'s existing convention for
553
+ "keep forgiving but explain".
554
+ """
555
+ local_dict = {name: sympy.Symbol(name) for name in (symbols or [])}
556
+ try:
557
+ # parse_expr() split open, so the guard can sit between the two
558
+ # halves -- see _validate_ast. `evaluate=False` is not used
559
+ # anywhere here, which is the only other thing parse_expr does.
560
+ code = stringify_expr(expr_str, local_dict, _SAFE_GLOBALS,
561
+ _TRANSFORMATIONS)
562
+ _validate_ast(code, local_dict)
563
+ result = eval_expr(code, local_dict, _SAFE_GLOBALS)
564
+ _reject_escaped_value(result)
565
+ if rewritten is not None:
566
+ rewritten.update(_rewritten_names(expr_str, result, local_dict))
567
+ return result
568
+ except MathParseError:
569
+ # A refusal from _validate_ast already says which name or node was
570
+ # refused and what to do instead. Re-wrapping it in "Could not
571
+ # parse expression '<the whole payload>'" buries that under the
572
+ # string the model just sent, and echoes the payload back into
573
+ # the transcript for no reason.
574
+ raise
575
+ except Exception as e:
576
+ raise MathParseError(f"Could not parse expression '{expr_str}': {e}") from e
577
+
578
+
579
+ def serialize(value: Any, what: str = "") -> Any:
580
+ """
581
+ Generic SymPy-object -> JSON-safe converter, used across all six math
582
+ tools -- AND THE ONE PLACE A MATH RESULT IS BOUNDED (batch 143).
583
+
584
+ Two raises, both `MathResultTooLarge`, both caught by the one
585
+ `except MathParseError` arm every caller already has.
586
+
587
+ H6, AT THE PRODUCER (batch 143). `str()` of an Integer above CPython's 4,300-digit cap
588
+ raises `ValueError`, and this function was called in the `return`
589
+ statement -- OUTSIDE every `try` -- in `symbolic_math`,
590
+ `linear_algebra` and `geometry`. So `series order=4000` and
591
+ `determinant([["factorial(3000)","0"],["0","1"]])` left `run()` as a
592
+ bare `ValueError`, were reported as `exit 1` ("did not return a usable
593
+ result"), and handed the model 400 characters of traceback naming
594
+ `C:\\Users\\<username>\\AppData\\Roaming\\...`. `discrete_math`
595
+ caught it and the other three did not; the guard belongs here because
596
+ this is what the three of them share. ROADMAP_v2 H6: *"A result the
597
+ tool cannot serialise is the tool's own error."*
598
+
599
+ THE CEILING (batch 143). No math tool bounded its own output. Measured,
600
+ the only ceiling was `tools/isolation.MAX_CHILD_OUTPUT_BYTES` at 4 MB
601
+ -- a TRANSPORT limit, 62x the premise the compaction arithmetic is
602
+ sized from (`config.yaml`: *"max_read_chars (50,000 chars = 12,500
603
+ proxy tokens, the largest a single bounded read can return)"*):
604
+
605
+ truth_table, 14 free symbols, 27 chars in 786,441 proxy tokens
606
+ series order=1000 -- the documented worst case 296,272
607
+ TECHNICAL_DEBT 41, the arxiv result already fixed 31,733
608
+
609
+ and `core/compaction.py` says what the headroom is for: nothing in
610
+ this harness recovers from a prompt that overflows the window. A
611
+ result 49x `compaction_min_headroom_tokens` cannot be contained by it.
612
+
613
+ `config.MAX_READ_CHARS` rather than a new constant, because that is
614
+ the number the arithmetic already names, and it is read at CALL time
615
+ so a config change needs no reimport. Measured blast radius at 50,000:
616
+ nothing in the legitimacy battery is refused -- `series order=200` is
617
+ 35,088 characters, `expand((x+y+z)**40)` is 27,340, every matrix,
618
+ geometry and factorial case is under 3,000 -- and the two things that
619
+ are refused are `series order=1000` and a truth table of 9+ symbols.
620
+ `logic` carries its own bound because it does not come through here
621
+ and because 2**n rows are predictable from the input, which is the H5
622
+ move; see `_MAX_TRUTH_TABLE_SYMBOLS` there.
623
+
624
+ REFUSE rather than truncate, per the owner's ruling: half a truth
625
+ table is a wrong answer presented as an answer, and `discrete_math`'s
626
+ H6 message and `isolation._child_main`'s own refusal are the wording
627
+ family this joins rather than a second convention.
628
+ """
629
+ label = ("The %s result" % what) if what else "The result"
630
+ try:
631
+ rendered = _rendered(value)
632
+ except ValueError as e:
633
+ # The ONLY ValueError `_rendered` can raise: it does isinstance
634
+ # checks, container recursion and `str()`, and `str()` of a large
635
+ # Integer is the documented cap.
636
+ raise MathResultTooLarge(
637
+ "%s was computed but is too large to return (%s). Ask for the "
638
+ "same operation with smaller inputs." % (label, e)) from None
639
+
640
+ # The JSON length rather than the character count of the pieces: this
641
+ # is what the child actually writes, escaping included, so the number
642
+ # in the message is the number that would have been sent.
643
+ size = len(json.dumps(rendered))
644
+ ceiling = int(config.MAX_READ_CHARS)
645
+ if size > ceiling:
646
+ raise MathResultTooLarge(
647
+ "%s was computed but is too large to return (%d characters, "
648
+ "against a ceiling of %d). Ask for the same operation with "
649
+ "smaller inputs -- fewer series terms, a lower order, a "
650
+ "smaller matrix." % (label, size, ceiling))
651
+ return rendered
652
+
653
+
654
+ def _rendered(value: Any) -> Any:
655
+ """`serialize` without the bounds. Separate so the size is measured
656
+ ONCE, on the finished structure, rather than on each recursion -- a
657
+ cap applied per level would pass a 4 MB matrix of one-character
658
+ cells."""
659
+ if isinstance(value, sympy.Matrix):
660
+ return [[str(cell) for cell in row] for row in value.tolist()]
661
+ if isinstance(value, dict):
662
+ return {str(k): _rendered(v) for k, v in value.items()}
663
+ if isinstance(value, (list, tuple)):
664
+ return [_rendered(v) for v in value]
665
+ if isinstance(value, bool):
666
+ return value
667
+ return str(value)