@yahyahammad/venastine 0.2.0 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -4
- package/CONFIG_ARCHITECTURE.md +2030 -0
- package/LICENSE +201 -201
- package/NOTICE +1 -1
- package/README.md +1034 -895
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -88
- package/agents/builtin/review.md +74 -74
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +68 -2
- package/agents/subagent_tool.py +472 -238
- package/agents/tui_commands.py +238 -151
- package/bin/venastine.mjs +150 -4
- package/bin/venastine.py +71 -0
- package/cli_commands.py +838 -0
- package/config.py +95 -1198
- package/config.yaml +1103 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +971 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -118
- package/core/ansi.py +452 -0
- package/core/approval.py +137 -115
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +2715 -1559
- package/core/commands.py +460 -0
- package/core/compaction.py +1735 -971
- package/core/config_loader.py +1982 -1563
- package/core/connect.py +686 -0
- package/core/events.py +123 -61
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +411 -328
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1490
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +190 -176
- package/core/provider_errors.py +234 -0
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -468
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +1561 -1382
- package/core/reasoning/output_writer.py +345 -286
- package/core/reasoning/payload_validation.py +524 -398
- package/core/reasoning/pipeline_storage.py +88 -22
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +458 -434
- package/core/reasoning/source_corpus.py +433 -327
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -183
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +148 -20
- package/database.py +333 -252
- package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +50 -9
- package/json_store.py +343 -123
- package/logging_setup.py +295 -185
- package/main.py +3433 -1961
- package/mcp_client/client.py +733 -499
- package/mcp_client/config.py +126 -21
- package/mcp_client/registration.py +343 -182
- package/memories/manager.py +147 -132
- package/memories/tui_commands.py +319 -244
- package/package.json +303 -73
- package/project_init/config_files.py +282 -278
- package/project_init/generator.py +728 -564
- package/project_init/manifest.py +18 -6
- package/project_init/tui_commands.py +147 -144
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +2 -0
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -262
- package/providers.json.example +4 -2
- package/pyproject.toml +149 -100
- package/requirements.txt +295 -72
- package/safety/policy_enforcement.py +1082 -746
- package/security/askpass.py +419 -0
- package/security/capability.py +207 -177
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +237 -190
- package/security/posture.py +124 -12
- package/security/protected_paths.py +501 -309
- package/security/sandbox.py +4108 -985
- package/security/secrets.py +855 -0
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -111
- package/storage.py +1338 -1005
- package/tools/base.py +161 -1
- package/tools/builtin/_math_common.py +667 -377
- package/tools/builtin/_net_common.py +225 -19
- package/tools/builtin/arxiv.py +497 -39
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +371 -153
- package/tools/builtin/file_ops.py +720 -412
- package/tools/builtin/geometry.py +48 -12
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +76 -66
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +227 -193
- package/tools/builtin/probability_stats.py +12 -16
- package/tools/builtin/project_docs.py +464 -405
- package/tools/builtin/remember.py +175 -158
- package/tools/builtin/shell.py +980 -461
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +149 -147
- package/tools/builtin/web_search.py +191 -13
- package/tools/context.py +159 -136
- package/tools/isolation.py +107 -34
- package/tools/registry.py +1242 -925
- package/tui/app.py +8318 -4100
- package/tui/app.tcss +747 -556
- package/tui/commands.py +26 -170
- package/tui/diffs.py +34 -13
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -1279
- package/tui/meters.py +364 -268
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -943
- package/tui/themes.py +599 -559
- package/tui/widgets.py +3813 -2332
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
|
@@ -1,377 +1,667 @@
|
|
|
1
|
-
"""
|
|
2
|
-
tools/builtin/_math_common.py
|
|
3
|
-
|
|
4
|
-
Shared safe-parsing foundation for every math tool. Prefixed with an
|
|
5
|
-
underscore since this isn't itself a registered tool -- it's imported by
|
|
6
|
-
the six math tools that are.
|
|
7
|
-
|
|
8
|
-
SAFETY NOTE: SymPy's parser (sympify/parse_expr) tokenizes and then calls
|
|
9
|
-
Python's eval() on the result, so an expression string is code. Every one
|
|
10
|
-
of them arrives from the model, which during the grounding passes has been
|
|
11
|
-
reading pages it did not author, and all six math tools are
|
|
12
|
-
`permission=True, approval=False` -- advertised and callable with no
|
|
13
|
-
prompt in plain chat AND in all ten research passes.
|
|
14
|
-
|
|
15
|
-
BLANKING `__builtins__` IS NOT SUFFICIENT, AND CANNOT BE (#52). That was
|
|
16
|
-
this module's original premise and it was wrong in two independent ways:
|
|
17
|
-
|
|
18
|
-
1. `_SAFE_GLOBALS` is built from `dir(sympy)`, so SymPy's OWN parser
|
|
19
|
-
entry points are in the namespace by construction -- `sympify`,
|
|
20
|
-
`parse_expr`, `var`, `init_printing`, `S`, `lambdify`. `sympify`'s
|
|
21
|
-
inner `parse_expr` builds its own globals via
|
|
22
|
-
`exec('from sympy import *', global_dict)`, and Python inserts REAL
|
|
23
|
-
`__builtins__` into any globals dict handed to exec/eval that lacks
|
|
24
|
-
it. The outer blanking never applies to the inner parse.
|
|
25
|
-
2. `().__class__.__bases__[0].__subclasses__()` needs no builtins at
|
|
26
|
-
all -- only attribute access on a literal -- and yields hundreds of
|
|
27
|
-
live classes. No namespace pruning can stop that.
|
|
28
|
-
|
|
29
|
-
A THIRD mechanism killed the first fix, and is why the guard looks the
|
|
30
|
-
way it does. That fix filtered NAME *tokens* against an allowlist. It was
|
|
31
|
-
bypassed within the hour:
|
|
32
|
-
|
|
33
|
-
f"{sympify('__import__(chr(111)+chr(115)).getpid()')}" -> the pid
|
|
34
|
-
|
|
35
|
-
On Python 3.11 an f-string is a SINGLE `STRING` token -- PEP 701 only
|
|
36
|
-
split it into `FSTRING_*` tokens in 3.12 -- so a NAME-token filter is
|
|
37
|
-
structurally blind to everything inside one, while the code inside is
|
|
38
|
-
evaluated with `_SAFE_GLOBALS` in scope, `sympify` included. `str.format`
|
|
39
|
-
reaches dunder attributes the same way, through its own mini-language.
|
|
40
|
-
|
|
41
|
-
The lesson generalises past f-strings: **a token stream is not the
|
|
42
|
-
language**, and which constructs the tokenizer hides is a property of the
|
|
43
|
-
Python version rather than of this code.
|
|
44
|
-
|
|
45
|
-
So the guard is `_validate_ast`, an allowlist over AST NODE TYPES applied
|
|
46
|
-
to exactly the string that will be evaluated -- see the long comment
|
|
47
|
-
above it. `parse_expr` is `stringify_expr` followed by `eval_expr`, and
|
|
48
|
-
the check sits between them.
|
|
49
|
-
|
|
50
|
-
One thing NOT to rely on: today the 26 submodule objects in
|
|
51
|
-
`_SAFE_GLOBALS` (`external.gmpy.os`, `printing.gtk.subprocess`,
|
|
52
|
-
`parsing.sympy_parser.builtins`, ...) happen to be unreachable, because
|
|
53
|
-
`auto_symbol` leaves a name alone only when it is a Basic, a type, or
|
|
54
|
-
callable -- and a module is none of the three, so it is rewritten to
|
|
55
|
-
`Symbol('external')`. That is an undocumented SymPy implementation detail
|
|
56
|
-
protecting a real hole. Attribute access is refused outright and the name
|
|
57
|
-
allowlist excludes every module, so nothing depends on it.
|
|
58
|
-
"""
|
|
59
|
-
|
|
60
|
-
from __future__ import annotations
|
|
61
|
-
|
|
62
|
-
import ast
|
|
63
|
-
import
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
import
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
#
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
#
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
#
|
|
167
|
-
#
|
|
168
|
-
#
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
#
|
|
176
|
-
#
|
|
177
|
-
#
|
|
178
|
-
#
|
|
179
|
-
#
|
|
180
|
-
#
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
#
|
|
199
|
-
#
|
|
200
|
-
#
|
|
201
|
-
#
|
|
202
|
-
#
|
|
203
|
-
#
|
|
204
|
-
#
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
#
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
`
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
#
|
|
314
|
-
#
|
|
315
|
-
#
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
#
|
|
357
|
-
#
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
if isinstance(value,
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
1
|
+
"""
|
|
2
|
+
tools/builtin/_math_common.py
|
|
3
|
+
|
|
4
|
+
Shared safe-parsing foundation for every math tool. Prefixed with an
|
|
5
|
+
underscore since this isn't itself a registered tool -- it's imported by
|
|
6
|
+
the six math tools that are.
|
|
7
|
+
|
|
8
|
+
SAFETY NOTE: SymPy's parser (sympify/parse_expr) tokenizes and then calls
|
|
9
|
+
Python's eval() on the result, so an expression string is code. Every one
|
|
10
|
+
of them arrives from the model, which during the grounding passes has been
|
|
11
|
+
reading pages it did not author, and all six math tools are
|
|
12
|
+
`permission=True, approval=False` -- advertised and callable with no
|
|
13
|
+
prompt in plain chat AND in all ten research passes.
|
|
14
|
+
|
|
15
|
+
BLANKING `__builtins__` IS NOT SUFFICIENT, AND CANNOT BE (#52). That was
|
|
16
|
+
this module's original premise and it was wrong in two independent ways:
|
|
17
|
+
|
|
18
|
+
1. `_SAFE_GLOBALS` is built from `dir(sympy)`, so SymPy's OWN parser
|
|
19
|
+
entry points are in the namespace by construction -- `sympify`,
|
|
20
|
+
`parse_expr`, `var`, `init_printing`, `S`, `lambdify`. `sympify`'s
|
|
21
|
+
inner `parse_expr` builds its own globals via
|
|
22
|
+
`exec('from sympy import *', global_dict)`, and Python inserts REAL
|
|
23
|
+
`__builtins__` into any globals dict handed to exec/eval that lacks
|
|
24
|
+
it. The outer blanking never applies to the inner parse.
|
|
25
|
+
2. `().__class__.__bases__[0].__subclasses__()` needs no builtins at
|
|
26
|
+
all -- only attribute access on a literal -- and yields hundreds of
|
|
27
|
+
live classes. No namespace pruning can stop that.
|
|
28
|
+
|
|
29
|
+
A THIRD mechanism killed the first fix, and is why the guard looks the
|
|
30
|
+
way it does. That fix filtered NAME *tokens* against an allowlist. It was
|
|
31
|
+
bypassed within the hour:
|
|
32
|
+
|
|
33
|
+
f"{sympify('__import__(chr(111)+chr(115)).getpid()')}" -> the pid
|
|
34
|
+
|
|
35
|
+
On Python 3.11 an f-string is a SINGLE `STRING` token -- PEP 701 only
|
|
36
|
+
split it into `FSTRING_*` tokens in 3.12 -- so a NAME-token filter is
|
|
37
|
+
structurally blind to everything inside one, while the code inside is
|
|
38
|
+
evaluated with `_SAFE_GLOBALS` in scope, `sympify` included. `str.format`
|
|
39
|
+
reaches dunder attributes the same way, through its own mini-language.
|
|
40
|
+
|
|
41
|
+
The lesson generalises past f-strings: **a token stream is not the
|
|
42
|
+
language**, and which constructs the tokenizer hides is a property of the
|
|
43
|
+
Python version rather than of this code.
|
|
44
|
+
|
|
45
|
+
So the guard is `_validate_ast`, an allowlist over AST NODE TYPES applied
|
|
46
|
+
to exactly the string that will be evaluated -- see the long comment
|
|
47
|
+
above it. `parse_expr` is `stringify_expr` followed by `eval_expr`, and
|
|
48
|
+
the check sits between them.
|
|
49
|
+
|
|
50
|
+
One thing NOT to rely on: today the 26 submodule objects in
|
|
51
|
+
`_SAFE_GLOBALS` (`external.gmpy.os`, `printing.gtk.subprocess`,
|
|
52
|
+
`parsing.sympy_parser.builtins`, ...) happen to be unreachable, because
|
|
53
|
+
`auto_symbol` leaves a name alone only when it is a Basic, a type, or
|
|
54
|
+
callable -- and a module is none of the three, so it is rewritten to
|
|
55
|
+
`Symbol('external')`. That is an undocumented SymPy implementation detail
|
|
56
|
+
protecting a real hole. Attribute access is refused outright and the name
|
|
57
|
+
allowlist excludes every module, so nothing depends on it.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
from __future__ import annotations
|
|
61
|
+
|
|
62
|
+
import ast
|
|
63
|
+
import json
|
|
64
|
+
import re
|
|
65
|
+
import types
|
|
66
|
+
from typing import Any, Optional
|
|
67
|
+
|
|
68
|
+
import sympy
|
|
69
|
+
from pydantic import ValidationError
|
|
70
|
+
from sympy.parsing.sympy_parser import (
|
|
71
|
+
convert_equals_signs,
|
|
72
|
+
eval_expr,
|
|
73
|
+
implicit_multiplication_application,
|
|
74
|
+
standard_transformations,
|
|
75
|
+
stringify_expr,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
import config
|
|
79
|
+
|
|
80
|
+
# SymPy's own public namespace (sin, cos, sqrt, pi, Symbol, Matrix, oo, ...)
|
|
81
|
+
# with __builtins__ explicitly neutered. Built once at import time.
|
|
82
|
+
#
|
|
83
|
+
# The blanking is kept -- it is not what makes this safe (see the module
|
|
84
|
+
# docstring), but removing it would re-open the one payload shape the
|
|
85
|
+
# original test covered, for no gain.
|
|
86
|
+
_SAFE_GLOBALS: dict[str, Any] = {
|
|
87
|
+
name: getattr(sympy, name) for name in dir(sympy) if not name.startswith("_")
|
|
88
|
+
}
|
|
89
|
+
_SAFE_GLOBALS["__builtins__"] = {}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class MathParseError(ValueError):
|
|
93
|
+
"""Raised when an expression string can't be safely parsed.
|
|
94
|
+
|
|
95
|
+
ALSO THE BASE OF THE TWO BELOW, neither of which is a parse failure.
|
|
96
|
+
They are in this family for one mechanical reason: every tool already
|
|
97
|
+
has exactly one `except MathParseError as e: return {"error": str(e)}`
|
|
98
|
+
arm, so a subclass is caught by a handler that is already there and
|
|
99
|
+
already tested. Two new `except` arms across six tools would be twelve
|
|
100
|
+
places to add and twelve chances to miss one, which is the shape #96
|
|
101
|
+
and EP6 are both about (batch 143).
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class MathResultTooLarge(MathParseError):
|
|
106
|
+
"""A result that was computed CORRECTLY and cannot be returned.
|
|
107
|
+
|
|
108
|
+
Two causes, one message family, because the model's remedy is the same
|
|
109
|
+
for both: CPython caps `int -> str` at 4,300 digits, and a math result
|
|
110
|
+
is bounded at `config.MAX_READ_CHARS`. See `serialize`.
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class MathParamsError(MathParseError):
|
|
115
|
+
"""The tool's own params did not validate. See `parse_params`."""
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# What a math EXPRESSION may name. Grouped by category rather than listed
|
|
119
|
+
# flat, so that widening it is a visible decision about a KIND of thing
|
|
120
|
+
# rather than a name appended to a pile.
|
|
121
|
+
#
|
|
122
|
+
# The rule for admitting a name: it computes over SymPy objects. Anything
|
|
123
|
+
# that parses, evaluates, compiles, renders or executes stays out --
|
|
124
|
+
# `sympify`, `parse_expr`, `var`, `S`, `lambdify`, `init_printing`,
|
|
125
|
+
# `preview`, and the 26 submodule objects, all of which are reachable
|
|
126
|
+
# names in `_SAFE_GLOBALS` and none of which a math expression needs.
|
|
127
|
+
#
|
|
128
|
+
# Being CLOSED is the property that matters. A future SymPy that adds a
|
|
129
|
+
# new evaluating callable, or that stops rewriting bare module names into
|
|
130
|
+
# Symbols, is refused by default rather than newly exposed. That is the
|
|
131
|
+
# whole reason this is an allowlist and not a denylist of known-bad names.
|
|
132
|
+
_ALLOWED_NAMES = frozenset("""
|
|
133
|
+
sin cos tan cot sec csc sinc asin acos atan atan2 acot asec acsc
|
|
134
|
+
sinh cosh tanh coth sech csch asinh acosh atanh acoth asech acsch
|
|
135
|
+
exp log ln sqrt cbrt root Abs sign floor ceiling frac Mod
|
|
136
|
+
factorial factorial2 subfactorial binomial gamma lowergamma uppergamma
|
|
137
|
+
loggamma digamma trigamma polygamma beta
|
|
138
|
+
erf erfc erfi erfinv erf2 zeta dirichlet_eta lerchphi polylog
|
|
139
|
+
LambertW Ei li Li Si Ci Shi Chi expint
|
|
140
|
+
besselj bessely besseli besselk hankel1 hankel2 airyai airybi
|
|
141
|
+
jacobi gegenbauer chebyshevt chebyshevu legendre assoc_legendre
|
|
142
|
+
hermite laguerre assoc_laguerre hyper meijerg elliptic_k elliptic_e
|
|
143
|
+
conjugate re im arg Add Mul Pow
|
|
144
|
+
fibonacci lucas tribonacci catalan harmonic bernoulli euler
|
|
145
|
+
primepi prime isprime nextprime prevprime factorint divisors totient
|
|
146
|
+
gcd lcm igcd ilcm mod_inverse
|
|
147
|
+
pi E I oo zoo nan GoldenRatio EulerGamma Catalan TribonacciConstant
|
|
148
|
+
true false Integer Float Rational Symbol Tuple Dict
|
|
149
|
+
Matrix ImmutableMatrix eye zeros ones diag
|
|
150
|
+
Interval FiniteSet Union Intersection Complement EmptySet
|
|
151
|
+
Range ImageSet ConditionSet ProductSet Naturals Integers Reals
|
|
152
|
+
Eq Ne Lt Le Gt Ge StrictLessThan StrictGreaterThan
|
|
153
|
+
And Or Not Xor Nand Nor Implies Equivalent ITE
|
|
154
|
+
diff integrate limit Sum Product Derivative Integral Limit Order
|
|
155
|
+
simplify expand factor together apart cancel collect
|
|
156
|
+
trigsimp radsimp powsimp logcombine expand_trig expand_log
|
|
157
|
+
solve solveset roots nsolve dsolve linsolve nonlinsolve
|
|
158
|
+
Poly degree LC LT rem quo div
|
|
159
|
+
transpose det trace adjugate
|
|
160
|
+
Function Lambda Piecewise Max Min Heaviside DiracDelta KroneckerDelta
|
|
161
|
+
Point Point2D Point3D Line Segment Ray Circle Ellipse Polygon Triangle
|
|
162
|
+
N nsimplify
|
|
163
|
+
""".split())
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
# implicit_multiplication_application lets "2x" parse as "2*x".
|
|
167
|
+
# convert_equals_signs lets "x**2 - 4 = 0" parse into an Eq(...) object
|
|
168
|
+
# instead of raising a syntax error on the bare "=".
|
|
169
|
+
_TRANSFORMATIONS = standard_transformations + (
|
|
170
|
+
implicit_multiplication_application,
|
|
171
|
+
convert_equals_signs,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
# ---------------------------------------------------------------------------
|
|
176
|
+
# ---- The guard: an AST allowlist over the code that will be EVALUATED ------
|
|
177
|
+
# ---------------------------------------------------------------------------
|
|
178
|
+
#
|
|
179
|
+
# WHY NOT A TOKEN FILTER. The first attempt at #52 rejected dunder and
|
|
180
|
+
# non-allowlisted NAME *tokens*. It was bypassed within the hour:
|
|
181
|
+
#
|
|
182
|
+
# f"{sympify('__import__(chr(111)+chr(115)).getpid()')}" -> the pid
|
|
183
|
+
#
|
|
184
|
+
# On Python 3.11 an f-string is a SINGLE `STRING` token (PEP 701 only
|
|
185
|
+
# split it into `FSTRING_*` tokens in 3.12), so a NAME-token filter is
|
|
186
|
+
# structurally blind to everything inside one -- and the code inside is
|
|
187
|
+
# evaluated with `_SAFE_GLOBALS` in scope, `sympify` included.
|
|
188
|
+
# `"{0.__class__.__bases__}".format(pi)` reaches dunder attributes the
|
|
189
|
+
# same way, through `str.format`'s own mini-language.
|
|
190
|
+
#
|
|
191
|
+
# The lesson generalises past f-strings: a token stream is not the
|
|
192
|
+
# language. Any construct whose contents the tokenizer does not expose as
|
|
193
|
+
# tokens is invisible to a token filter, and "which constructs are those"
|
|
194
|
+
# is a property of the Python version, not of this code. That is not
|
|
195
|
+
# something a filter can be argued sound against.
|
|
196
|
+
#
|
|
197
|
+
# So the guard validates the ABSTRACT SYNTAX TREE of exactly the string
|
|
198
|
+
# that gets evaluated. `parse_expr` is `stringify_expr` (apply the
|
|
199
|
+
# transformations, emit code) followed by `eval_expr` (eval it), so
|
|
200
|
+
# splitting it open puts the check between those two steps -- after
|
|
201
|
+
# "2x" has become `Integer(2)*Symbol('x')`, and before anything runs.
|
|
202
|
+
#
|
|
203
|
+
# What that buys, and it is the whole point: the allowed set is CLOSED
|
|
204
|
+
# over node types. A construct nobody thought of -- an f-string, a walrus,
|
|
205
|
+
# a comprehension, a lambda, a generator, whatever a future Python adds --
|
|
206
|
+
# is refused because it is not on the list, not because someone
|
|
207
|
+
# remembered it.
|
|
208
|
+
|
|
209
|
+
_ALLOWED_NODES = (
|
|
210
|
+
ast.Expression,
|
|
211
|
+
ast.BinOp, ast.UnaryOp, ast.BoolOp, ast.Compare,
|
|
212
|
+
ast.Call, ast.keyword,
|
|
213
|
+
ast.Name, ast.Load,
|
|
214
|
+
ast.Constant,
|
|
215
|
+
ast.Tuple, ast.List,
|
|
216
|
+
ast.Subscript, ast.Slice,
|
|
217
|
+
# operators
|
|
218
|
+
ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow,
|
|
219
|
+
ast.LShift, ast.RShift, ast.BitOr, ast.BitXor, ast.BitAnd, ast.MatMult,
|
|
220
|
+
ast.UAdd, ast.USub, ast.Not, ast.Invert,
|
|
221
|
+
ast.And, ast.Or,
|
|
222
|
+
ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE,
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
# `ast.Attribute` is deliberately ABSENT, and it is the single most
|
|
226
|
+
# load-bearing omission. Attribute access IS #52's mechanism 2 --
|
|
227
|
+
# `().__class__.__bases__[0].__subclasses__()` needs no name lookup at
|
|
228
|
+
# all -- and it is also how `str.format` and every bound-method route is
|
|
229
|
+
# reached. No transformation emits an attribute for a legitimate
|
|
230
|
+
# expression, so nothing needs it: the functional forms (`diff(f, x)`
|
|
231
|
+
# rather than `f.diff(x)`) are what the tools take anyway.
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
# The only two calls `stringify_expr` gives a string argument to. Kept
|
|
235
|
+
# minimal on purpose: each entry is a callable that must be argued not to
|
|
236
|
+
# sympify what it is handed.
|
|
237
|
+
_STRING_ARG_CONSTRUCTORS = frozenset({"Symbol", "Float"})
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _validate_ast(code: str, local_dict: dict) -> None:
|
|
241
|
+
"""Refuse anything outside the closed node allowlist. Raises
|
|
242
|
+
MathParseError; returns None when the expression is acceptable."""
|
|
243
|
+
try:
|
|
244
|
+
tree = ast.parse(code, mode="eval")
|
|
245
|
+
except SyntaxError as e:
|
|
246
|
+
raise MathParseError(f"Could not parse expression: {e}") from e
|
|
247
|
+
|
|
248
|
+
for node in ast.walk(tree):
|
|
249
|
+
if not isinstance(node, _ALLOWED_NODES):
|
|
250
|
+
raise MathParseError(
|
|
251
|
+
f"{type(node).__name__} is not allowed in a math "
|
|
252
|
+
f"expression")
|
|
253
|
+
|
|
254
|
+
# `**` UNPACKING IS THE ONE CONSTRUCT A LISTED NODE SPELLS
|
|
255
|
+
# (TECHNICAL_DEBT 55, batch 160). `f(*args)` is `ast.Starred`,
|
|
256
|
+
# which is unlisted. `f(**mapping)` is `ast.keyword(arg=None)`, and
|
|
257
|
+
# `keyword` is listed for ordinary keyword arguments, so the second
|
|
258
|
+
# form walked past the node check and was refused only later, by
|
|
259
|
+
# SymPy ("sin takes exactly 1 argument"). It was inert, because no
|
|
260
|
+
# str-keyed mapping is reachable here, but inert because of what is
|
|
261
|
+
# unreachable is not what this allowlist claims. No legitimate
|
|
262
|
+
# expression needs it: measured, even ordinary keyword arguments do
|
|
263
|
+
# not survive `auto_symbol` (`prec=20` is refused as a name).
|
|
264
|
+
if isinstance(node, ast.keyword) and node.arg is None:
|
|
265
|
+
raise MathParseError(
|
|
266
|
+
"** unpacking is not allowed in a math expression")
|
|
267
|
+
|
|
268
|
+
# Every name is resolved against _SAFE_GLOBALS at eval time, so
|
|
269
|
+
# every name must be one we chose. Unknown names never reach here
|
|
270
|
+
# -- auto_symbol has already rewritten them to Symbol('name') --
|
|
271
|
+
# except the ones the CALLER declared via `symbols=`, which is
|
|
272
|
+
# what local_dict holds.
|
|
273
|
+
if isinstance(node, ast.Name):
|
|
274
|
+
if node.id not in _ALLOWED_NAMES and node.id not in local_dict:
|
|
275
|
+
raise MathParseError(
|
|
276
|
+
f"'{node.id}' is not a permitted function or "
|
|
277
|
+
f"constant. Use a mathematical function, or a plain "
|
|
278
|
+
f"name for a variable.")
|
|
279
|
+
|
|
280
|
+
# A STRING may appear only where the transformations put one.
|
|
281
|
+
#
|
|
282
|
+
# Seven allowlisted functions call `sympify()` on their own arguments
|
|
283
|
+
# -- `factor`, `cancel`, `together`, `apart`, `roots`, `degree`,
|
|
284
|
+
# `nsolve` -- which was found by auditing the allowlist's source
|
|
285
|
+
# rather than by guessing. Handing any of them a string literal
|
|
286
|
+
# re-enters the parser with SymPy's own default globals, real
|
|
287
|
+
# builtins included, and that is #52's mechanism 1 through a side
|
|
288
|
+
# door:
|
|
289
|
+
#
|
|
290
|
+
# factor(" __import__(chr(111)+chr(115)).getpid() ") -> the pid
|
|
291
|
+
#
|
|
292
|
+
# The leading space matters: an earlier version of this guard only
|
|
293
|
+
# tripped on a string that *started or ended* with a dunder, so one
|
|
294
|
+
# space defeated it. Prefix matching on a payload is the same
|
|
295
|
+
# category of mistake as `chr(111)+chr(115)` defeating a search for
|
|
296
|
+
# the literal "os".
|
|
297
|
+
#
|
|
298
|
+
# `stringify_expr` emits string arguments in exactly two places --
|
|
299
|
+
# `Symbol('x')` for every free variable and `Float('1.5')` for every
|
|
300
|
+
# non-integer literal -- so licensing those two and refusing every
|
|
301
|
+
# other string keeps the property closed rather than filtered. This
|
|
302
|
+
# also removes `"%s" % pi` and every other string-valued construct
|
|
303
|
+
# without needing a rule per construct.
|
|
304
|
+
#
|
|
305
|
+
# A `bytes` LITERAL IS REFUSED IN THE SAME BREATH, and that is the one
|
|
306
|
+
# non-`str` type it has to be: `eval()` and `compile()` both accept
|
|
307
|
+
# bytes where they reject every other non-string object, so a future
|
|
308
|
+
# SymPy callable that sympifies what it is handed would land there.
|
|
309
|
+
# The rule here used to name `str` alone, and `factor(b"1+1")`,
|
|
310
|
+
# `b"abc"` and `roots(b"x**2-1")` were all accepted -- measured, with
|
|
311
|
+
# the `str` spelling of each refused as the control. No expression
|
|
312
|
+
# needs one: `stringify_expr` never emits a bytes argument, so the
|
|
313
|
+
# licensing pass below deliberately licenses `str` only, which refuses
|
|
314
|
+
# even `Symbol(b"x")` (batch 143).
|
|
315
|
+
#
|
|
316
|
+
# Naming one of the two string types is the same category of mistake
|
|
317
|
+
# as the dunder tripwire above: a near-miss on a type check rather
|
|
318
|
+
# than on a payload.
|
|
319
|
+
licensed = set()
|
|
320
|
+
for node in ast.walk(tree):
|
|
321
|
+
if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
|
|
322
|
+
and node.func.id in _STRING_ARG_CONSTRUCTORS):
|
|
323
|
+
for arg in node.args:
|
|
324
|
+
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
|
325
|
+
licensed.add(id(arg))
|
|
326
|
+
|
|
327
|
+
for node in ast.walk(tree):
|
|
328
|
+
if (isinstance(node, ast.Constant)
|
|
329
|
+
and isinstance(node.value, (str, bytes))
|
|
330
|
+
and id(node) not in licensed):
|
|
331
|
+
raise MathParseError(
|
|
332
|
+
"string literals are not allowed in a math expression")
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _reject_escaped_value(value: Any, _depth: int = 0) -> None:
|
|
336
|
+
"""Backstop: refuse a RESULT that is not a mathematical value.
|
|
337
|
+
|
|
338
|
+
Every guard above this one is an allowlist over the *input*. This is
|
|
339
|
+
the one check on the *output*, and it is deliberately a DENY of the
|
|
340
|
+
escape signature rather than an allowlist: its job is to catch an
|
|
341
|
+
escape nobody anticipated, at the moment it succeeds, and an
|
|
342
|
+
allowlist of return types can only catch escapes whose shape was
|
|
343
|
+
already imagined.
|
|
344
|
+
|
|
345
|
+
The signature is simple. A math expression evaluates to a VALUE -- a
|
|
346
|
+
number, an expression tree, a matrix, a list of solutions. It never
|
|
347
|
+
evaluates to a module, to a plain function, or to a class from
|
|
348
|
+
outside SymPy. Those are what every escape in #52 produced or reached
|
|
349
|
+
for: `<module 'os'>`, `__subclasses__`, a bound method's `__globals__`.
|
|
350
|
+
|
|
351
|
+
SymPy's own classes ARE allowed: `sin` is a `FunctionClass`, and
|
|
352
|
+
`Function('f')` legitimately evaluates to a class. Both are subclasses
|
|
353
|
+
of `Basic`, which is what separates them from `type` or `os`.
|
|
354
|
+
"""
|
|
355
|
+
if _depth > 6:
|
|
356
|
+
# FAILS CLOSED. This used to `return`, which gave the one check
|
|
357
|
+
# written for an escape nobody anticipated an anticipated DEPTH:
|
|
358
|
+
# `[[[os]]]` was refused and `[[[[[[[[os]]]]]]]]` was accepted
|
|
359
|
+
# uninspected -- measured, with the depth-3 case as the control
|
|
360
|
+
# proving the function was live. The comment that stood here
|
|
361
|
+
# ("containers are shallow here") was an assertion about today's
|
|
362
|
+
# return shapes inside the one function whose stated job is the
|
|
363
|
+
# opposite of relying on them.
|
|
364
|
+
#
|
|
365
|
+
# The cap's job is to bound the WALK, which it still does. The
|
|
366
|
+
# deepest legitimate return shape in the battery is `eigenvects()`
|
|
367
|
+
# at 4, and `test_the_backstop_allows_every_legitimate_return_shape`
|
|
368
|
+
# is what goes red if that stops being true (batch 143).
|
|
369
|
+
raise MathParseError(
|
|
370
|
+
"expression evaluated to a value nested more than 6 deep, "
|
|
371
|
+
"which is not a mathematical value")
|
|
372
|
+
|
|
373
|
+
if isinstance(value, types.ModuleType):
|
|
374
|
+
raise MathParseError(
|
|
375
|
+
"expression evaluated to a module, which is never a "
|
|
376
|
+
"mathematical value")
|
|
377
|
+
|
|
378
|
+
if isinstance(value, type):
|
|
379
|
+
if not issubclass(value, (sympy.Basic, sympy.matrices.MatrixBase)):
|
|
380
|
+
raise MathParseError(
|
|
381
|
+
f"expression evaluated to the class {value.__name__!r}, "
|
|
382
|
+
f"which is not a SymPy type")
|
|
383
|
+
return
|
|
384
|
+
|
|
385
|
+
# No types.ModuleType here: the branch above already raised for one,
|
|
386
|
+
# so a second listing is unreachable and reads as if this arm were the
|
|
387
|
+
# module guard.
|
|
388
|
+
if isinstance(value, (types.FunctionType, types.BuiltinFunctionType,
|
|
389
|
+
types.MethodType, types.BuiltinMethodType,
|
|
390
|
+
types.MethodWrapperType)):
|
|
391
|
+
raise MathParseError(
|
|
392
|
+
"expression evaluated to a function or method, which is "
|
|
393
|
+
"never a mathematical value")
|
|
394
|
+
|
|
395
|
+
# Raw bytes are not a mathematical value either, and they are listed
|
|
396
|
+
# here for the same reason the AST rule names them: `b'1+1'`,
|
|
397
|
+
# `[b'x']` and `{"k": b'x'}` all walked past this function, because
|
|
398
|
+
# bytes is not a module, a type, a function, or one of the four
|
|
399
|
+
# containers below (batch 143).
|
|
400
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
401
|
+
raise MathParseError(
|
|
402
|
+
"expression evaluated to raw bytes, which is never a "
|
|
403
|
+
"mathematical value")
|
|
404
|
+
|
|
405
|
+
if isinstance(value, dict):
|
|
406
|
+
for k, v in value.items():
|
|
407
|
+
_reject_escaped_value(k, _depth + 1)
|
|
408
|
+
_reject_escaped_value(v, _depth + 1)
|
|
409
|
+
elif isinstance(value, (list, tuple, set, frozenset)):
|
|
410
|
+
for item in value:
|
|
411
|
+
_reject_escaped_value(item, _depth + 1)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
# A name token, ignoring one preceded by a word character or a dot, so the
|
|
415
|
+
# `e5` inside `1e5` is not read as a name the caller wrote.
|
|
416
|
+
_NAME_TOKEN = re.compile(r"(?<![\w.])[A-Za-z_][A-Za-z0-9_]*")
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _rewritten_names(expr_str: str, result: Any, local_dict: dict) -> set:
|
|
420
|
+
"""The multi-character names the caller wrote that did NOT survive the
|
|
421
|
+
parse as themselves.
|
|
422
|
+
|
|
423
|
+
`_TRANSFORMATIONS` carries `split_symbols`, so a multi-character name
|
|
424
|
+
that is not in `local_dict` is rewritten into a PRODUCT of single-letter
|
|
425
|
+
Symbols before anything is computed: `mass` becomes `a*m*s**2`, and
|
|
426
|
+
`determinant([[mass,0],[0,mass]])` then answers `a**2*m**2*s**4` as a
|
|
427
|
+
successful result naming a quantity nobody asked about (batch 131).
|
|
428
|
+
|
|
429
|
+
Decided by comparing what was written against what came back rather than
|
|
430
|
+
by reimplementing SymPy's `_token_splittable`: whatever that predicate
|
|
431
|
+
does with a given spelling, a name is reported here exactly when it is
|
|
432
|
+
not among the result's free symbols. Three exclusions, each for its own
|
|
433
|
+
reason:
|
|
434
|
+
|
|
435
|
+
* a ONE-character name cannot be split, so it is never interesting;
|
|
436
|
+
* a name in `_ALLOWED_NAMES` is a SymPy binding the caller may well
|
|
437
|
+
have meant -- `E` is Euler's number and `sin` is a function, and
|
|
438
|
+
deciding that those were mistakes is not this function's business
|
|
439
|
+
(the `symbols=` route is how a caller overrides one);
|
|
440
|
+
* a DECLARED name is in `local_dict` and therefore already protected,
|
|
441
|
+
which is the whole point of declaring it.
|
|
442
|
+
"""
|
|
443
|
+
try:
|
|
444
|
+
survived = {str(s) for s in result.free_symbols}
|
|
445
|
+
except AttributeError:
|
|
446
|
+
# A plain bool/int/str result has no free symbols; nothing survived,
|
|
447
|
+
# which is the right answer for a name that should have.
|
|
448
|
+
survived = set()
|
|
449
|
+
found = set()
|
|
450
|
+
for token in _NAME_TOKEN.findall(expr_str):
|
|
451
|
+
if len(token) < 2 or token in _ALLOWED_NAMES or token in local_dict:
|
|
452
|
+
continue
|
|
453
|
+
if token not in survived:
|
|
454
|
+
found.add(token)
|
|
455
|
+
return found
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def _split_warnings(rewritten: set, field: str = "symbols") -> list[str]:
|
|
459
|
+
"""One sentence per split name, for a tool's `warnings` list.
|
|
460
|
+
|
|
461
|
+
ONE copy, shared by every tool that reports this: four lists once
|
|
462
|
+
answered "what counts as a project file" and no two agreed (#96), and
|
|
463
|
+
four spellings of this sentence would be the same defect. The text names
|
|
464
|
+
the remedy because the model is the only caller there is, and nothing in
|
|
465
|
+
any schema told it the rule until batch 131.
|
|
466
|
+
|
|
467
|
+
`field` EXISTS BECAUSE THE REMEDY IS NOT THE SAME WORD EVERYWHERE
|
|
468
|
+
(batch 143). Three tools call the declaring parameter `symbols`;
|
|
469
|
+
`logic` calls it `variables`. One sentence with the field named by its
|
|
470
|
+
caller is one copy; a sentence that hardcodes `symbols=` is one copy
|
|
471
|
+
that is wrong for a quarter of its callers, which is worse than four.
|
|
472
|
+
"""
|
|
473
|
+
return [
|
|
474
|
+
"'%s' was read as a product of single letters, not as one symbol. "
|
|
475
|
+
"Pass %s=['%s'] to keep it whole." % (name, field, name)
|
|
476
|
+
for name in sorted(rewritten)
|
|
477
|
+
]
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def parse_params(model, params: dict):
|
|
481
|
+
"""Validate one tool's params, or refuse in a sentence the model can act on.
|
|
482
|
+
|
|
483
|
+
EVERY math tool built `Model(**params)` on the line ABOVE its `try`, so
|
|
484
|
+
a field constraint that fired left `run()` as a bare
|
|
485
|
+
`pydantic.ValidationError`. Measured through the real transport, with
|
|
486
|
+
`operation="nope"` -- the most ordinary mistake a caller can make:
|
|
487
|
+
|
|
488
|
+
{"error": "symbolic_math did not return a usable result (exit 1).
|
|
489
|
+
... line 263, in __init__ ... pydantic_core._pydantic_core.
|
|
490
|
+
ValidationError: 1 validation error for SymbolicMathParams ..."}
|
|
491
|
+
|
|
492
|
+
plus the same traceback at WARNING, which the TUI's
|
|
493
|
+
`TranscriptLogHandler` draws on screen. That is the branch
|
|
494
|
+
`test_a_crashed_child_is_NOT_reported_as_a_limit` reserves for a
|
|
495
|
+
harness BUG, and it is H6's defect one line higher up the same
|
|
496
|
+
function: a routine, input-shaped outcome taking the path that means
|
|
497
|
+
"this tool is broken". Three of the three constraints that already
|
|
498
|
+
existed -- `precision` (le=50), `order` (ge=1) and the `operation`
|
|
499
|
+
`Literal` itself -- reached it.
|
|
500
|
+
|
|
501
|
+
It is also why each tool's `else: return {"error": "Unknown
|
|
502
|
+
operation"}` arm has never run: the `Literal` refuses first, so the
|
|
503
|
+
readable message that arm was written to give was unreachable. This is
|
|
504
|
+
where it gets given.
|
|
505
|
+
|
|
506
|
+
Not a pydantic `ValidationError` passed through verbatim: that names
|
|
507
|
+
a file, a line and a pydantic.dev URL, none of which a model can act
|
|
508
|
+
on (batch 143).
|
|
509
|
+
"""
|
|
510
|
+
try:
|
|
511
|
+
return model(**params)
|
|
512
|
+
except ValidationError as exc:
|
|
513
|
+
raise MathParamsError(_params_message(exc)) from None
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _params_message(exc: ValidationError) -> str:
|
|
517
|
+
"""One sentence per rejected field, in the model's own vocabulary.
|
|
518
|
+
|
|
519
|
+
`err["msg"]` is pydantic's own wording, which for a `Literal` lists the
|
|
520
|
+
operations verbatim -- exactly what the caller needs and the one thing
|
|
521
|
+
the generic transport error cannot say.
|
|
522
|
+
"""
|
|
523
|
+
parts = []
|
|
524
|
+
for err in exc.errors():
|
|
525
|
+
field = ".".join(str(p) for p in err.get("loc", ())) or "parameters"
|
|
526
|
+
parts.append("'%s' %s" % (field, err.get("msg", "is not valid")))
|
|
527
|
+
return ("The parameters were not accepted: %s. Correct the parameter "
|
|
528
|
+
"and call again." % "; ".join(parts))
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def safe_parse(expr_str: str, symbols: Optional[list[str]] = None,
|
|
532
|
+
rewritten: Optional[set] = None):
|
|
533
|
+
"""
|
|
534
|
+
Parses a math expression string into a SymPy object without exposing
|
|
535
|
+
Python builtins.
|
|
536
|
+
|
|
537
|
+
`symbols` IS THE ONLY WAY TO KEEP A MULTI-CHARACTER NAME WHOLE, and
|
|
538
|
+
that is the load-bearing half of this parameter rather than the
|
|
539
|
+
convenience it used to be documented as. `_TRANSFORMATIONS` carries
|
|
540
|
+
`split_symbols`, so a name absent from `local_dict` is rewritten into a
|
|
541
|
+
product of single letters before anything is computed -- `mass` parses
|
|
542
|
+
as `a*m*s**2`. Declaring it binds the name to a Symbol first, so it
|
|
543
|
+
survives. (It also keeps variable identity consistent across calls, e.g.
|
|
544
|
+
the same 'x' in an expression and a separate bound for a definite
|
|
545
|
+
integral, which is what this docstring said when `symbols=` was
|
|
546
|
+
reachable from only two of the four parsing tools.)
|
|
547
|
+
|
|
548
|
+
`rewritten`, when a set is passed, collects the names that were split
|
|
549
|
+
anyway -- a caller that cannot know in advance what the model will send
|
|
550
|
+
reports those rather than returning a confident answer about a
|
|
551
|
+
different expression. See `_rewritten_names`. The tools put them in a
|
|
552
|
+
`warnings` list, which is `probability_stats`'s existing convention for
|
|
553
|
+
"keep forgiving but explain".
|
|
554
|
+
"""
|
|
555
|
+
local_dict = {name: sympy.Symbol(name) for name in (symbols or [])}
|
|
556
|
+
try:
|
|
557
|
+
# parse_expr() split open, so the guard can sit between the two
|
|
558
|
+
# halves -- see _validate_ast. `evaluate=False` is not used
|
|
559
|
+
# anywhere here, which is the only other thing parse_expr does.
|
|
560
|
+
code = stringify_expr(expr_str, local_dict, _SAFE_GLOBALS,
|
|
561
|
+
_TRANSFORMATIONS)
|
|
562
|
+
_validate_ast(code, local_dict)
|
|
563
|
+
result = eval_expr(code, local_dict, _SAFE_GLOBALS)
|
|
564
|
+
_reject_escaped_value(result)
|
|
565
|
+
if rewritten is not None:
|
|
566
|
+
rewritten.update(_rewritten_names(expr_str, result, local_dict))
|
|
567
|
+
return result
|
|
568
|
+
except MathParseError:
|
|
569
|
+
# A refusal from _validate_ast already says which name or node was
|
|
570
|
+
# refused and what to do instead. Re-wrapping it in "Could not
|
|
571
|
+
# parse expression '<the whole payload>'" buries that under the
|
|
572
|
+
# string the model just sent, and echoes the payload back into
|
|
573
|
+
# the transcript for no reason.
|
|
574
|
+
raise
|
|
575
|
+
except Exception as e:
|
|
576
|
+
raise MathParseError(f"Could not parse expression '{expr_str}': {e}") from e
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def serialize(value: Any, what: str = "") -> Any:
|
|
580
|
+
"""
|
|
581
|
+
Generic SymPy-object -> JSON-safe converter, used across all six math
|
|
582
|
+
tools -- AND THE ONE PLACE A MATH RESULT IS BOUNDED (batch 143).
|
|
583
|
+
|
|
584
|
+
Two raises, both `MathResultTooLarge`, both caught by the one
|
|
585
|
+
`except MathParseError` arm every caller already has.
|
|
586
|
+
|
|
587
|
+
H6, AT THE PRODUCER (batch 143). `str()` of an Integer above CPython's 4,300-digit cap
|
|
588
|
+
raises `ValueError`, and this function was called in the `return`
|
|
589
|
+
statement -- OUTSIDE every `try` -- in `symbolic_math`,
|
|
590
|
+
`linear_algebra` and `geometry`. So `series order=4000` and
|
|
591
|
+
`determinant([["factorial(3000)","0"],["0","1"]])` left `run()` as a
|
|
592
|
+
bare `ValueError`, were reported as `exit 1` ("did not return a usable
|
|
593
|
+
result"), and handed the model 400 characters of traceback naming
|
|
594
|
+
`C:\\Users\\<username>\\AppData\\Roaming\\...`. `discrete_math`
|
|
595
|
+
caught it and the other three did not; the guard belongs here because
|
|
596
|
+
this is what the three of them share. ROADMAP_v2 H6: *"A result the
|
|
597
|
+
tool cannot serialise is the tool's own error."*
|
|
598
|
+
|
|
599
|
+
THE CEILING (batch 143). No math tool bounded its own output. Measured,
|
|
600
|
+
the only ceiling was `tools/isolation.MAX_CHILD_OUTPUT_BYTES` at 4 MB
|
|
601
|
+
-- a TRANSPORT limit, 62x the premise the compaction arithmetic is
|
|
602
|
+
sized from (`config.yaml`: *"max_read_chars (50,000 chars = 12,500
|
|
603
|
+
proxy tokens, the largest a single bounded read can return)"*):
|
|
604
|
+
|
|
605
|
+
truth_table, 14 free symbols, 27 chars in 786,441 proxy tokens
|
|
606
|
+
series order=1000 -- the documented worst case 296,272
|
|
607
|
+
TECHNICAL_DEBT 41, the arxiv result already fixed 31,733
|
|
608
|
+
|
|
609
|
+
and `core/compaction.py` says what the headroom is for: nothing in
|
|
610
|
+
this harness recovers from a prompt that overflows the window. A
|
|
611
|
+
result 49x `compaction_min_headroom_tokens` cannot be contained by it.
|
|
612
|
+
|
|
613
|
+
`config.MAX_READ_CHARS` rather than a new constant, because that is
|
|
614
|
+
the number the arithmetic already names, and it is read at CALL time
|
|
615
|
+
so a config change needs no reimport. Measured blast radius at 50,000:
|
|
616
|
+
nothing in the legitimacy battery is refused -- `series order=200` is
|
|
617
|
+
35,088 characters, `expand((x+y+z)**40)` is 27,340, every matrix,
|
|
618
|
+
geometry and factorial case is under 3,000 -- and the two things that
|
|
619
|
+
are refused are `series order=1000` and a truth table of 9+ symbols.
|
|
620
|
+
`logic` carries its own bound because it does not come through here
|
|
621
|
+
and because 2**n rows are predictable from the input, which is the H5
|
|
622
|
+
move; see `_MAX_TRUTH_TABLE_SYMBOLS` there.
|
|
623
|
+
|
|
624
|
+
REFUSE rather than truncate, per the owner's ruling: half a truth
|
|
625
|
+
table is a wrong answer presented as an answer, and `discrete_math`'s
|
|
626
|
+
H6 message and `isolation._child_main`'s own refusal are the wording
|
|
627
|
+
family this joins rather than a second convention.
|
|
628
|
+
"""
|
|
629
|
+
label = ("The %s result" % what) if what else "The result"
|
|
630
|
+
try:
|
|
631
|
+
rendered = _rendered(value)
|
|
632
|
+
except ValueError as e:
|
|
633
|
+
# The ONLY ValueError `_rendered` can raise: it does isinstance
|
|
634
|
+
# checks, container recursion and `str()`, and `str()` of a large
|
|
635
|
+
# Integer is the documented cap.
|
|
636
|
+
raise MathResultTooLarge(
|
|
637
|
+
"%s was computed but is too large to return (%s). Ask for the "
|
|
638
|
+
"same operation with smaller inputs." % (label, e)) from None
|
|
639
|
+
|
|
640
|
+
# The JSON length rather than the character count of the pieces: this
|
|
641
|
+
# is what the child actually writes, escaping included, so the number
|
|
642
|
+
# in the message is the number that would have been sent.
|
|
643
|
+
size = len(json.dumps(rendered))
|
|
644
|
+
ceiling = int(config.MAX_READ_CHARS)
|
|
645
|
+
if size > ceiling:
|
|
646
|
+
raise MathResultTooLarge(
|
|
647
|
+
"%s was computed but is too large to return (%d characters, "
|
|
648
|
+
"against a ceiling of %d). Ask for the same operation with "
|
|
649
|
+
"smaller inputs -- fewer series terms, a lower order, a "
|
|
650
|
+
"smaller matrix." % (label, size, ceiling))
|
|
651
|
+
return rendered
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def _rendered(value: Any) -> Any:
|
|
655
|
+
"""`serialize` without the bounds. Separate so the size is measured
|
|
656
|
+
ONCE, on the finished structure, rather than on each recursion -- a
|
|
657
|
+
cap applied per level would pass a 4 MB matrix of one-character
|
|
658
|
+
cells."""
|
|
659
|
+
if isinstance(value, sympy.Matrix):
|
|
660
|
+
return [[str(cell) for cell in row] for row in value.tolist()]
|
|
661
|
+
if isinstance(value, dict):
|
|
662
|
+
return {str(k): _rendered(v) for k, v in value.items()}
|
|
663
|
+
if isinstance(value, (list, tuple)):
|
|
664
|
+
return [_rendered(v) for v in value]
|
|
665
|
+
if isinstance(value, bool):
|
|
666
|
+
return value
|
|
667
|
+
return str(value)
|