matryoshka-rlm 0.2.6 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/README.md +170 -84
  2. package/dist/adapters/base.d.ts +3 -3
  3. package/dist/adapters/base.d.ts.map +1 -1
  4. package/dist/adapters/base.js +41 -22
  5. package/dist/adapters/base.js.map +1 -1
  6. package/dist/adapters/deepseek.d.ts.map +1 -1
  7. package/dist/adapters/deepseek.js +4 -1
  8. package/dist/adapters/deepseek.js.map +1 -1
  9. package/dist/adapters/nucleus.d.ts.map +1 -1
  10. package/dist/adapters/nucleus.js +142 -21
  11. package/dist/adapters/nucleus.js.map +1 -1
  12. package/dist/adapters/qwen-synthesis.d.ts.map +1 -1
  13. package/dist/adapters/qwen-synthesis.js +2 -0
  14. package/dist/adapters/qwen-synthesis.js.map +1 -1
  15. package/dist/adapters/qwen.d.ts.map +1 -1
  16. package/dist/adapters/qwen.js +21 -11
  17. package/dist/adapters/qwen.js.map +1 -1
  18. package/dist/config/grammar-config.d.ts +55 -0
  19. package/dist/config/grammar-config.d.ts.map +1 -0
  20. package/dist/config/grammar-config.js +140 -0
  21. package/dist/config/grammar-config.js.map +1 -0
  22. package/dist/config/index.d.ts +7 -0
  23. package/dist/config/index.d.ts.map +1 -0
  24. package/dist/config/index.js +7 -0
  25. package/dist/config/index.js.map +1 -0
  26. package/dist/config.d.ts.map +1 -1
  27. package/dist/config.js +78 -2
  28. package/dist/config.js.map +1 -1
  29. package/dist/constraints/verifier.d.ts.map +1 -1
  30. package/dist/constraints/verifier.js +153 -27
  31. package/dist/constraints/verifier.js.map +1 -1
  32. package/dist/engine/handle-session.d.ts +59 -0
  33. package/dist/engine/handle-session.d.ts.map +1 -1
  34. package/dist/engine/handle-session.js +171 -9
  35. package/dist/engine/handle-session.js.map +1 -1
  36. package/dist/engine/nucleus-engine.d.ts +6 -0
  37. package/dist/engine/nucleus-engine.d.ts.map +1 -1
  38. package/dist/engine/nucleus-engine.js +92 -11
  39. package/dist/engine/nucleus-engine.js.map +1 -1
  40. package/dist/feedback/error-analyzer.d.ts.map +1 -1
  41. package/dist/feedback/error-analyzer.js +21 -2
  42. package/dist/feedback/error-analyzer.js.map +1 -1
  43. package/dist/fuzzy-search.d.ts +1 -1
  44. package/dist/fuzzy-search.d.ts.map +1 -1
  45. package/dist/fuzzy-search.js +1 -0
  46. package/dist/fuzzy-search.js.map +1 -1
  47. package/dist/index.js +24 -2
  48. package/dist/index.js.map +1 -1
  49. package/dist/lattice-mcp-server.js +72 -14
  50. package/dist/lattice-mcp-server.js.map +1 -1
  51. package/dist/llm/deepseek.d.ts.map +1 -1
  52. package/dist/llm/deepseek.js +20 -4
  53. package/dist/llm/deepseek.js.map +1 -1
  54. package/dist/llm/index.d.ts +1 -0
  55. package/dist/llm/index.d.ts.map +1 -1
  56. package/dist/llm/index.js +14 -7
  57. package/dist/llm/index.js.map +1 -1
  58. package/dist/llm/ollama.d.ts.map +1 -1
  59. package/dist/llm/ollama.js +15 -3
  60. package/dist/llm/ollama.js.map +1 -1
  61. package/dist/llm/retry.d.ts +21 -0
  62. package/dist/llm/retry.d.ts.map +1 -0
  63. package/dist/llm/retry.js +46 -0
  64. package/dist/llm/retry.js.map +1 -0
  65. package/dist/logic/constraint-resolver.d.ts.map +1 -1
  66. package/dist/logic/constraint-resolver.js +115 -19
  67. package/dist/logic/constraint-resolver.js.map +1 -1
  68. package/dist/logic/lc-compiler.d.ts.map +1 -1
  69. package/dist/logic/lc-compiler.js +54 -19
  70. package/dist/logic/lc-compiler.js.map +1 -1
  71. package/dist/logic/lc-interpreter.d.ts +5 -1
  72. package/dist/logic/lc-interpreter.d.ts.map +1 -1
  73. package/dist/logic/lc-interpreter.js +461 -71
  74. package/dist/logic/lc-interpreter.js.map +1 -1
  75. package/dist/logic/lc-parser.d.ts +0 -3
  76. package/dist/logic/lc-parser.d.ts.map +1 -1
  77. package/dist/logic/lc-parser.js +156 -25
  78. package/dist/logic/lc-parser.js.map +1 -1
  79. package/dist/logic/lc-solver.d.ts +7 -0
  80. package/dist/logic/lc-solver.d.ts.map +1 -1
  81. package/dist/logic/lc-solver.js +442 -119
  82. package/dist/logic/lc-solver.js.map +1 -1
  83. package/dist/logic/relational-solver.d.ts.map +1 -1
  84. package/dist/logic/relational-solver.js +106 -27
  85. package/dist/logic/relational-solver.js.map +1 -1
  86. package/dist/logic/synthesis-integrator.d.ts +6 -0
  87. package/dist/logic/synthesis-integrator.d.ts.map +1 -1
  88. package/dist/logic/synthesis-integrator.js +204 -49
  89. package/dist/logic/synthesis-integrator.js.map +1 -1
  90. package/dist/logic/type-inference.js +4 -4
  91. package/dist/logic/type-inference.js.map +1 -1
  92. package/dist/logic/types.d.ts +26 -2
  93. package/dist/logic/types.d.ts.map +1 -1
  94. package/dist/mcp-server.d.ts.map +1 -1
  95. package/dist/mcp-server.js +75 -1
  96. package/dist/mcp-server.js.map +1 -1
  97. package/dist/minikanren/common.d.ts +2 -2
  98. package/dist/minikanren/common.d.ts.map +1 -1
  99. package/dist/minikanren/common.js +18 -6
  100. package/dist/minikanren/common.js.map +1 -1
  101. package/dist/minikanren/reify.js +7 -4
  102. package/dist/minikanren/reify.js.map +1 -1
  103. package/dist/minikanren/streams.d.ts.map +1 -1
  104. package/dist/minikanren/streams.js +10 -1
  105. package/dist/minikanren/streams.js.map +1 -1
  106. package/dist/minikanren/sugar.d.ts +2 -2
  107. package/dist/minikanren/sugar.d.ts.map +1 -1
  108. package/dist/minikanren/sugar.js +38 -16
  109. package/dist/minikanren/sugar.js.map +1 -1
  110. package/dist/minikanren/unify.js +5 -2
  111. package/dist/minikanren/unify.js.map +1 -1
  112. package/dist/persistence/checkpoint.d.ts.map +1 -1
  113. package/dist/persistence/checkpoint.js +27 -4
  114. package/dist/persistence/checkpoint.js.map +1 -1
  115. package/dist/persistence/fts5-search.d.ts +1 -0
  116. package/dist/persistence/fts5-search.d.ts.map +1 -1
  117. package/dist/persistence/fts5-search.js +78 -22
  118. package/dist/persistence/fts5-search.js.map +1 -1
  119. package/dist/persistence/handle-ops.d.ts.map +1 -1
  120. package/dist/persistence/handle-ops.js +40 -10
  121. package/dist/persistence/handle-ops.js.map +1 -1
  122. package/dist/persistence/handle-registry.d.ts +12 -0
  123. package/dist/persistence/handle-registry.d.ts.map +1 -1
  124. package/dist/persistence/handle-registry.js +24 -19
  125. package/dist/persistence/handle-registry.js.map +1 -1
  126. package/dist/persistence/predicate-compiler.d.ts +12 -5
  127. package/dist/persistence/predicate-compiler.d.ts.map +1 -1
  128. package/dist/persistence/predicate-compiler.js +157 -50
  129. package/dist/persistence/predicate-compiler.js.map +1 -1
  130. package/dist/persistence/session-db.d.ts +45 -0
  131. package/dist/persistence/session-db.d.ts.map +1 -1
  132. package/dist/persistence/session-db.js +304 -22
  133. package/dist/persistence/session-db.js.map +1 -1
  134. package/dist/rag/manager.d.ts.map +1 -1
  135. package/dist/rag/manager.js +60 -16
  136. package/dist/rag/manager.js.map +1 -1
  137. package/dist/rag/similarity.d.ts +0 -9
  138. package/dist/rag/similarity.d.ts.map +1 -1
  139. package/dist/rag/similarity.js +47 -10
  140. package/dist/rag/similarity.js.map +1 -1
  141. package/dist/rlm.d.ts.map +1 -1
  142. package/dist/rlm.js +181 -161
  143. package/dist/rlm.js.map +1 -1
  144. package/dist/sandbox.d.ts.map +1 -1
  145. package/dist/sandbox.js +39 -9
  146. package/dist/sandbox.js.map +1 -1
  147. package/dist/session.d.ts +0 -14
  148. package/dist/session.d.ts.map +1 -1
  149. package/dist/session.js +15 -0
  150. package/dist/session.js.map +1 -1
  151. package/dist/synthesis/coordinator.d.ts.map +1 -1
  152. package/dist/synthesis/coordinator.js +74 -12
  153. package/dist/synthesis/coordinator.js.map +1 -1
  154. package/dist/synthesis/evalo/compile.d.ts +1 -11
  155. package/dist/synthesis/evalo/compile.d.ts.map +1 -1
  156. package/dist/synthesis/evalo/compile.js +64 -33
  157. package/dist/synthesis/evalo/compile.js.map +1 -1
  158. package/dist/synthesis/evalo/evalo.d.ts +2 -6
  159. package/dist/synthesis/evalo/evalo.d.ts.map +1 -1
  160. package/dist/synthesis/evalo/evalo.js +74 -30
  161. package/dist/synthesis/evalo/evalo.js.map +1 -1
  162. package/dist/synthesis/evalo/typeo.d.ts +1 -1
  163. package/dist/synthesis/evalo/typeo.d.ts.map +1 -1
  164. package/dist/synthesis/evalo/typeo.js +31 -4
  165. package/dist/synthesis/evalo/typeo.js.map +1 -1
  166. package/dist/synthesis/evolutionary.d.ts +0 -4
  167. package/dist/synthesis/evolutionary.d.ts.map +1 -1
  168. package/dist/synthesis/evolutionary.js +69 -20
  169. package/dist/synthesis/evolutionary.js.map +1 -1
  170. package/dist/synthesis/extractor/synthesis.d.ts.map +1 -1
  171. package/dist/synthesis/extractor/synthesis.js +65 -36
  172. package/dist/synthesis/extractor/synthesis.js.map +1 -1
  173. package/dist/synthesis/knowledge-base.d.ts +9 -0
  174. package/dist/synthesis/knowledge-base.d.ts.map +1 -1
  175. package/dist/synthesis/knowledge-base.js +99 -5
  176. package/dist/synthesis/knowledge-base.js.map +1 -1
  177. package/dist/synthesis/regex/synthesis.d.ts +1 -1
  178. package/dist/synthesis/regex/synthesis.d.ts.map +1 -1
  179. package/dist/synthesis/regex/synthesis.js +61 -18
  180. package/dist/synthesis/regex/synthesis.js.map +1 -1
  181. package/dist/synthesis/relational/interpreter.d.ts +1 -4
  182. package/dist/synthesis/relational/interpreter.d.ts.map +1 -1
  183. package/dist/synthesis/relational/interpreter.js +74 -20
  184. package/dist/synthesis/relational/interpreter.js.map +1 -1
  185. package/dist/synthesis/sandbox-tools.d.ts.map +1 -1
  186. package/dist/synthesis/sandbox-tools.js +102 -22
  187. package/dist/synthesis/sandbox-tools.js.map +1 -1
  188. package/dist/tool/adapters/claude-code.d.ts.map +1 -1
  189. package/dist/tool/adapters/claude-code.js +9 -1
  190. package/dist/tool/adapters/claude-code.js.map +1 -1
  191. package/dist/tool/adapters/http.d.ts +6 -1
  192. package/dist/tool/adapters/http.d.ts.map +1 -1
  193. package/dist/tool/adapters/http.js +136 -30
  194. package/dist/tool/adapters/http.js.map +1 -1
  195. package/dist/tool/adapters/pipe.d.ts +2 -0
  196. package/dist/tool/adapters/pipe.d.ts.map +1 -1
  197. package/dist/tool/adapters/pipe.js +74 -19
  198. package/dist/tool/adapters/pipe.js.map +1 -1
  199. package/dist/tool/ai-tools.d.ts +96 -0
  200. package/dist/tool/ai-tools.d.ts.map +1 -0
  201. package/dist/tool/ai-tools.js +210 -0
  202. package/dist/tool/ai-tools.js.map +1 -0
  203. package/dist/tool/index.d.ts +1 -0
  204. package/dist/tool/index.d.ts.map +1 -1
  205. package/dist/tool/index.js +2 -0
  206. package/dist/tool/index.js.map +1 -1
  207. package/dist/tool/lattice-tool.d.ts.map +1 -1
  208. package/dist/tool/lattice-tool.js +55 -12
  209. package/dist/tool/lattice-tool.js.map +1 -1
  210. package/dist/treesitter/builtin-grammars.d.ts +37 -0
  211. package/dist/treesitter/builtin-grammars.d.ts.map +1 -0
  212. package/dist/treesitter/builtin-grammars.js +313 -0
  213. package/dist/treesitter/builtin-grammars.js.map +1 -0
  214. package/dist/treesitter/index.d.ts +14 -0
  215. package/dist/treesitter/index.d.ts.map +1 -0
  216. package/dist/treesitter/index.js +14 -0
  217. package/dist/treesitter/index.js.map +1 -0
  218. package/dist/treesitter/language-map.d.ts +46 -0
  219. package/dist/treesitter/language-map.d.ts.map +1 -0
  220. package/dist/treesitter/language-map.js +165 -0
  221. package/dist/treesitter/language-map.js.map +1 -0
  222. package/dist/treesitter/parser-registry.d.ts +71 -0
  223. package/dist/treesitter/parser-registry.d.ts.map +1 -0
  224. package/dist/treesitter/parser-registry.js +172 -0
  225. package/dist/treesitter/parser-registry.js.map +1 -0
  226. package/dist/treesitter/symbol-extractor.d.ts +45 -0
  227. package/dist/treesitter/symbol-extractor.d.ts.map +1 -0
  228. package/dist/treesitter/symbol-extractor.js +325 -0
  229. package/dist/treesitter/symbol-extractor.js.map +1 -0
  230. package/dist/treesitter/types.d.ts +63 -0
  231. package/dist/treesitter/types.d.ts.map +1 -0
  232. package/dist/treesitter/types.js +5 -0
  233. package/dist/treesitter/types.js.map +1 -0
  234. package/package.json +12 -2
package/README.md CHANGED
@@ -1,4 +1,6 @@
1
- # Recursive Language Model (RLM)
1
+ # Matryoshka
2
+
3
+ [![Tests](https://github.com/yogthos/Matryoshka/actions/workflows/test.yml/badge.svg)](https://github.com/yogthos/Matryoshka/actions/workflows/test.yml)
2
4
 
3
5
  Process documents 100x larger than your LLM's context window—without vector databases or chunking heuristics.
4
6
 
@@ -63,9 +65,9 @@ The Lattice engine (`src/logic/`) processes Nucleus commands:
63
65
 
64
66
  Lattice uses **miniKanren** (a relational programming engine) for pattern classification and filtering operations.
65
67
 
66
- ### SQLite Handle-Based Persistence
68
+ ### In-Memory Handle Storage
67
69
 
68
- For large result sets, RLM uses a handle-based architecture (`src/persistence/`) that achieves **97%+ token savings**:
70
+ For large result sets, RLM uses a handle-based architecture with in-memory SQLite (`src/persistence/`) that achieves **97%+ token savings**:
69
71
 
70
72
  ```
71
73
  Traditional: LLM sees full array [15,000 tokens for 1000 results]
@@ -85,29 +87,6 @@ Handle-based: LLM sees stub [50 tokens: "$res1: Array(1000) [preview...
85
87
  - `FTS5Search` - Phrase queries, boolean operators, relevance ranking
86
88
  - `CheckpointManager` - Save/restore session state
87
89
 
88
- ### Pre-Search Optimization
89
-
90
- Before calling the LLM, the system extracts keywords from your query and pre-runs grep:
91
-
92
- ```
93
- Query: "What is the total of all north sales data values?"
94
-
95
-
96
- ┌─────────────────────────────────────────────────────┐
97
- │ Pre-search extracts: "north", "sales", "data" │
98
- │ Tries compound patterns: SALES.*NORTH, NORTH.*SALES │
99
- │ Pre-populates RESULTS before LLM is called │
100
- └─────────────────────────────────────────────────────┘
101
-
102
-
103
- ┌─────────────────────────────────────────────────────┐
104
- │ LLM receives: "RESULTS has 1 match" │
105
- │ LLM outputs: (sum RESULTS) ← skips search step! │
106
- └─────────────────────────────────────────────────────┘
107
- ```
108
-
109
- This saves turns by pre-populating `RESULTS` so the model can immediately aggregate.
110
-
111
90
  ### The Role of the LLM
112
91
 
113
92
  The LLM does **reasoning**, not code generation:
@@ -126,9 +105,8 @@ The LLM never writes JavaScript. It outputs Nucleus commands that Lattice execut
126
105
  | **Nucleus Adapter** | Prompts LLM to output Nucleus commands |
127
106
  | **Lattice Parser** | Parses S-expressions to AST |
128
107
  | **Lattice Solver** | Executes commands against document |
129
- | **SQLite Persistence** | Handle-based storage with FTS5 (97% token savings) |
108
+ | **In-Memory Handles** | Handle-based storage with FTS5 (97% token savings) |
130
109
  | **miniKanren** | Relational engine for classification |
131
- | **Pre-Search** | Extracts keywords and pre-runs grep |
132
110
  | **RAG Hints** | Few-shot examples from past successes |
133
111
 
134
112
  ## Installation
@@ -152,7 +130,6 @@ The package provides several CLI tools:
152
130
  | Command | Description |
153
131
  |---------|-------------|
154
132
  | `rlm` | Main CLI for document analysis with LLM reasoning |
155
- | `rlm-mcp` | MCP server exposing `analyze_document` tool |
156
133
  | `lattice-mcp` | MCP server exposing direct Nucleus commands (no LLM required) |
157
134
  | `lattice-repl` | Interactive REPL for Nucleus commands |
158
135
  | `lattice-http` | HTTP server for Nucleus queries |
@@ -210,55 +187,24 @@ rlm --help
210
187
 
211
188
  ### MCP Integration
212
189
 
213
- RLM includes an MCP (Model Context Protocol) server that exposes the `analyze_document` tool. This allows coding agents to analyze documents that exceed their context window.
214
-
215
- #### MCP Tool: `analyze_document`
190
+ RLM includes `lattice-mcp`, an MCP (Model Context Protocol) server for direct access to the Nucleus engine. This allows coding agents to analyze documents with **80%+ token savings** compared to reading files directly.
216
191
 
217
- | Parameter | Type | Required | Description |
218
- |-----------|------|----------|-------------|
219
- | `query` | string | Yes | The question or task to perform on the document |
220
- | `filePath` | string | Yes | Absolute path to the document file |
221
- | `maxTurns` | number | No | Maximum exploration turns (default: 10) |
222
- | `timeoutMs` | number | No | Timeout per turn in milliseconds (default: 30000) |
192
+ The key advantage is **handle-based results**: query results are stored server-side in SQLite, and the agent receives compact stubs like `$res1: Array(1000) [preview...]` instead of full data. Operations chain server-side without roundtripping data.
223
193
 
224
- #### Example MCP config
225
-
226
- ```json
227
- {
228
- "mcp": {
229
- "rlm": {
230
- "type": "stdio",
231
- "command": "rlm-mcp"
232
- }
233
- }
234
- }
235
- ```
236
-
237
- #### Testing the MCP Server
238
-
239
- ```bash
240
- rlm-mcp --test
241
- # Output: MCP server ready
242
- # Output: Available tools: analyze_document
243
- ```
244
-
245
- ### Lattice MCP Server
246
-
247
- For direct access to the Nucleus engine without LLM orchestration, use `lattice-mcp`. This is useful when you want to run precise, programmatic queries with **80%+ token savings** compared to reading files directly.
248
-
249
- #### Lattice MCP Tools
194
+ #### Available Tools
250
195
 
251
196
  | Tool | Description |
252
197
  |------|-------------|
253
198
  | `lattice_load` | Load a document for analysis |
254
199
  | `lattice_query` | Execute Nucleus commands on the loaded document |
200
+ | `lattice_expand` | Expand a handle to see full data (with optional limit/offset) |
255
201
  | `lattice_close` | Close the session and free memory |
256
202
  | `lattice_status` | Get session status and document info |
257
203
  | `lattice_bindings` | Show current variable bindings |
258
204
  | `lattice_reset` | Reset bindings but keep document loaded |
259
205
  | `lattice_help` | Get Nucleus command reference |
260
206
 
261
- #### Example Lattice MCP config
207
+ #### Example MCP config
262
208
 
263
209
  ```json
264
210
  {
@@ -275,17 +221,19 @@ For direct access to the Nucleus engine without LLM orchestration, use `lattice-
275
221
 
276
222
  ```
277
223
  1. lattice_load("/path/to/large-file.txt") # Load document (use for >500 lines)
278
- 2. lattice_query('(grep "ERROR")') # Search - shows preview of first 20
279
- 3. lattice_query('(filter RESULTS ...)') # Narrow down - updates RESULTS
280
- 4. lattice_query('(count RESULTS)') # Get count without listing all
281
- 5. lattice_close() # Free memory when done
224
+ 2. lattice_query('(grep "ERROR")') # Search - returns handle stub $res1
225
+ 3. lattice_query('(filter RESULTS ...)') # Narrow down - returns handle stub $res2
226
+ 4. lattice_query('(count RESULTS)') # Get count without seeing data
227
+ 5. lattice_expand("$res2", limit=10) # Expand only what you need to see
228
+ 6. lattice_close() # Free memory when done
282
229
  ```
283
230
 
284
231
  **Token efficiency tips:**
285
- - Use `(count RESULTS)` instead of viewing all results
232
+ - Query results return handle stubs, not full data
233
+ - Use `lattice_expand` with `limit` to see only what you need
286
234
  - Chain `grep → filter → count/sum` to refine progressively
287
- - Results show preview (first 20), use filter to narrow down
288
- - Previous results available as `_1`, `_2`, etc.
235
+ - Use `RESULTS` in queries (always points to last result)
236
+ - Use `$res1`, `$res2` etc. with `lattice_expand` to inspect specific results
289
237
 
290
238
  ### Programmatic
291
239
 
@@ -311,27 +259,29 @@ const result = await runRLM("What is the total of all sales values?", "./report.
311
259
  ```
312
260
  $ rlm "What is the total of all north sales data values?" ./report.txt --verbose
313
261
 
314
- [Pre-search] Found 1 data matches for "SALES.*NORTH"
315
- [Pre-search] RESULTS pre-populated with 1 matches
316
-
317
262
  ──────────────────────────────────────────────────
318
263
  [Turn 1/10] Querying LLM...
319
- [Turn 1] Term: (sum RESULTS)
320
- [Turn 1] Console output:
264
+ [Turn 1] Term: (grep "SALES.*NORTH")
265
+ [Turn 1] Result: 1 matches
266
+
267
+ ──────────────────────────────────────────────────
268
+ [Turn 2/10] Querying LLM...
269
+ [Turn 2] Term: (sum RESULTS)
270
+ [Turn 2] Console output:
321
271
  [Lattice] Summing 1 values
322
272
  [Lattice] Sum = 2340000
323
- [Turn 1] Result: 2340000
273
+ [Turn 2] Result: 2340000
324
274
 
325
275
  ──────────────────────────────────────────────────
326
- [Turn 2/10] Querying LLM...
327
- [Turn 2] Final answer received
276
+ [Turn 3/10] Querying LLM...
277
+ [Turn 3] Final answer received
328
278
 
329
279
  2340000
330
280
  ```
331
281
 
332
282
  The model:
333
- 1. Received pre-populated RESULTS (pre-search found the data)
334
- 2. Immediately summed the results (no grep needed)
283
+ 1. Searched for relevant data with grep
284
+ 2. Summed the matching results
335
285
  3. Output the final answer
336
286
 
337
287
  ## Nucleus DSL Reference
@@ -344,6 +294,135 @@ The model:
344
294
  (text_stats) ; Document metadata (length, line count, samples)
345
295
  ```
346
296
 
297
+ ### Symbol Operations (Code Files)
298
+
299
+ For code files, Lattice uses tree-sitter to extract structural symbols. This enables code-aware queries that understand functions, classes, methods, and other language constructs.
300
+
301
+ **Built-in languages (packages included):**
302
+ - TypeScript (.ts, .tsx), JavaScript (.js, .jsx), Python (.py), Go (.go)
303
+ - HTML (.html), CSS (.css), JSON (.json)
304
+
305
+ **Additional languages (install package to enable):**
306
+ - Rust, C, C++, Java, Ruby, PHP, C#, Kotlin, Swift, Scala, Lua, Haskell, Bash, SQL, and more
307
+
308
+ ```scheme
309
+ (list_symbols) ; List all symbols (functions, classes, methods, etc.)
310
+ (list_symbols "function") ; Filter by kind: "function", "class", "method", "interface", "type", "struct"
311
+ (get_symbol_body "myFunc") ; Get source code body for a symbol by name
312
+ (get_symbol_body RESULTS) ; Get body for symbol from previous query result
313
+ (find_references "myFunc") ; Find all references to an identifier
314
+ ```
315
+
316
+ **Example workflow for code analysis:**
317
+
318
+ ```
319
+ 1. lattice_load("./src/app.ts") # Load a code file
320
+ 2. lattice_query('(list_symbols)') # Get all symbols → $res1
321
+ 3. lattice_query('(list_symbols "function")') # Just functions → $res2
322
+ 4. lattice_expand("$res2", limit=5) # See function names and line numbers
323
+ 5. lattice_query('(get_symbol_body "handleRequest")') # Get function body
324
+ 6. lattice_query('(find_references "handleRequest")') # Find all usages
325
+ ```
326
+
327
+ Symbols include metadata like name, kind, start/end lines, and parent relationships (e.g., methods within classes).
328
+
329
+ #### Adding Language Support
330
+
331
+ Matryoshka includes built-in symbol mappings for 20+ languages. To enable a language, install its tree-sitter grammar package:
332
+
333
+ ```bash
334
+ # Enable Rust support
335
+ npm install tree-sitter-rust
336
+
337
+ # Enable Java support
338
+ npm install tree-sitter-java
339
+
340
+ # Enable Ruby support
341
+ npm install tree-sitter-ruby
342
+ ```
343
+
344
+ **Languages with built-in mappings:**
345
+ - TypeScript, JavaScript, Python, Go, Rust, C, C++, Java
346
+ - Ruby, PHP, C#, Kotlin, Swift, Scala, Lua, Haskell, Elixir
347
+ - HTML, CSS, JSON, YAML, TOML, Markdown, SQL, Bash
348
+
349
+ Once a package is installed, the language is automatically available for symbol extraction.
350
+
351
+ #### Custom Language Configuration
352
+
353
+ For languages without built-in mappings, or to override existing mappings, create a config file at `~/.matryoshka/config.json`:
354
+
355
+ ```json
356
+ {
357
+ "grammars": {
358
+ "mylang": {
359
+ "package": "tree-sitter-mylang",
360
+ "extensions": [".ml", ".mli"],
361
+ "moduleExport": "mylang",
362
+ "symbols": {
363
+ "function_definition": "function",
364
+ "method_definition": "method",
365
+ "class_definition": "class",
366
+ "module_definition": "module"
367
+ }
368
+ }
369
+ }
370
+ }
371
+ ```
372
+
373
+ **Configuration fields:**
374
+
375
+ | Field | Required | Description |
376
+ |-------|----------|-------------|
377
+ | `package` | Yes | npm package name for the tree-sitter grammar |
378
+ | `extensions` | Yes | File extensions to associate with this language |
379
+ | `symbols` | Yes | Maps tree-sitter node types to symbol kinds |
380
+ | `moduleExport` | No | Submodule export name (e.g., `"typescript"` for tree-sitter-typescript) |
381
+
382
+ **Symbol kinds:** `function`, `method`, `class`, `interface`, `type`, `struct`, `enum`, `trait`, `module`, `variable`, `constant`, `property`
383
+
384
+ #### Finding Tree-sitter Node Types
385
+
386
+ To configure symbol mappings for a new language, you need to know the tree-sitter node types. You can explore them using the tree-sitter CLI:
387
+
388
+ ```bash
389
+ # Install tree-sitter CLI
390
+ npm install -g tree-sitter-cli
391
+
392
+ # Parse a sample file and see the AST
393
+ tree-sitter parse sample.mylang
394
+ ```
395
+
396
+ Or use the [tree-sitter playground](https://tree-sitter.github.io/tree-sitter/playground) to explore node types interactively.
397
+
398
+ **Example: Adding OCaml support**
399
+
400
+ 1. Find the grammar package: `tree-sitter-ocaml`
401
+ 2. Install it: `npm install tree-sitter-ocaml`
402
+ 3. Explore the AST to find node types for functions, modules, etc.
403
+ 4. Add to `~/.matryoshka/config.json`:
404
+
405
+ ```json
406
+ {
407
+ "grammars": {
408
+ "ocaml": {
409
+ "package": "tree-sitter-ocaml",
410
+ "extensions": [".ml", ".mli"],
411
+ "moduleExport": "ocaml",
412
+ "symbols": {
413
+ "value_definition": "function",
414
+ "let_binding": "variable",
415
+ "type_definition": "type",
416
+ "module_definition": "module",
417
+ "module_type_definition": "interface"
418
+ }
419
+ }
420
+ }
421
+ }
422
+ ```
423
+
424
+ **Note:** Some tree-sitter packages use native Node.js bindings that may not compile on all systems. If installation fails, check if the package supports your Node.js version or look for WASM alternatives.
425
+
347
426
  ### Collection Operations
348
427
 
349
428
  ```scheme
@@ -479,14 +558,20 @@ src/
479
558
  │ ├── lc-solver.ts # Command executor (uses miniKanren)
480
559
  │ ├── type-inference.ts
481
560
  │ └── constraint-resolver.ts
482
- ├── persistence/ # SQLite handle-based storage (97% token savings)
561
+ ├── persistence/ # In-memory handle storage (97% token savings)
483
562
  │ ├── session-db.ts # In-memory SQLite with FTS5
484
563
  │ ├── handle-registry.ts # Handle creation and stubs
485
564
  │ ├── handle-ops.ts # Server-side operations
486
565
  │ ├── fts5-search.ts # Full-text search
487
566
  │ └── checkpoint.ts # Session persistence
567
+ ├── treesitter/ # Code-aware symbol extraction
568
+ │ ├── parser-registry.ts # Tree-sitter parser management
569
+ │ ├── symbol-extractor.ts # AST → symbol extraction
570
+ │ ├── language-map.ts # Extension → language mapping
571
+ │ └── types.ts # Symbol interfaces
488
572
  ├── engine/ # Nucleus execution engine
489
- └── nucleus-engine.ts
573
+ ├── nucleus-engine.ts
574
+ │ └── handle-session.ts # Session with symbol support
490
575
  ├── minikanren/ # Relational programming engine
491
576
  ├── synthesis/ # Program synthesis (Barliman-style)
492
577
  │ └── evalo/ # Extractor DSL
@@ -501,6 +586,7 @@ This project incorporates ideas and code from:
501
586
  - **[Nucleus](https://github.com/michaelwhitford/nucleus)** - A symbolic S-expression language by Michael Whitford. RLM uses Nucleus syntax for the constrained DSL that the LLM outputs, providing a rigid grammar that reduces model errors.
502
587
  - **[ramo](https://github.com/wjlewis/ramo)** - A miniKanren implementation in TypeScript by Will Lewis. Used for constraint-based program synthesis.
503
588
  - **[Barliman](https://github.com/webyrd/Barliman)** - A prototype smart editor by William Byrd and Greg Rosenblatt that uses program synthesis to assist programmers. The Barliman-style approach of providing input/output constraints instead of code inspired the synthesis workflow.
589
+ - **[tree-sitter](https://tree-sitter.github.io/tree-sitter/)** - A parser generator tool and incremental parsing library. Used for extracting structural symbols (functions, classes, methods) from code files to enable code-aware queries.
504
590
 
505
591
  ## License
506
592
 
@@ -24,17 +24,17 @@ declare function getNoCodeFeedback(): string;
24
24
  /**
25
25
  * Get feedback message when code execution fails
26
26
  */
27
- declare function getErrorFeedback(error: string): string;
27
+ declare function getErrorFeedback(error: string, code?: string): string;
28
28
  /**
29
29
  * Get feedback message after successful code execution
30
30
  * Generic reminder about continuing exploration or providing final answer
31
31
  */
32
- declare function getSuccessFeedback(): string;
32
+ declare function getSuccessFeedback(resultCount?: number, previousCount?: number, query?: string): string;
33
33
  /**
34
34
  * Get feedback message when model repeats the same code
35
35
  * Encourages a different approach
36
36
  */
37
- declare function getRepeatedCodeFeedback(): string;
37
+ declare function getRepeatedCodeFeedback(resultCount?: number): string;
38
38
  /**
39
39
  * Create the base adapter instance
40
40
  */
@@ -1 +1 @@
1
- {"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/adapters/base.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEzE;;GAEG;AACH,iBAAS,iBAAiB,CACxB,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM,EACtB,KAAK,CAAC,EAAE,QAAQ,GACf,MAAM,CAyDR;AAED;;GAEG;AACH,iBAAS,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAUpD;AAED;;GAEG;AACH,iBAAS,kBAAkB,CACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAClC,MAAM,GAAG,cAAc,GAAG,IAAI,CA6ChC;AAED;;GAEG;AACH,iBAAS,iBAAiB,IAAI,MAAM,CAMnC;AAED;;GAEG;AACH,iBAAS,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED;;;GAGG;AACH,iBAAS,kBAAkB,IAAI,MAAM,CAEpC;AAED;;;GAGG;AACH,iBAAS,uBAAuB,IAAI,MAAM,CASzC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,YAAY,CAWhD;AAGD,OAAO,EACL,iBAAiB,IAAI,qBAAqB,EAC1C,WAAW,IAAI,eAAe,EAC9B,kBAAkB,IAAI,sBAAsB,EAC5C,iBAAiB,IAAI,qBAAqB,EAC1C,gBAAgB,IAAI,oBAAoB,EACxC,kBAAkB,IAAI,sBAAsB,EAC5C,uBAAuB,IAAI,2BAA2B,GACvD,CAAC"}
1
+ {"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/adapters/base.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEzE;;GAEG;AACH,iBAAS,iBAAiB,CACxB,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM,EACtB,KAAK,CAAC,EAAE,QAAQ,GACf,MAAM,CA0DR;AAED;;GAEG;AACH,iBAAS,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAapD;AAED;;GAEG;AACH,iBAAS,kBAAkB,CACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAClC,MAAM,GAAG,cAAc,GAAG,IAAI,CA8ChC;AAED;;GAEG;AACH,iBAAS,iBAAiB,IAAI,MAAM,CAMnC;AAED;;GAEG;AACH,iBAAS,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAO9D;AAED;;;GAGG;AACH,iBAAS,kBAAkB,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAQhG;AAED;;;GAGG;AACH,iBAAS,uBAAuB,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAS7D;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,YAAY,CAWhD;AAGD,OAAO,EACL,iBAAiB,IAAI,qBAAqB,EAC1C,WAAW,IAAI,eAAe,EAC9B,kBAAkB,IAAI,sBAAsB,EAC5C,iBAAiB,IAAI,qBAAqB,EAC1C,gBAAgB,IAAI,oBAAoB,EACxC,kBAAkB,IAAI,sBAAsB,EAC5C,uBAAuB,IAAI,2BAA2B,GACvD,CAAC"}
@@ -8,6 +8,8 @@
8
8
  * Build the default system prompt for the RLM
9
9
  */
10
10
  function buildSystemPrompt(contextLength, toolInterfaces, hints) {
11
+ if (!Number.isFinite(contextLength) || contextLength < 0)
12
+ contextLength = 0;
11
13
  const formattedLength = contextLength.toLocaleString();
12
14
  return `You are a headless JavaScript runtime. You have NO EYES. You cannot read the document directly.
13
15
  The document is loaded in the global variable \`context\` (length: ${formattedLength}).
@@ -69,12 +71,17 @@ Reminder: You are blind. Write code to see.`;
69
71
  */
70
72
  function extractCode(response) {
71
73
  // Match typescript, ts, javascript, or js code blocks
72
- const codeBlockRegex = /```(?:typescript|ts|javascript|js)\n([\s\S]*?)```/;
73
- const match = response.match(codeBlockRegex);
74
- if (match && match[1]) {
75
- return match[1].trim();
76
- }
77
- return null;
74
+ // Use indexOf-based approach to avoid ReDoS with [\s\S]*? on unclosed fences
75
+ const openPattern = /```(?:typescript|ts|javascript|js)\n/;
76
+ const openMatch = response.match(openPattern);
77
+ if (!openMatch || openMatch.index === undefined)
78
+ return null;
79
+ const codeStart = openMatch.index + openMatch[0].length;
80
+ const closeIdx = response.indexOf("```", codeStart);
81
+ if (closeIdx === -1)
82
+ return null;
83
+ const code = response.slice(codeStart, closeIdx).trim();
84
+ return code || null;
78
85
  }
79
86
  /**
80
87
  * Extract final answer from LLM response
@@ -84,8 +91,9 @@ function extractFinalAnswer(response) {
84
91
  return null;
85
92
  }
86
93
  // Check for FINAL_VAR(variableName)
94
+ const DANGEROUS_VAR_NAMES = /^(__proto__|constructor|prototype|eval|Function|__defineGetter__|__defineSetter__|__lookupGetter__|__lookupSetter__|hasOwnProperty|toString|valueOf|toLocaleString|isPrototypeOf|propertyIsEnumerable)$/i;
87
95
  const varMatch = response.match(/FINAL_VAR\((\w+)\)/);
88
- if (varMatch) {
96
+ if (varMatch && !DANGEROUS_VAR_NAMES.test(varMatch[1])) {
89
97
  return { type: "var", name: varMatch[1] };
90
98
  }
91
99
  // Check for <<<FINAL>>>...<<<END>>> delimiters
@@ -98,24 +106,24 @@ function extractFinalAnswer(response) {
98
106
  if (jsonMatch) {
99
107
  try {
100
108
  const parsed = JSON.parse(jsonMatch[1]);
101
- // Check for common answer field names
102
- if (parsed.summary)
109
+ // Check for common answer field names (must be strings)
110
+ if (typeof parsed.summary === "string")
103
111
  return parsed.summary;
104
- if (parsed.response)
112
+ if (typeof parsed.response === "string")
105
113
  return parsed.response;
106
- if (parsed.answer)
114
+ if (typeof parsed.answer === "string")
107
115
  return parsed.answer;
108
116
  // Check for any field that looks like a final value (case-insensitive)
109
117
  const valueFields = ['total', 'result', 'value', 'totalsales', 'total_sales', 'count', 'sum', 'answer', 'totals'];
110
- const keys = Object.keys(parsed);
118
+ const MAX_KEYS = 100;
119
+ const keys = Object.keys(parsed).slice(0, MAX_KEYS);
111
120
  const foundKey = keys.find(k => valueFields.includes(k.toLowerCase().replace(/_/g, '')));
112
- const foundValue = foundKey;
113
- if (foundValue !== undefined) {
114
- const value = parsed[foundValue];
115
- if (parsed.notes) {
121
+ if (foundKey !== undefined) {
122
+ const value = parsed[foundKey];
123
+ if (typeof parsed.notes === "string") {
116
124
  return `${parsed.notes}\n\nResult: ${typeof value === 'number' ? value.toLocaleString() : value}`;
117
125
  }
118
- return JSON.stringify(parsed, null, 2);
126
+ return JSON.stringify(parsed, null, 2).slice(0, 50_000);
119
127
  }
120
128
  }
121
129
  catch {
@@ -137,22 +145,33 @@ console.log(JSON.stringify(hits, null, 2));
137
145
  /**
138
146
  * Get feedback message when code execution fails
139
147
  */
140
- function getErrorFeedback(error) {
141
- return `The previous code had an error: ${error}\nFix the code and try again.`;
148
+ function getErrorFeedback(error, code) {
149
+ const safeError = error.slice(0, 500);
150
+ let feedback = `The previous code had an error: ${safeError}\nFix the code and try again.`;
151
+ if (code) {
152
+ feedback += `\nCode that failed: ${code.slice(0, 200)}`;
153
+ }
154
+ return feedback;
142
155
  }
143
156
  /**
144
157
  * Get feedback message after successful code execution
145
158
  * Generic reminder about continuing exploration or providing final answer
146
159
  */
147
- function getSuccessFeedback() {
160
+ function getSuccessFeedback(resultCount, previousCount, query) {
161
+ if (resultCount === 0) {
162
+ return `No results found. Try different search terms or approach.`;
163
+ }
164
+ if (resultCount !== undefined && resultCount > 0) {
165
+ return `Found ${resultCount} results. Continue exploring, OR output final answer using <<<FINAL>>> and <<<END>>> tags.`;
166
+ }
148
167
  return `Variables persist between turns. Continue exploring, OR output final answer using <<<FINAL>>> and <<<END>>> tags.`;
149
168
  }
150
169
  /**
151
170
  * Get feedback message when model repeats the same code
152
171
  * Encourages a different approach
153
172
  */
154
- function getRepeatedCodeFeedback() {
155
- return `ERROR: You are repeating the same code. This will give the same output.
173
+ function getRepeatedCodeFeedback(resultCount) {
174
+ return `ERROR: You are repeating the same code. This will give the same output.${resultCount !== undefined ? ` (${resultCount} results already available)` : ""}
156
175
 
157
176
  Try a DIFFERENT approach:
158
177
  - Use different search terms with grep()
@@ -1 +1 @@
1
- {"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/adapters/base.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;GAEG;AACH,SAAS,iBAAiB,CACxB,aAAqB,EACrB,cAAsB,EACtB,KAAgB;IAEhB,MAAM,eAAe,GAAG,aAAa,CAAC,cAAc,EAAE,CAAC;IAEvD,OAAO;qEAC4D,eAAe;;;;;;EAMlF,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2Cd,KAAK,EAAE,SAAS,IAAI,EAAE,GAAG,KAAK,EAAE,kBAAkB,IAAI,EAAE;;;4CAGd,CAAC;AAC7C,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,QAAgB;IACnC,sDAAsD;IACtD,MAAM,cAAc,GAAG,mDAAmD,CAAC;IAC3E,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAE7C,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,QAAmC;IAEnC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oCAAoC;IACpC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACtD,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;IAED,+CAA+C;IAC/C,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpE,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;IAED,6FAA6F;IAC7F,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC5D,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,sCAAsC;YACtC,IAAI,MAAM,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC,OAAO,CAAC;YAC1C,IAAI,MAAM,CAAC,QAAQ;gBAAE,OAAO,MAAM,CAAC,QAAQ,CAAC;YAC5C,IAAI,MAAM,CAAC,MAAM;gBAAE,OAAO,MAAM,CAAC,MAAM,CAAC;YACxC,uEAAuE;YACvE,MAAM,WAAW,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAClH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YACzF,MAAM,UAAU,GAAG,QAAQ,CAAC;YAE5B,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;gBACjC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBACjB,OAAO,GAAG,MAAM,CAAC,KAAK,eAAe,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;gBACpG,CAAC;gBACD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB;IACxB,OAAO;;;;OAIF,CAAC;AACR,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,KAAa;IACrC,OAAO,mCAAmC,KAAK,+BAA+B,CAAC;AACjF,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB;IACzB,OAAO,mHAAmH,CAAC;AAC7H,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB;IAC9B,OAAO;;;;;;;oBAOW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,iBAAiB;QACjB,WAAW;QACX,kBAAkB;QAClB,iBAAiB;QACjB,gBAAgB;QAChB,kBAAkB;QAClB,uBAAuB;KACxB,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,OAAO,EACL,iBAAiB,IAAI,qBAAqB,EAC1C,WAAW,IAAI,eAAe,EAC9B,kBAAkB,IAAI,sBAAsB,EAC5C,iBAAiB,IAAI,qBAAqB,EAC1C,gBAAgB,IAAI,oBAAoB,EACxC,kBAAkB,IAAI,sBAAsB,EAC5C,uBAAuB,IAAI,2BAA2B,GACvD,CAAC"}
1
+ {"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/adapters/base.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;GAEG;AACH,SAAS,iBAAiB,CACxB,aAAqB,EACrB,cAAsB,EACtB,KAAgB;IAEhB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,CAAC;QAAE,aAAa,GAAG,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,aAAa,CAAC,cAAc,EAAE,CAAC;IAEvD,OAAO;qEAC4D,eAAe;;;;;;EAMlF,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2Cd,KAAK,EAAE,SAAS,IAAI,EAAE,GAAG,KAAK,EAAE,kBAAkB,IAAI,EAAE;;;4CAGd,CAAC;AAC7C,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,QAAgB;IACnC,sDAAsD;IACtD,6EAA6E;IAC7E,MAAM,WAAW,GAAG,sCAAsC,CAAC;IAC3D,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC9C,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAE7D,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACxD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACpD,IAAI,QAAQ,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAEjC,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;IACxD,OAAO,IAAI,IAAI,IAAI,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,QAAmC;IAEnC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oCAAoC;IACpC,MAAM,mBAAmB,GAAG,0MAA0M,CAAC;IACvO,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACtD,IAAI,QAAQ,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;IAED,+CAA+C;IAC/C,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpE,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;IAED,6FAA6F;IAC7F,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC5D,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,wDAAwD;YACxD,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ;gBAAE,OAAO,MAAM,CAAC,OAAO,CAAC;YAC9D,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO,MAAM,CAAC,QAAQ,CAAC;YAChE,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ;gBAAE,OAAO,MAAM,CAAC,MAAM,CAAC;YAC5D,uEAAuE;YACvE,MAAM,WAAW,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAClH,MAAM,QAAQ,GAAG,GAAG,CAAC;YACrB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;YACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAEzF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC/B,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;oBACrC,OAAO,GAAG,MAAM,CAAC,KAAK,eAAe,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;gBACpG,CAAC;gBACD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB;IACxB,OAAO;;;;OAIF,CAAC;AACR,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAAa;IACpD,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtC,IAAI,QAAQ,GAAG,mCAAmC,SAAS,+BAA+B,CAAC;IAC3F,IAAI,IAAI,EAAE,CAAC;QACT,QAAQ,IAAI,uBAAuB,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAC1D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB,CAAC,WAAoB,EAAE,aAAsB,EAAE,KAAc;IACtF,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,2DAA2D,CAAC;IACrE,CAAC;IACD,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACjD,OAAO,SAAS,WAAW,4FAA4F,CAAC;IAC1H,CAAC;IACD,OAAO,mHAAmH,CAAC;AAC7H,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB,CAAC,WAAoB;IACnD,OAAO,0EAA0E,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,WAAW,6BAA6B,CAAC,CAAC,CAAC,EAAE;;;;;;;oBAO7I,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB;IAC/B,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,iBAAiB;QACjB,WAAW;QACX,kBAAkB;QAClB,iBAAiB;QACjB,gBAAgB;QAChB,kBAAkB;QAClB,uBAAuB;KACxB,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,OAAO,EACL,iBAAiB,IAAI,qBAAqB,EAC1C,WAAW,IAAI,eAAe,EAC9B,kBAAkB,IAAI,sBAAsB,EAC5C,iBAAiB,IAAI,qBAAqB,EAC1C,gBAAgB,IAAI,oBAAoB,EACxC,kBAAkB,IAAI,sBAAsB,EAC5C,uBAAuB,IAAI,2BAA2B,GACvD,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"deepseek.d.ts","sourceRoot":"","sources":["../../src/adapters/deepseek.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAA4B,MAAM,YAAY,CAAC;AAsIzE;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,YAAY,CAWpD"}
1
+ {"version":3,"file":"deepseek.d.ts","sourceRoot":"","sources":["../../src/adapters/deepseek.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAA4B,MAAM,YAAY,CAAC;AAwIzE;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,YAAY,CAWpD"}
@@ -12,6 +12,8 @@ import { baseExtractCode, baseExtractFinalAnswer, } from "./base.js";
12
12
  * Uses clear structure and explicit role definition
13
13
  */
14
14
  function buildSystemPrompt(contextLength, toolInterfaces, hints) {
15
+ if (!Number.isFinite(contextLength) || contextLength < 0)
16
+ contextLength = 0;
15
17
  const formattedLength = contextLength.toLocaleString();
16
18
  return `# Role
17
19
  You are a JavaScript runtime that executes code to analyze documents. You cannot read documents directly - you must write code.
@@ -92,7 +94,8 @@ Provide JavaScript code to continue analysis.`;
92
94
  * Error feedback for DeepSeek
93
95
  */
94
96
  function getErrorFeedback(error) {
95
- return `Execution error: ${error}
97
+ const safeError = error.slice(0, 500);
98
+ return `Execution error: ${safeError}
96
99
 
97
100
  Please fix the code and provide a corrected version:
98
101
  \`\`\`javascript
@@ -1 +1 @@
1
- {"version":3,"file":"deepseek.js","sourceRoot":"","sources":["../../src/adapters/deepseek.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EACL,eAAe,EACf,sBAAsB,GACvB,MAAM,WAAW,CAAC;AAEnB;;;GAGG;AACH,SAAS,iBAAiB,CACxB,aAAqB,EACrB,cAAsB,EACtB,KAAgB;IAEhB,MAAM,eAAe,GAAG,aAAa,CAAC,cAAc,EAAE,CAAC;IAEvD,OAAO;;;;oDAI2C,eAAe;;;;;;;EAOjE,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4Bd,KAAK,EAAE,SAAS,IAAI,EAAE,GAAG,KAAK,EAAE,kBAAkB,IAAI,EAAE;;;wCAGlB,CAAC;AACzC,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,QAAgB;IACnC,iEAAiE;IACjE,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,QAAmC;IAEnC,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,kEAAkE;IAClE,OAAO,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB;IACxB,OAAO;;;;;;;;8CAQqC,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,KAAa;IACrC,OAAO,oBAAoB,KAAK;;;;;OAK3B,CAAC;AACR,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB;IACzB,OAAO;iFACwE,CAAC;AAClF,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB;IAC9B,OAAO;;;;;;;;;+BASsB,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACnC,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,iBAAiB;QACjB,WAAW;QACX,kBAAkB;QAClB,iBAAiB;QACjB,gBAAgB;QAChB,kBAAkB;QAClB,uBAAuB;KACxB,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"deepseek.js","sourceRoot":"","sources":["../../src/adapters/deepseek.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EACL,eAAe,EACf,sBAAsB,GACvB,MAAM,WAAW,CAAC;AAEnB;;;GAGG;AACH,SAAS,iBAAiB,CACxB,aAAqB,EACrB,cAAsB,EACtB,KAAgB;IAEhB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,CAAC;QAAE,aAAa,GAAG,CAAC,CAAC;IAC5E,MAAM,eAAe,GAAG,aAAa,CAAC,cAAc,EAAE,CAAC;IAEvD,OAAO;;;;oDAI2C,eAAe;;;;;;;EAOjE,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4Bd,KAAK,EAAE,SAAS,IAAI,EAAE,GAAG,KAAK,EAAE,kBAAkB,IAAI,EAAE;;;wCAGlB,CAAC;AACzC,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,QAAgB;IACnC,iEAAiE;IACjE,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,QAAmC;IAEnC,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,kEAAkE;IAClE,OAAO,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB;IACxB,OAAO;;;;;;;;8CAQqC,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,KAAa;IACrC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtC,OAAO,oBAAoB,SAAS;;;;;OAK/B,CAAC;AACR,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB;IACzB,OAAO;iFACwE,CAAC;AAClF,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB;IAC9B,OAAO;;;;;;;;;+BASsB,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACnC,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,iBAAiB;QACjB,WAAW;QACX,kBAAkB;QAClB,iBAAiB;QACjB,gBAAgB;QAChB,kBAAkB;QAClB,uBAAuB;KACxB,CAAC;AACJ,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"nucleus.d.ts","sourceRoot":"","sources":["../../src/adapters/nucleus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,YAAY,EAA4B,MAAM,YAAY,CAAC;AAuUzE;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,YAAY,CAWnD"}
1
+ {"version":3,"file":"nucleus.d.ts","sourceRoot":"","sources":["../../src/adapters/nucleus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,YAAY,EAA4B,MAAM,YAAY,CAAC;AAuazE;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,YAAY,CAWnD"}