matryoshka-rlm 0.1.1 → 0.2.0

package/README.md

@@ -4,10 +4,110 @@ Process documents 100x larger than your LLM's context window—without vector da
 
 ## The Problem
 
-LLMs have fixed context windows. Traditional solutions (RAG, chunking) lose information or miss connections across chunks. RLM takes a different approach: the model writes code to explore documents programmatically, deciding at runtime how to decompose and analyze the data.
+LLMs have fixed context windows. Traditional solutions (RAG, chunking) lose information or miss connections across chunks. RLM takes a different approach: the model reasons about your query and outputs symbolic commands that a logic engine executes against the document.
 
 Based on the [Recursive Language Models paper](https://arxiv.org/abs/2512.24601).
 
+## How It Works
+
+Unlike traditional approaches where an LLM writes arbitrary code, RLM uses **[Nucleus](https://github.com/michaelwhitford/nucleus)**—a constrained symbolic language based on S-expressions. The LLM outputs Nucleus commands, which are parsed, type-checked, and executed by **Lattice**, our logic engine.
+
+```
+┌─────────────────┐     ┌─────────────────┐     ┌─────────────────┐
+│   User Query    │────▶│   LLM Reasons   │────▶│ Nucleus Command │
+│ "total sales?"  │     │  about intent   │     │  (sum RESULTS)  │
+└─────────────────┘     └─────────────────┘     └────────┬────────┘
+                                                         │
+┌─────────────────┐     ┌─────────────────┐     ┌────────▼────────┐
+│  Final Answer   │◀────│ Lattice Engine  │◀────│     Parser      │
+│   13,000,000    │     │    Executes     │     │    Validates    │
+└─────────────────┘     └─────────────────┘     └─────────────────┘
+```
+
+**Why this works better than code generation:**
+
+1. **Reduced entropy** - Nucleus has a rigid grammar with fewer valid outputs than JavaScript
+2. **Fail-fast validation** - Parser rejects malformed commands before execution
+3. **Safe execution** - Lattice only executes known operations, no arbitrary code
+4. **Small model friendly** - 7B models handle symbolic grammars better than freeform code
+
+## Architecture
+
+### The Nucleus DSL
+
+The LLM outputs commands in the Nucleus DSL—an S-expression language designed for document analysis:
+
+```scheme
+; Search for patterns
+(grep "SALES_DATA")
+
+; Filter results
+(filter RESULTS (lambda x (match x "NORTH" 0)))
+
+; Aggregate
+(sum RESULTS)    ; Auto-extracts numbers like "$2,340,000" from lines
+(count RESULTS)  ; Count matching items
+
+; Final answer
+<<<FINAL>>>13000000<<<END>>>
+```
+
+### The Lattice Engine
+
+The Lattice engine (`src/logic/`) processes Nucleus commands:
+
+1. **Parser** (`lc-parser.ts`) - Parses S-expressions into an AST
+2. **Type Inference** (`type-inference.ts`) - Validates types before execution
+3. **Constraint Resolver** (`constraint-resolver.ts`) - Handles symbolic constraints like `[Σ⚡μ]`
+4. **Solver** (`lc-solver.ts`) - Executes commands against the document
+
+Lattice uses **miniKanren** (a relational programming engine) for pattern classification and filtering operations.
+
+### Pre-Search Optimization
+
+Before calling the LLM, the system extracts keywords from your query and pre-runs grep:
+
+```
+Query: "What is the total of all north sales data values?"
+                    │
+                    ▼
+┌─────────────────────────────────────────────────────┐
+│ Pre-search extracts: "north", "sales", "data"       │
+│ Tries compound patterns: SALES.*NORTH, NORTH.*SALES │
+│ Pre-populates RESULTS before LLM is called          │
+└─────────────────────────────────────────────────────┘
+                    │
+                    ▼
+┌─────────────────────────────────────────────────────┐
+│ LLM receives: "RESULTS has 1 match"                 │
+│ LLM outputs: (sum RESULTS)  ← skips search step!   │
+└─────────────────────────────────────────────────────┘
+```
+
+This saves turns by pre-populating `RESULTS` so the model can immediately aggregate.
+
+### The Role of the LLM
+
+The LLM does **reasoning**, not code generation:
+
+1. **Understands intent** - Interprets "total of north sales" as needing grep + filter + sum
+2. **Chooses operations** - Decides which Nucleus commands achieve the goal
+3. **Verifies results** - Checks if the current results answer the query
+4. **Iterates** - Refines search if results are too broad or narrow
+
+The LLM never writes JavaScript. It outputs Nucleus commands that Lattice executes safely.
+
+### Components Summary
+
+| Component | Purpose |
+|-----------|---------|
+| **Nucleus Adapter** | Prompts LLM to output Nucleus commands |
+| **Lattice Parser** | Parses S-expressions to AST |
+| **Lattice Solver** | Executes commands against document |
+| **miniKanren** | Relational engine for classification |
+| **Pre-Search** | Extracts keywords and pre-runs grep |
+| **RAG Hints** | Few-shot examples from past successes |
+
 ## Installation
 
 ### npm (recommended)
@@ -19,7 +119,7 @@ npm install -g matryoshka-rlm
 ### npx (no install)
 
 ```bash
-npx matryoshka-rlm "Summarize this document" ./document.txt
+npx matryoshka-rlm "What is the total of all sales values?" ./report.txt
 ```
 
 ### From source
@@ -43,7 +143,7 @@ Copy `config.example.json` to `config.json` and configure your LLM provider:
   "providers": {
     "ollama": {
       "baseUrl": "http://localhost:11434",
-      "model": "qwen3-coder:30b",
+      "model": "qwen2.5-coder:7b",
       "options": { "temperature": 0.2, "num_ctx": 8192 }
     },
     "deepseek": {
@@ -62,10 +162,10 @@ Copy `config.example.json` to `config.json` and configure your LLM provider:
 
 ```bash
 # Basic usage
-rlm "Summarize this document" ./path/to/document.txt
+rlm "What is the total of all sales values?" ./report.txt
 
 # With options
-rlm "Find all error codes" ./logs.txt --max-turns 15 --verbose
+rlm "Count all ERROR entries" ./logs.txt --max-turns 15 --verbose
 
 # See all options
 rlm --help
@@ -73,7 +173,7 @@ rlm --help
 
 ### MCP Integration
 
-RLM includes an MCP (Model Context Protocol) server that exposes the `analyze_document` tool. This allows coding agents like [Crush](https://github.com/charmbracelet/crush) to analyze documents that exceed its context window.
+RLM includes an MCP (Model Context Protocol) server that exposes the `analyze_document` tool. This allows coding agents to analyze documents that exceed their context window.
 
 #### MCP Tool: `analyze_document`
 
@@ -84,9 +184,7 @@ RLM includes an MCP (Model Context Protocol) server that exposes the `analyze_do
 | `maxTurns` | number | No | Maximum exploration turns (default: 10) |
 | `timeoutMs` | number | No | Timeout per turn in milliseconds (default: 30000) |
 
-#### Crush MCP example
-
-Add to your `crush.json` config:
+#### Example MCP config
 
 ```json
 {
@@ -98,16 +196,10 @@ Add to your `crush.json` config:
   }
 }
 ```
-Then ask Crush to analyze documents:
-
-> Use the analyze_document tool to find all sales figures in /path/to/report.txt and calculate the total
-
-See [Crush](https://github.com/charmbracelet/crush) for more details.
 
 #### Testing the MCP Server
 
 ```bash
-# Verify the server starts correctly
 rlm-mcp --test
 # Output: MCP server ready
 # Output: Available tools: analyze_document
@@ -121,143 +213,204 @@ import { createLLMClient } from "matryoshka-rlm";
 
 const llmClient = createLLMClient("ollama", {
   baseUrl: "http://localhost:11434",
-  model: "qwen3-coder:30b",
+  model: "qwen2.5-coder:7b",
   options: { temperature: 0.2 }
 });
 
-const result = await runRLM("What are the main themes?", "./book.txt", {
+const result = await runRLM("What is the total of all sales values?", "./report.txt", {
   llmClient,
   maxTurns: 10,
   turnTimeoutMs: 30000,
 });
 ```
 
-## Architecture
+## Example Session
 
-```mermaid
-sequenceDiagram
-    participant User
-    participant RLM as RLM Engine
-    participant Sandbox as JavaScript Sandbox
-    participant LLM as LLM Provider
-
-    User->>RLM: query + document path
-    RLM->>Sandbox: Create sandbox with document as `context`
-
-    loop Until FINAL or maxTurns
-        RLM->>LLM: System prompt + history
-        LLM-->>RLM: JavaScript code block
-        RLM->>Sandbox: Execute code (with timeout)
-        Sandbox-->>RLM: { result, logs, error }
-        Note over RLM: Append output to history
-    end
-
-    RLM-->>User: Final answer
 ```
+$ rlm "What is the total of all north sales data values?" ./report.txt --verbose
 
-### Components
+[Pre-search] Found 1 data matches for "SALES.*NORTH"
+[Pre-search] RESULTS pre-populated with 1 matches
 
-| Component | Purpose |
-|-----------|---------|
-| **RLM Engine** | Orchestrates the turn loop, builds prompts, extracts answers |
-| **Sandbox** | Isolated VM executing LLM-generated JavaScript with timeout protection |
-| **Tools** | `text_stats()`, `fuzzy_search()`, `llm_query()` available in sandbox |
-| **Memory** | Persistent array for accumulating findings across turns |
+──────────────────────────────────────────────────
+[Turn 1/10] Querying LLM...
+[Turn 1] Term: (sum RESULTS)
+[Turn 1] Console output:
+  [Lattice] Summing 1 values
+  [Lattice] Sum = 2340000
+[Turn 1] Result: 2340000
 
-### How It Works
+──────────────────────────────────────────────────
+[Turn 2/10] Querying LLM...
+[Turn 2] Final answer received
 
-1. Document loads into sandbox as read-only `context` variable
-2. LLM receives system prompt with available tools and writes JavaScript
-3. Code executes in sandbox, results feed back to LLM
-4. LLM iterates until it outputs `<<<FINAL>>>answer<<<END>>>`
-5. Sub-queries via `llm_query()` enable recursive decomposition
+2340000
+```
 
-### Sandbox Tools
+The model:
+1. Received pre-populated RESULTS (pre-search found the data)
+2. Immediately summed the results (no grep needed)
+3. Output the final answer
 
-The LLM has access to these tools when exploring documents:
+## Nucleus DSL Reference
 
-| Tool | Description |
-|------|-------------|
-| `text_stats()` | Returns document metadata: length, line count, samples from start/middle/end |
-| `fuzzy_search(query, limit?)` | Finds approximate matches, returns lines with scores |
-| `llm_query(prompt)` | Spawns a sub-LLM call for complex analysis (limited by `maxSubCalls`) |
-| `context` | The full document text (read-only string) |
-| `memory` | Persistent array to accumulate findings across turns |
+### Search Commands
 
-### Safety
+```scheme
+(grep "pattern")              ; Regex search, returns matches with line numbers
+(fuzzy_search "query" 10)     ; Fuzzy search, returns top N matches with scores
+(text_stats)                  ; Document metadata (length, line count, samples)
+```
 
-- Sandbox isolates code execution (no filesystem, network, or process access)
-- Configurable timeout per turn
-- `maxSubCalls` limit prevents infinite recursion
-- Sub-LLM calls receive only the prompt, never parent context
-- Auto-fixes common syntax errors in LLM-generated code
+### Collection Operations
 
-## Troubleshooting
+```scheme
+(filter RESULTS (lambda x (match x "pattern" 0)))  ; Filter by regex
+(map RESULTS (lambda x (match x "(\\d+)" 1)))      ; Extract from each
+(sum RESULTS)                                       ; Sum numbers in results
+(count RESULTS)                                     ; Count items
+```
 
-### Model Answers Immediately Without Exploring
+### String Operations
 
-**Symptom**: The model provides an answer on the first turn without running any code, often with hallucinated data.
+```scheme
+(match str "pattern" 0)       ; Regex match, return group N
+(replace str "from" "to")     ; String replacement
+(split str "," 0)             ; Split and get index
+(parseInt str)                ; Parse integer
+(parseFloat str)              ; Parse float
+```
 
-**Cause**: Smaller or less capable models may not follow the instruction to explore via code before answering.
+### Type Coercion
 
-**Solutions**:
+When the model sees data that needs parsing, it can use declarative type coercion:
+
+```scheme
+; Date parsing (returns ISO format YYYY-MM-DD)
+(parseDate "Jan 15, 2024")           ; -> "2024-01-15"
+(parseDate "01/15/2024" "US")        ; -> "2024-01-15" (MM/DD/YYYY)
+(parseDate "15/01/2024" "EU")        ; -> "2024-01-15" (DD/MM/YYYY)
+
+; Currency parsing (handles $, €, commas, etc.)
+(parseCurrency "$1,234.56")          ; -> 1234.56
+(parseCurrency "€1.234,56")          ; -> 1234.56 (EU format)
 
-1. **Use a more capable model** - Models like `deepseek-chat` or larger Ollama models follow instructions better
-2. **Make your query more specific** - Instead of vague queries, be explicit:
-   ```bash
-   # Vague (may cause hallucination)
-   rlm "What are the sales figures?" ./report.txt
+; Number parsing
+(parseNumber "1,234,567")            ; -> 1234567
+(parseNumber "50%")                  ; -> 0.5
 
-   # Specific (guides exploration)
-   rlm "Search for SALES_DATA entries and sum the dollar amounts" ./report.txt
-   ```
-3. **Include data patterns in your query** - If you know how data is formatted, mention it:
-   ```bash
-   rlm "Find lines matching 'Total:' and extract the numbers" ./data.txt
-   ```
+; General coercion
+(coerce value "date")                ; Coerce to date
+(coerce value "currency")            ; Coerce to currency
+(coerce value "number")              ; Coerce to number
 
-### Max Turns Reached Without Answer
+; Extract and coerce in one step
+(extract str "\\$[\\d,]+" 0 "currency")  ; Extract and parse as currency
+```
+
+Use in map for batch transformations:
+
+```scheme
+; Parse all dates in results
+(map RESULTS (lambda x (parseDate (match x "[A-Za-z]+ \\d+, \\d+" 0))))
+
+; Extract and sum currencies
+(map RESULTS (lambda x (parseCurrency (match x "\\$[\\d,]+" 0))))
+```
+
+### Program Synthesis
+
+For complex transformations, the model can synthesize functions from examples:
+
+```scheme
+; Synthesize from input/output pairs
+(synthesize
+  ("$100" 100)
+  ("$1,234" 1234)
+  ("$50,000" 50000))
+; -> Returns a function that extracts numbers from currency strings
+```
 
-**Symptom**: Output shows "Max turns (N) reached without final answer"
+This uses Barliman-style relational synthesis with miniKanren to automatically build extraction functions.
 
-**Cause**: The model keeps exploring but never terminates properly.
+### Cross-Turn State
+
+Results from previous turns are available:
+- `RESULTS` - Latest array result (updated by grep, filter)
+- `_0`, `_1`, `_2`, ... - Results from specific turns
+
+### Final Answer
+
+```scheme
+<<<FINAL>>>your answer here<<<END>>>
+```
+
+## Troubleshooting
+
+### Model Answers Without Exploring
+
+**Symptom**: The model provides an answer immediately with hallucinated data.
 
 **Solutions**:
+1. Use a more capable model (7B+ recommended)
+2. Be specific in your query: "Find lines containing SALES_DATA and sum the dollar amounts"
+
+### Max Turns Reached
 
+**Symptom**: "Max turns (N) reached without final answer"
+
+**Solutions**:
 1. Increase `--max-turns` for complex documents
-2. Check if the model is stuck in a loop (`--verbose` shows repeated patterns)
-3. Simplify the query to require less exploration
+2. Check `--verbose` output for repeated patterns (model stuck in loop)
+3. Simplify the query
 
-### Sandbox Execution Errors
+### Parse Errors
 
-**Symptom**: Repeated "Error: Unexpected token" or similar JavaScript errors
+**Symptom**: "Parse error: no valid command"
 
-**Cause**: The model is generating invalid JavaScript code.
+**Cause**: Model output malformed S-expression.
 
 **Solutions**:
-
-1. The system auto-fixes common issues (missing semicolons, TypeScript syntax)
-2. If errors persist, try a different model - some are better at generating valid code
-3. Use `--verbose` to see what code the model is generating
+1. The system auto-converts JSON to S-expressions as fallback
+2. Use `--verbose` to see what the model is generating
+3. Try a different model tuned for code/symbolic output
 
 ## Development
 
 ```bash
-# Run tests
-npm test
+npm test                              # Run tests
+npm test -- --coverage                # With coverage
+RUN_E2E=1 npm test -- tests/e2e.test.ts  # E2E tests (requires Ollama)
+npm run build                         # Build
+npm run typecheck                     # Type check
+```
 
-# Run with coverage
-npm test -- --coverage
+## Project Structure
 
-# E2E tests (requires Ollama running locally)
-RUN_E2E=1 npm test -- --run tests/e2e.test.ts
+```
+src/
+├── adapters/           # Model-specific prompting
+│   ├── nucleus.ts      # Nucleus DSL adapter
+│   └── types.ts        # Adapter interface
+├── logic/              # Lattice engine
+│   ├── lc-parser.ts    # Nucleus parser
+│   ├── lc-solver.ts    # Command executor (uses miniKanren)
+│   ├── type-inference.ts
+│   └── constraint-resolver.ts
+├── minikanren/         # Relational programming engine
+├── synthesis/          # Program synthesis (Barliman-style)
+│   └── evalo/          # Extractor DSL
+├── rag/                # Few-shot hint retrieval
+└── rlm.ts              # Main execution loop
+```
 
-# Build
-npm run build
+## Acknowledgements
 
-# Type check
-npm run typecheck
-```
+This project incorporates ideas and code from:
+
+- **[Nucleus](https://github.com/michaelwhitford/nucleus)** - A symbolic S-expression language by Michael Whitford. RLM uses Nucleus syntax for the constrained DSL that the LLM outputs, providing a rigid grammar that reduces model errors.
+- **[ramo](https://github.com/wjlewis/ramo)** - A miniKanren implementation in TypeScript by Will Lewis. Used for constraint-based program synthesis.
+- **[Barliman](https://github.com/webyrd/Barliman)** - A prototype smart editor by William Byrd and Greg Rosenblatt that uses program synthesis to assist programmers. The Barliman-style approach of providing input/output constraints instead of code inspired the synthesis workflow.
 
 ## License
 
@@ -268,3 +421,4 @@ MIT
 - [RLM Paper](https://arxiv.org/abs/2512.24601)
 - [Original Implementation](https://github.com/alexzhang13/rlm)
 - [Model Context Protocol](https://modelcontextprotocol.io/)
+- [miniKanren](http://minikanren.org/)

package/config.example.json

@@ -20,11 +20,6 @@
       }
     }
   },
-  "sandbox": {
-    "maxSubCalls": 10,
-    "turnTimeoutMs": 30000,
-    "memoryLimitMb": 128
-  },
   "rlm": {
     "maxTurns": 10
   }

package/dist/adapters/base.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Base Model Adapter
+ *
+ * Default adapter implementation that works with most models.
+ * Other adapters can spread this and override specific methods.
+ */
+import type { ModelAdapter, FinalVarMarker, RAGHints } from "./types.js";
+/**
+ * Build the default system prompt for the RLM
+ */
+declare function buildSystemPrompt(contextLength: number, toolInterfaces: string, hints?: RAGHints): string;
+/**
+ * Extract code from LLM response
+ */
+declare function extractCode(response: string): string | null;
+/**
+ * Extract final answer from LLM response
+ */
+declare function extractFinalAnswer(response: string | undefined | null): string | FinalVarMarker | null;
+/**
+ * Get feedback message when model provides no code block
+ */
+declare function getNoCodeFeedback(): string;
+/**
+ * Get feedback message when code execution fails
+ */
+declare function getErrorFeedback(error: string): string;
+/**
+ * Get feedback message after successful code execution
+ * Generic reminder about continuing exploration or providing final answer
+ */
+declare function getSuccessFeedback(): string;
+/**
+ * Get feedback message when model repeats the same code
+ * Encourages a different approach
+ */
+declare function getRepeatedCodeFeedback(): string;
+/**
+ * Create the base adapter instance
+ */
+export declare function createBaseAdapter(): ModelAdapter;
+export { buildSystemPrompt as baseBuildSystemPrompt, extractCode as baseExtractCode, extractFinalAnswer as baseExtractFinalAnswer, getNoCodeFeedback as baseGetNoCodeFeedback, getErrorFeedback as baseGetErrorFeedback, getSuccessFeedback as baseGetSuccessFeedback, getRepeatedCodeFeedback as baseGetRepeatedCodeFeedback, };
+//# sourceMappingURL=base.d.ts.map

package/dist/adapters/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/adapters/base.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEzE;;GAEG;AACH,iBAAS,iBAAiB,CACxB,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM,EACtB,KAAK,CAAC,EAAE,QAAQ,GACf,MAAM,CAyDR;AAED;;GAEG;AACH,iBAAS,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAUpD;AAED;;GAEG;AACH,iBAAS,kBAAkB,CACzB,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAClC,MAAM,GAAG,cAAc,GAAG,IAAI,CA6ChC;AAED;;GAEG;AACH,iBAAS,iBAAiB,IAAI,MAAM,CAMnC;AAED;;GAEG;AACH,iBAAS,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE/C;AAED;;;GAGG;AACH,iBAAS,kBAAkB,IAAI,MAAM,CAEpC;AAED;;;GAGG;AACH,iBAAS,uBAAuB,IAAI,MAAM,CASzC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,YAAY,CAWhD;AAGD,OAAO,EACL,iBAAiB,IAAI,qBAAqB,EAC1C,WAAW,IAAI,eAAe,EAC9B,kBAAkB,IAAI,sBAAsB,EAC5C,iBAAiB,IAAI,qBAAqB,EAC1C,gBAAgB,IAAI,oBAAoB,EACxC,kBAAkB,IAAI,sBAAsB,EAC5C,uBAAuB,IAAI,2BAA2B,GACvD,CAAC"}