matryoshka-rlm 0.2.30 → 0.2.32

package/dist/logic/lc-solver.js

@@ -19,6 +19,60 @@ import { synthesizeExtractor, compileToFunction, prettyPrint } from "../synthesi
 import { synthesizeFromExamples } from "./relational-solver.js";
 import { SynthesisIntegrator } from "./synthesis-integrator.js";
 const moduleQStore = new QValueStore();
+/**
+ * Walk the pattern with a paren stack to detect any group that has
+ * an internal quantifier AND a trailing quantifier — the canonical
+ * shape of catastrophic backtracking, e.g. (a+)+, (a*)+, (.*(.*).*)+,
+ * ((a|b)+)+, (?:a+)+. A regex-only check can't see through nested
+ * parens; the scanner skips escapes and character classes so it
+ * doesn't false-positive on literal `(`/`)`/quantifier chars.
+ */
+function hasNestedQuantifier(pattern) {
+    const stack = [];
+    const isQuant = (c) => c === "+" || c === "*" || c === "{";
+    let i = 0;
+    while (i < pattern.length) {
+        const c = pattern[i];
+        if (c === "\\") {
+            i += 2;
+            continue;
+        }
+        if (c === "[") {
+            // Skip character class — its contents aren't group structure.
+            i++;
+            while (i < pattern.length && pattern[i] !== "]") {
+                if (pattern[i] === "\\")
+                    i++;
+                i++;
+            }
+            i++;
+            continue;
+        }
+        if (c === "(") {
+            stack.push({ hasQuantInside: false });
+            i++;
+            continue;
+        }
+        if (c === ")") {
+            const top = stack.pop();
+            i++;
+            if (i < pattern.length && isQuant(pattern[i])) {
+                if (top?.hasQuantInside)
+                    return true;
+                // Trailing quantifier on this group counts as a quantifier
+                // in the parent group's body for the next iteration.
+                if (stack.length > 0)
+                    stack[stack.length - 1].hasQuantInside = true;
+            }
+            continue;
+        }
+        if (isQuant(c) && stack.length > 0) {
+            stack[stack.length - 1].hasQuantInside = true;
+        }
+        i++;
+    }
+    return false;
+}
 /**
  * Validate a regex pattern for safety (ReDoS protection)
  */
@@ -31,9 +85,8 @@ export function validateRegex(pattern) {
     if (pattern.length > 500) {
         return { valid: false, error: `Regex pattern too long (${pattern.length} chars, max 500)` };
     }
-    // Reject nested quantifiers that cause catastrophic backtracking
-    // Patterns like (a+)+, (a*)+, (a+)*, (a{1,})+, etc.
-    if (/(\((?:[^()]*[+*{])[^()]*\))[+*{]|\(\?[^)]*[+*{][^)]*\)[+*{]/.test(pattern)) {
+    // Reject nested quantifiers that cause catastrophic backtracking.
+    if (hasNestedQuantifier(pattern)) {
         return { valid: false, error: "Regex contains nested quantifiers which may cause catastrophic backtracking" };
     }
     // Verify the pattern is a valid regex
@@ -76,6 +129,80 @@ function canonicalizeOneOf(response, oneOf) {
     }
     return null;
 }
+/**
+ * Convert a resolved `(context EXPR)` value into a clean line-oriented
+ * document for a child Nucleus session.
+ *
+ * Strings pass through untouched. Arrays join their stringified items
+ * by newlines so the child's `^anchor` regexes work — the primary
+ * differentiator from the existing llm_query path, which JSON-
+ * stringifies the binding and pollutes line starts with `[`/`{`/quotes.
+ *
+ * Per-item stringification: grep-result-like objects (anything with a
+ * string `.line` field) yield that field's value, since that's the
+ * "one item per line" representation users expect when piping grep
+ * results into a child. Other objects fall back to compact JSON.
+ * Strings, numbers, booleans become their natural string form;
+ * null/undefined become "".
+ *
+ * Output is capped at MAX_CONTEXT_DOC_CHARS to bound child memory
+ * footprint; longer values are silently truncated. The cap is
+ * intentionally larger than llm_query's 500k interpolation cap
+ * because rlm_query passes the value as a child DOCUMENT (where the
+ * matryoshka 50MB doc limit applies), not as prompt text.
+ */
+const MAX_CONTEXT_DOC_CHARS = 5_000_000;
+function stringifyItem(item) {
+    if (item === null || item === undefined)
+        return "";
+    if (typeof item === "string")
+        return item;
+    if (typeof item === "number" || typeof item === "boolean")
+        return String(item);
+    if (typeof item === "object") {
+        const obj = item;
+        if (typeof obj.line === "string")
+            return obj.line;
+        try {
+            return JSON.stringify(item);
+        }
+        catch {
+            return String(item);
+        }
+    }
+    return String(item);
+}
+/**
+ * Top-level dispatch for `(rlm_query …)` context materialization.
+ * Returns a string; an empty string ("") is a valid output (e.g., the
+ * user passed `(context [])` — empty array). Callers MUST distinguish
+ * this from `null` (no `(context …)` form supplied) when deciding
+ * whether to fall back to using the prompt as the child's document.
+ */
+function materializeContext(value) {
+    let out;
+    if (value === null || value === undefined) {
+        out = "";
+    }
+    else if (typeof value === "string") {
+        out = value;
+    }
+    else if (Array.isArray(value)) {
+        out = value.map(stringifyItem).join("\n");
+    }
+    else if (typeof value === "number" || typeof value === "boolean") {
+        out = String(value);
+    }
+    else {
+        // Object that isn't an array — use the same line-extraction
+        // heuristic as for array items.
+        out = stringifyItem(value);
+    }
+    if (out.length > MAX_CONTEXT_DOC_CHARS) {
+        out = out.slice(0, MAX_CONTEXT_DOC_CHARS);
+    }
+    return out;
+}
 /**
  * Solve an LC term using miniKanren as the logic engine
  * @param term The LC term to evaluate
@@ -176,6 +303,68 @@ async function evaluate(term, tools, bindings, log, depth = 0) {
             if (!regexValidation.valid) {
                 throw new Error(`Invalid regex pattern: ${regexValidation.error}`);
             }
+            // Phase 3 — optional haystack arg. The haystack MUST evaluate
+            // to a string. Numbers/booleans get rendered cleanly via
+            // String(); arrays/objects/null/undefined are rejected with a
+            // specific error so a typo (e.g. `(grep "X" RESULTS)` where
+            // RESULTS is the latest grep result array) doesn't silently
+            // coerce to "[object Object]" and produce garbage matches.
+            // Per project rule: correctness > performance — surface user
+            // mistakes loudly.
+            if (term.haystack) {
+                const hayValue = await evaluate(term.haystack, tools, bindings, log, depth + 1);
+                let haystack;
+                if (typeof hayValue === "string") {
+                    haystack = hayValue;
+                }
+                else if (typeof hayValue === "number" || typeof hayValue === "boolean") {
+                    haystack = String(hayValue);
+                }
+                else {
+                    const shape = Array.isArray(hayValue)
+                        ? `array(${hayValue.length})`
+                        : hayValue === null
+                            ? "null"
+                            : hayValue === undefined
+                                ? "undefined"
+                                : typeof hayValue;
+                    throw new Error(`(grep "${term.pattern}" HAYSTACK): haystack must evaluate to a string, got ${shape}. ` +
+                        `If you have an array, materialize it first (e.g., join lines) or grep an individual element.`);
+                }
+                log(`[Solver] Executing grep("${pattern}") over haystack (${haystack.length} chars)`);
+                if (tools.grepIn) {
+                    const results = tools.grepIn(pattern, haystack);
+                    log(`[Solver] Found ${results.length} matches in haystack`);
+                    return results;
+                }
+                // Fallback in-line search — same shape as the production
+                // grep callback. Used by test stubs that don't provide
+                // grepIn AND legacy SolverTools instances. ReDoS validated
+                // upstream by the same validateRegex call we already ran.
+                const flags = "gmi";
+                const regex = new RegExp(pattern, flags);
+                const lines = haystack.split("\n");
+                const results = [];
+                const MAX_HAYSTACK_MATCHES = 10_000;
+                let m;
+                while ((m = regex.exec(haystack)) !== null) {
+                    const beforeMatch = haystack.slice(0, m.index);
+                    const lineNum = (beforeMatch.match(/\n/g) || []).length + 1;
+                    results.push({
+                        match: m[0],
+                        line: lines[lineNum - 1] ?? "",
+                        lineNum,
+                        index: m.index,
+                        groups: m.slice(1).filter((g) => g !== undefined),
+                    });
+                    if (m[0].length === 0)
+                        regex.lastIndex++;
+                    if (results.length >= MAX_HAYSTACK_MATCHES)
+                        break;
+                }
+                log(`[Solver] Found ${results.length} matches in haystack (in-line fallback)`);
+                return results;
+            }
             log(`[Solver] Executing grep("${pattern}")`);
             const results = tools.grep(pattern);
             log(`[Solver] Found ${results.length} matches`);
@@ -187,6 +376,71 @@ async function evaluate(term, tools, bindings, log, depth = 0) {
             }
             return results;
         }
+        case "show_vars": {
+            // Phase 4 — binding introspection. Returns a string summary
+            // that can be inlined into a sub-LLM prompt, used in a
+            // FINAL_VAR substitution, or just passed back to the user as
+            // the answer. Format mirrors `lattice_bindings` in spirit but
+            // is reachable from inside a query.
+            //
+            // Internal bindings start with `_` followed by a non-digit
+            // character (e.g., `_sessionDB`, `_compaction_trace`). These
+            // are private plumbing the user shouldn't see — surfacing
+            // them would expose internal state and confuse the LLM about
+            // what it can FINAL_VAR-reference. User-visible bindings:
+            // RESULTS, plain names, and `_<digits>` (per-turn results).
+            const isInternal = (name) => name.startsWith("_") && name.length > 1 && !/^_\d/.test(name);
+            const visibleEntries = [...bindings].filter(([n]) => !isInternal(n));
+            if (visibleEntries.length === 0) {
+                return "No bindings yet — run a query to populate RESULTS or a binding name.";
+            }
+            const lines = [];
+            for (const [name, value] of visibleEntries) {
+                let shape;
+                if (Array.isArray(value)) {
+                    shape = `Array(${value.length})`;
+                }
+                else if (typeof value === "string") {
+                    shape = `String(${value.length})`;
+                }
+                else if (typeof value === "number") {
+                    shape = `Number(${value})`;
+                }
+                else if (typeof value === "boolean") {
+                    shape = `Boolean(${value})`;
+                }
+                else if (value === null) {
+                    shape = "null";
+                }
+                else if (value === undefined) {
+                    shape = "undefined";
+                }
+                else if (typeof value === "object") {
+                    shape = "object";
+                }
+                else {
+                    shape = typeof value;
+                }
+                lines.push(`  ${name}: ${shape}`);
+            }
+            return `Bindings (${visibleEntries.length}):\n${lines.join("\n")}`;
+        }
+        case "context": {
+            // Phase 3 — `(context N)` selector. Returns the Nth loaded
+            // context's content. Falls back to `tools.context` when
+            // `contexts` is unset and the index is 0 — back-compat for
+            // single-doc workflows that haven't wired the new field.
+            const idx = term.index;
+            if (tools.contexts) {
+                if (idx < 0 || idx >= tools.contexts.length) {
+                    throw new Error(`(context ${idx}): index out of range — ${tools.contexts.length} context(s) loaded`);
+                }
+                return tools.contexts[idx];
+            }
+            if (idx === 0)
+                return tools.context;
+            throw new Error(`(context ${idx}): only 1 context loaded — use runRLMFromContent with an array to enable multi-context`);
+        }
         case "fuzzy_search": {
             const fuzzyLimit = Math.min(Math.max(1, term.limit ?? 10), 1000);
             log(`[Solver] Executing fuzzy_search("${term.query}", ${fuzzyLimit})`);
@@ -615,6 +869,9 @@ async function evaluate(term, tools, bindings, log, depth = 0) {
             if (typeof str !== "string") {
                 throw new Error(`replace: expected string, got ${typeof str}`);
             }
+            if (typeof term.to !== "string") {
+                throw new Error(`replace: expected string for 'to', got ${typeof term.to}`);
+            }
             const replaceValidation = validateRegex(term.from);
             if (!replaceValidation.valid) {
                 throw new Error(`replace: ${replaceValidation.error}`);
@@ -637,7 +894,12 @@ async function evaluate(term, tools, bindings, log, depth = 0) {
             if (!term.delim || term.delim.length === 0 || term.delim.length > 1000)
                 return null;
             const MAX_SPLIT_PARTS = 10_000;
-            const parts = str.split(term.delim);
+            // Bound the split itself, not just the post-allocation check —
+            // an unbounded `str.split(delim)` on a multi-MB string with a
+            // common delimiter materializes the full part array (potentially
+            // millions of strings) before we ever look at .length. Asking
+            // for one extra part is enough to detect overflow.
+            const parts = str.split(term.delim, MAX_SPLIT_PARTS + 1);
             if (parts.length > MAX_SPLIT_PARTS)
                 return null;
             return parts[term.index] ?? null;
@@ -1184,6 +1446,89 @@ async function evaluate(term, tools, bindings, log, depth = 0) {
             }
             return response;
         }
+        case "rlm_query": {
+            // Phase 1 recursive primitive — spawn a child Nucleus FSM with
+            // an explicit query and (optional) clean line-oriented document.
+            // The actual child-spawning logic lives behind the `rlmQuery`
+            // callback the parent's RLM loop wires up; this case is just
+            // the materialization-and-dispatch layer.
+            if (!tools.rlmQuery) {
+                throw new Error("rlm_query is not available in this execution context. " +
+                    "The RLM loop provides it once the parent threads an " +
+                    "rlmQuery callback through SolverTools. Standalone " +
+                    "NucleusEngine / HandleSession instances must pass an " +
+                    "rlmQuery option to enable it.");
+            }
+            // Resolve (context EXPR) when present. The materialized form is
+            // line-oriented: arrays join their stringified items by \n;
+            // strings pass through unchanged; null/undefined become null.
+            // This is the structural difference from llm_query — the child
+            // gets a clean document its grep/lines/chunk primitives can
+            // operate on, not a JSON-stringified blob.
+            let contextDoc = null;
+            if (term.context) {
+                const resolved = await evaluate(term.context, tools, bindings, log, depth + 1);
+                contextDoc = materializeContext(resolved);
+            }
+            log(`[Solver] rlm_query prompt length: ${term.prompt.length} chars, ` +
+                `context: ${contextDoc === null ? "null" : `${contextDoc.length} chars`}`);
+            const response = await tools.rlmQuery(term.prompt, contextDoc);
+            log(`[Solver] rlm_query response length: ${response.length} chars`);
+            return response;
+        }
+        case "rlm_batch": {
+            // Phase 2 — concurrent variant of `(map COLL (lambda x (rlm_query …)))`.
+            // Builds N (prompt, contextDoc) pairs from the inner rlm_query
+            // template and dispatches them through tools.rlmBatch in a
+            // single call. The implementation fans them out concurrently
+            // (typically Promise.all over child sessions); the solver
+            // itself is just the materialization-and-dispatch layer.
+            if (!tools.rlmBatch) {
+                throw new Error("rlm_batch is not available in this execution context. " +
+                    "The RLM loop provides it once the parent threads an " +
+                    "rlmBatch callback through SolverTools. Standalone " +
+                    "NucleusEngine / HandleSession instances must pass an " +
+                    "rlmBatch option to enable it.");
+            }
+            const collection = await evaluate(term.collection, tools, bindings, log, depth + 1);
+            if (!Array.isArray(collection)) {
+                throw new Error(`rlm_batch: expected array, got ${typeof collection}`);
+            }
+            // Empty collection short-circuit — match llm_batch's behavior
+            // so callers don't pay a dispatch round-trip for zero work.
+            if (collection.length === 0) {
+                log(`[Solver] rlm_batch: empty collection, skipping dispatch`);
+                return [];
+            }
+            const items = [];
+            for (const item of collection) {
+                // Bind the lambda param to the RAW item (no toItemString
+                // coercion). rlm_batch funnels each item through
+                // materializeContext, which knows how to render arrays as
+                // line-oriented documents and objects with a `.line` field
+                // as their line text. Stringifying the item up-front would
+                // collapse all that structure into a flat string and defeat
+                // the handle-as-document property we built for rlm_query.
+                const scopedBindings = new Map(bindings);
+                scopedBindings.set(term.param, item);
+                let contextDoc = null;
+                if (term.context) {
+                    const resolved = await evaluate(term.context, tools, scopedBindings, log, depth + 1);
+                    contextDoc = materializeContext(resolved);
+                }
+                items.push({ prompt: term.prompt, contextDoc });
+            }
+            log(`[Solver] rlm_batch dispatching ${items.length} child sessions in a single call`);
+            const responses = await tools.rlmBatch(items);
+            if (!Array.isArray(responses)) {
+                throw new Error(`rlm_batch: tools.rlmBatch must return an array, got ${typeof responses}`);
+            }
+            if (responses.length !== items.length) {
+                throw new Error(`rlm_batch: expected ${items.length} responses, got ${responses.length}`);
+            }
+            log(`[Solver] rlm_batch received ${responses.length} responses`);
+            return responses;
+        }
         case "llm_batch": {
             // Batched sibling of `llm_query`. Drop-in replacement for
             //   (map COLL (lambda x (llm_query "…" …bindings)))
@@ -1388,6 +1733,9 @@ async function evaluateWithBinding(body, param, value, tools, bindings, log, dep
             const str = body.str.tag === "var" && body.str.name === param
                 ? String(value)
                 : String(await evaluateWithBinding(body.str, param, value, tools, bindings, log, depth + 1));
+            if (typeof body.to !== "string") {
+                throw new Error(`replace: expected string for 'to', got ${typeof body.to}`);
+            }
             const replaceVal = validateRegex(body.from);
             if (!replaceVal.valid) {
                 throw new Error(`replace: ${replaceVal.error}`);