@probelabs/probe 0.6.0-rc240 → 0.6.0-rc242

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@probelabs/probe",
-  "version": "0.6.0-rc240",
+  "version": "0.6.0-rc242",
   "description": "Node.js wrapper for the probe code search tool",
   "main": "src/index.js",
   "module": "src/index.js",

package/src/agent/ProbeAgent.js

@@ -102,6 +102,7 @@ import { formatErrorForAI, ParameterError } from '../utils/error-types.js';
 import { getCommonPrefix, toRelativePath, safeRealpath } from '../utils/path-validation.js';
 import { truncateIfNeeded, getMaxOutputTokens } from './outputTruncator.js';
 import { DelegationManager } from '../delegate.js';
+import { extractRawOutputBlocks } from '../tools/executePlan.js';
 import {
   TaskManager,
   createTaskTool,
@@ -3610,6 +3611,18 @@ Follow these instructions carefully:
 
                 let toolResultContent = typeof executionResult === 'string' ? executionResult : JSON.stringify(executionResult, null, 2);
 
+                // Extract raw output blocks and pass them through to output buffer (before truncation)
+                // This prevents LLM from processing/hallucinating large structured output from execute_plan
+                if (this._outputBuffer) {
+                  const { cleanedContent, extractedBlocks } = extractRawOutputBlocks(toolResultContent, this._outputBuffer);
+                  if (extractedBlocks.length > 0) {
+                    toolResultContent = cleanedContent;
+                    if (this.debug) {
+                      console.log(`[DEBUG] Extracted ${extractedBlocks.length} raw output blocks (${extractedBlocks.reduce((sum, b) => sum + b.length, 0)} chars) to output buffer`);
+                    }
+                  }
+                }
+
                 // Truncate if output exceeds token limit
                 try {
                   const truncateResult = await truncateIfNeeded(toolResultContent, this.tokenCounter, this.sessionId, this.maxOutputTokens);
@@ -3856,6 +3869,18 @@ Follow these instructions carefully:
                   toolResultContent = toolResultContent.split(wsPrefix).join('');
                 }
 
+                // Extract raw output blocks and pass them through to output buffer (before truncation)
+                // This prevents LLM from processing/hallucinating large structured output from execute_plan
+                if (this._outputBuffer) {
+                  const { cleanedContent, extractedBlocks } = extractRawOutputBlocks(toolResultContent, this._outputBuffer);
+                  if (extractedBlocks.length > 0) {
+                    toolResultContent = cleanedContent;
+                    if (this.debug) {
+                      console.log(`[DEBUG] Extracted ${extractedBlocks.length} raw output blocks (${extractedBlocks.reduce((sum, b) => sum + b.length, 0)} chars) to output buffer`);
+                    }
+                  }
+                }
+
                 // Truncate if output exceeds token limit
                 try {
                   const truncateResult = await truncateIfNeeded(toolResultContent, this.tokenCounter, this.sessionId, this.maxOutputTokens);

package/src/agent/dsl/environment.js

@@ -227,9 +227,21 @@ export function generateSandboxGlobals(options) {
   }
 
   // LLM() built-in — delegate already has its own OTEL, but we add a DSL-level span
+  // When schema is provided, auto-parse the JSON result for easier downstream processing
   if (llmCall) {
     const rawLLM = async (instruction, data, opts = {}) => {
-      return llmCall(instruction, data, opts);
+      const result = await llmCall(instruction, data, opts);
+      // Auto-parse JSON when schema is provided and result is a string
+      if (opts.schema && typeof result === 'string') {
+        try {
+          return JSON.parse(result);
+        } catch (e) {
+          // If parsing fails, return the raw string (may have formatting issues)
+          logFn?.('[LLM] Warning: schema provided but result is not valid JSON');
+          return result;
+        }
+      }
+      return result;
     };
     globals.LLM = traceToolCall('LLM', rawLLM, tracer, logFn);
   }
@@ -306,6 +318,81 @@ export function generateSandboxGlobals(options) {
     return chunks;
   };
 
+  // chunkByKey() - chunk data ensuring same-key items stay together
+  // - Chunks CAN have multiple keys (customers)
+  // - But same key NEVER splits across chunks
+  globals.chunkByKey = (data, keyFn, maxTokens = 20000) => {
+    const CHARS_PER_TOKEN = 4;
+    const maxChars = maxTokens * CHARS_PER_TOKEN;
+    const text = typeof data === 'string' ? data : JSON.stringify(data);
+
+    // Find all File: markers
+    const blockRegex = /^File: ([^\n]+)/gm;
+    const markers = [];
+    let match;
+    while ((match = blockRegex.exec(text)) !== null) {
+      markers.push({ index: match.index, file: match[1].trim() });
+    }
+
+    // No File: headers - fallback to regular chunk
+    if (markers.length === 0) {
+      return globals.chunk(data, maxTokens);
+    }
+
+    const chunks = [];
+    let currentChunk = '';
+    let currentSize = 0;
+    let keysInChunk = new Set();  // Track which keys are in current chunk
+
+    // Process each block
+    for (let i = 0; i < markers.length; i++) {
+      const start = markers[i].index;
+      const end = i + 1 < markers.length ? markers[i + 1].index : text.length;
+      const block = text.slice(start, end).trim();
+      const file = markers[i].file;
+      const key = typeof keyFn === 'function' ? keyFn(file) : file;
+
+      const blockSize = block.length + 2; // +2 for \n\n separator
+      const wouldOverflow = currentSize + blockSize > maxChars;
+      const keyAlreadyInChunk = keysInChunk.has(key);
+
+      // Decision logic:
+      // - If key already in chunk: MUST add (never split a key)
+      // - If new key and would overflow: flush first, then add
+      // - If new key and fits: add to current chunk
+
+      if (!keyAlreadyInChunk && wouldOverflow && currentChunk) {
+        // New key would overflow - flush current chunk first
+        chunks.push(currentChunk.trim());
+        currentChunk = '';
+        currentSize = 0;
+        keysInChunk = new Set();
+      }
+
+      // Add block to current chunk
+      if (currentChunk) currentChunk += '\n\n';
+      currentChunk += block;
+      currentSize += blockSize;
+      keysInChunk.add(key);
+    }
+
+    // Flush final chunk
+    if (currentChunk.trim()) {
+      chunks.push(currentChunk.trim());
+    }
+
+    return chunks.length > 0 ? chunks : [''];
+  };
+
+  // extractPaths() - extract unique file paths from search results
+  // Parses File: headers and returns deduplicated array of paths
+  globals.extractPaths = (searchResults) => {
+    const text = typeof searchResults === 'string' ? searchResults : JSON.stringify(searchResults);
+    const matches = text.match(/^File: ([^\n]+)/gm) || [];
+    const paths = matches.map(m => m.replace('File: ', '').trim());
+    return [...new Set(paths)];
+  };
+
   // Utility functions (pure, no async)
   globals.log = (message) => {
     // Collected by the runtime for the execution log

package/src/agent/dsl/validator.js

@@ -9,6 +9,81 @@
 import * as acorn from 'acorn';
 import * as walk from 'acorn-walk';
 
+/**
+ * Convert a character offset to line and column numbers.
+ * @param {string} code - The source code
+ * @param {number} offset - Character offset
+ * @returns {{ line: number, column: number }}
+ */
+function offsetToLineColumn(code, offset) {
+  const lines = code.split('\n');
+  let pos = 0;
+  for (let i = 0; i < lines.length; i++) {
+    const lineLength = lines[i].length + 1; // +1 for newline
+    if (pos + lineLength > offset) {
+      return { line: i + 1, column: offset - pos + 1 };
+    }
+    pos += lineLength;
+  }
+  return { line: lines.length, column: 1 };
+}
+
+/**
+ * Generate a code snippet with an arrow pointing to the error location.
+ * @param {string} code - The source code
+ * @param {number} line - Line number (1-based)
+ * @param {number} column - Column number (1-based)
+ * @param {number} contextLines - Number of lines to show before/after (default: 2)
+ * @returns {string}
+ */
+function generateErrorSnippet(code, line, column, contextLines = 2) {
+  const lines = code.split('\n');
+  const startLine = Math.max(0, line - 1 - contextLines);
+  const endLine = Math.min(lines.length, line + contextLines);
+
+  const snippetLines = [];
+  const lineNumWidth = String(endLine).length;
+
+  for (let i = startLine; i < endLine; i++) {
+    const lineNum = String(i + 1).padStart(lineNumWidth, ' ');
+    const marker = (i + 1 === line) ? '>' : ' ';
+    snippetLines.push(`${marker} ${lineNum} | ${lines[i]}`);
+
+    // Add arrow line for the error line
+    if (i + 1 === line) {
+      const padding = ' '.repeat(lineNumWidth + 4); // "  123 | " prefix
+      const arrow = ' '.repeat(Math.max(0, column - 1)) + '^';
+      snippetLines.push(`${padding}${arrow}`);
+    }
+  }
+
+  return snippetLines.join('\n');
+}
+
+/**
+ * Format an error message with code snippet.
+ * @param {string} message - The error message
+ * @param {string} code - The source code
+ * @param {number} offset - Character offset (optional, use -1 if line/column provided)
+ * @param {number} line - Line number (optional)
+ * @param {number} column - Column number (optional)
+ * @returns {string}
+ */
+function formatErrorWithSnippet(message, code, offset = -1, line = 0, column = 0) {
+  if (offset >= 0) {
+    const loc = offsetToLineColumn(code, offset);
+    line = loc.line;
+    column = loc.column;
+  }
+
+  if (line <= 0) {
+    return message;
+  }
+
+  const snippet = generateErrorSnippet(code, line, column);
+  return `${message}\n\n${snippet}`;
+}
+
 // Node types the LLM is allowed to generate
 const ALLOWED_NODE_TYPES = new Set([
   'Program',
@@ -102,16 +177,32 @@ export function validateDSL(code) {
       ecmaVersion: 2022,
       sourceType: 'script',
       allowReturnOutsideFunction: true,
+      locations: true, // Enable location tracking for better error messages
     });
   } catch (e) {
-    return { valid: false, errors: [`Syntax error: ${e.message}`] };
+    // Acorn errors have loc property with line/column
+    const line = e.loc?.line || 0;
+    const column = e.loc?.column ? e.loc.column + 1 : 0; // Acorn column is 0-based
+    const formattedError = formatErrorWithSnippet(
+      `Syntax error: ${e.message}`,
+      code,
+      -1,
+      line,
+      column
+    );
+    return { valid: false, errors: [formattedError] };
   }
 
+  // Helper to add error with code snippet
+  const addError = (message, position) => {
+    errors.push(formatErrorWithSnippet(message, code, position));
+  };
+
   // Step 2: Walk every node and validate
   walk.full(ast, (node) => {
     // Check node type against whitelist
     if (!ALLOWED_NODE_TYPES.has(node.type)) {
-      errors.push(`Blocked node type: ${node.type} at position ${node.start}`);
+      addError(`Blocked node type: ${node.type}`, node.start);
       return;
     }
 
@@ -121,7 +212,7 @@ export function validateDSL(code) {
         node.type === 'FunctionExpression') &&
       node.async
     ) {
-      errors.push(`Async functions are not allowed at position ${node.start}. Write synchronous code — the runtime handles async.`);
+      addError(`Async functions are not allowed. Write synchronous code — the runtime handles async.`, node.start);
     }
 
     // Block generator functions
@@ -129,19 +220,19 @@ export function validateDSL(code) {
       (node.type === 'FunctionExpression') &&
       node.generator
     ) {
-      errors.push(`Generator functions are not allowed at position ${node.start}`);
+      addError(`Generator functions are not allowed`, node.start);
     }
 
 
     // Check identifiers against blocklist
     if (node.type === 'Identifier' && BLOCKED_IDENTIFIERS.has(node.name)) {
-      errors.push(`Blocked identifier: '${node.name}' at position ${node.start}`);
+      addError(`Blocked identifier: '${node.name}'`, node.start);
     }
 
     // Check member expressions for blocked properties
     if (node.type === 'MemberExpression' && !node.computed) {
       if (node.property.type === 'Identifier' && BLOCKED_PROPERTIES.has(node.property.name)) {
-        errors.push(`Blocked property access: '.${node.property.name}' at position ${node.property.start}`);
+        addError(`Blocked property access: '.${node.property.name}'`, node.property.start);
       }
     }
 
@@ -149,7 +240,7 @@ export function validateDSL(code) {
     if (node.type === 'MemberExpression' && node.computed) {
       if (node.property.type === 'Literal' && typeof node.property.value === 'string') {
         if (BLOCKED_PROPERTIES.has(node.property.value) || BLOCKED_IDENTIFIERS.has(node.property.value)) {
-          errors.push(`Blocked computed property access: '["${node.property.value}"]' at position ${node.property.start}`);
+          addError(`Blocked computed property access: '["${node.property.value}"]'`, node.property.start);
         }
       }
     }
@@ -157,7 +248,7 @@ export function validateDSL(code) {
     // Block variable declarations named with blocked identifiers
     if (node.type === 'VariableDeclarator' && node.id.type === 'Identifier') {
       if (BLOCKED_IDENTIFIERS.has(node.id.name)) {
-        errors.push(`Cannot declare variable with blocked name: '${node.id.name}' at position ${node.id.start}`);
+        addError(`Cannot declare variable with blocked name: '${node.id.name}'`, node.id.start);
       }
     }
   });

package/src/tools/executePlan.js

@@ -16,8 +16,42 @@ import { glob } from 'glob';
 
 export { executePlanSchema };
 
+/**
+ * Decode common HTML entities that LLMs sometimes produce when generating code.
+ * This handles entities like &amp;&amp; → &&, &lt;= → <=, etc.
+ */
+function decodeHtmlEntities(str) {
+  const entities = {
+    '&amp;': '&',
+    '&lt;': '<',
+    '&gt;': '>',
+    '&quot;': '"',
+    '&apos;': "'",
+    '&#39;': "'",
+    '&#x27;': "'",
+  };
+
+  // Replace named/common entities
+  let result = str.replace(/&(?:amp|lt|gt|quot|apos|#39|#x27);/gi, (match) => {
+    return entities[match.toLowerCase()] || match;
+  });
+
+  // Handle numeric entities (decimal): &#60; → <
+  result = result.replace(/&#(\d+);/g, (match, dec) => {
+    return String.fromCharCode(parseInt(dec, 10));
+  });
+
+  // Handle numeric entities (hex): &#x3C; → <
+  result = result.replace(/&#x([0-9a-f]+);/gi, (match, hex) => {
+    return String.fromCharCode(parseInt(hex, 16));
+  });
+
+  return result;
+}
+
 /**
  * Strip markdown fences and XML tags that LLMs sometimes wrap code in.
+ * Also decodes HTML entities that may appear in XML-extracted code.
  */
 function stripCodeWrapping(code) {
   let s = String(code || '');
@@ -25,6 +59,8 @@ function stripCodeWrapping(code) {
   s = s.replace(/^```(?:javascript|js)?\n?/gm, '').replace(/```$/gm, '');
   // Strip XML-style tags: <execute_plan>, </execute_plan>, <code>, </code>
   s = s.replace(/<\/?(?:execute_plan|code)>/g, '');
+  // Decode HTML entities (e.g., &amp;&amp; → &&, &lt;= → <=)
+  s = decodeHtmlEntities(s);
   return s.trim();
 }
 
@@ -384,6 +420,51 @@ RULES REMINDER:
   });
 }
 
+// Delimiters for raw output passthrough - prevents LLM from processing/hallucinating large structured output
+export const RAW_OUTPUT_START = '<<<RAW_OUTPUT>>>';
+export const RAW_OUTPUT_END = '<<<END_RAW_OUTPUT>>>';
+
+/**
+ * Extract raw output blocks from tool result content and pass them through to the output buffer.
+ * This prevents parent LLMs from processing/hallucinating large structured output.
+ *
+ * @param {string} content - The tool result content
+ * @param {Object} [outputBuffer] - The output buffer to append extracted content to
+ * @returns {{ cleanedContent: string, extractedBlocks: string[] }} - Content with blocks removed and extracted blocks
+ */
+export function extractRawOutputBlocks(content, outputBuffer = null) {
+  if (typeof content !== 'string') {
+    return { cleanedContent: content, extractedBlocks: [] };
+  }
+
+  const extractedBlocks = [];
+  const regex = new RegExp(`${RAW_OUTPUT_START}\\n([\\s\\S]*?)\\n${RAW_OUTPUT_END}`, 'g');
+
+  let cleanedContent = content;
+  let match;
+
+  // Extract all blocks
+  while ((match = regex.exec(content)) !== null) {
+    extractedBlocks.push(match[1]);
+  }
+
+  // Remove the blocks and any following instruction line from content
+  cleanedContent = content
+    .replace(new RegExp(`${RAW_OUTPUT_START}\\n[\\s\\S]*?\\n${RAW_OUTPUT_END}`, 'g'), '')
+    .replace(/\n\n\[The above raw output \(\d+ chars\) will be passed directly to the final response\. Do NOT repeat, summarize, or modify it\.\]/g, '')
+    .trim();
+
+  // If output buffer provided, append extracted content
+  if (outputBuffer && extractedBlocks.length > 0) {
+    for (const block of extractedBlocks) {
+      outputBuffer.items = outputBuffer.items || [];
+      outputBuffer.items.push(block);
+    }
+  }
+
+  return { cleanedContent, extractedBlocks };
+}
+
 function formatSuccess(result, description, attempt, outputBuffer) {
   let output = '';
 
@@ -416,10 +497,12 @@ function formatSuccess(result, description, attempt, outputBuffer) {
     }
   }
 
-  // If output buffer has content, tell the LLM the data was written to direct output
+  // If output buffer has content, wrap it in delimiters for passthrough
+  // This prevents parent LLMs from processing/hallucinating the raw data
   if (outputBuffer && outputBuffer.items && outputBuffer.items.length > 0) {
-    const totalChars = outputBuffer.items.reduce((sum, item) => sum + item.length, 0);
-    output += `\n\n[Output buffer: ${totalChars} chars written via output(). This content will be appended directly to your response. Do NOT repeat or summarize it.]`;
+    const rawContent = outputBuffer.items.join('\n');
+    output += `\n\n${RAW_OUTPUT_START}\n${rawContent}\n${RAW_OUTPUT_END}`;
+    output += `\n\n[The above raw output (${rawContent.length} chars) will be passed directly to the final response. Do NOT repeat, summarize, or modify it.]`;
   }
 
   return output;