@probelabs/probe 0.6.0-rc254 → 0.6.0-rc256

package/build/agent/mcp/xmlBridge.js

@@ -6,6 +6,7 @@
 import { MCPClientManager } from './client.js';
 import { loadMCPConfiguration } from './config.js';
 import { processXmlWithThinkingAndRecovery } from '../xmlParsingUtils.js';
+import { unescapeXmlEntities } from '../../tools/common.js';
 
 /**
  * Convert MCP tool to XML definition format
@@ -111,7 +112,7 @@ export function parseXmlMcpToolCall(xmlString, mcpToolNames = []) {
       let match;
       while ((match = paramPattern.exec(content)) !== null) {
         const [, paramName, paramValue] = match;
-        params[paramName] = paramValue.trim();
+        params[paramName] = unescapeXmlEntities(paramValue.trim());
       }
     }
 
@@ -393,7 +394,7 @@ function parseNativeXmlTool(xmlString, toolName) {
     const [, paramName, paramValue] = match;
     // Skip if this is the params tag itself (MCP format)
     if (paramName !== 'params') {
-      params[paramName] = paramValue.trim();
+      params[paramName] = unescapeXmlEntities(paramValue.trim());
     }
   }
 

package/build/agent/schemaUtils.js

@@ -603,6 +603,17 @@ export function validateJsonResponse(response, options = {}) {
       }
     }
 
+    // Quick recovery: try to extract a valid JSON prefix before returning error.
+    // This handles the common case where AI returns valid JSON followed by markdown content,
+    // e.g., "Unexpected non-whitespace character after JSON at position 477" (issue #447).
+    const prefixResult = tryExtractValidJsonPrefix(responseToValidate, { schema, debug });
+    if (prefixResult && prefixResult.isValid) {
+      if (debug) {
+        console.log(`[DEBUG] JSON validation: Recovered valid JSON prefix (${prefixResult.extracted.length} chars) from response with trailing content`);
+      }
+      return { isValid: true, parsed: prefixResult.parsed };
+    }
+
     // Create enhanced error message with context snippet
     let enhancedError = error.message;
     let errorContext = null;
@@ -667,6 +678,122 @@ ${pointer} here`;
   }
 }
 
+/**
+ * Try to extract a valid JSON prefix from a string that has trailing non-JSON content.
+ * This handles the common case where an AI returns valid JSON followed by markdown text.
+ *
+ * Uses bracket-matching to find the end of the first complete JSON object/array,
+ * then validates the prefix with JSON.parse(). Optionally validates against a schema.
+ *
+ * @param {string} response - The full response string
+ * @param {Object} [options] - Options
+ * @param {Object|string} [options.schema] - JSON schema to validate against
+ * @param {boolean} [options.debug=false] - Enable debug logging
+ * @returns {Object|null} - {isValid: true, parsed: Object, extracted: string} or null if no valid prefix found
+ */
+export function tryExtractValidJsonPrefix(response, options = {}) {
+  const { schema = null, debug = false } = options;
+
+  if (!response || typeof response !== 'string') {
+    return null;
+  }
+
+  const trimmed = response.trim();
+  if (trimmed.length === 0) {
+    return null;
+  }
+
+  // Must start with { or [
+  const firstChar = trimmed[0];
+  if (firstChar !== '{' && firstChar !== '[') {
+    return null;
+  }
+
+  // First, check if the full string already parses - no need to extract prefix
+  try {
+    JSON.parse(trimmed);
+    return null; // Full string is valid JSON, no extraction needed
+  } catch {
+    // Expected - continue with prefix extraction
+  }
+
+  // Find the end of the first complete JSON object/array using bracket matching
+  const openChar = firstChar;
+  const closeChar = openChar === '{' ? '}' : ']';
+  let depth = 0;
+  let inString = false;
+  let escapeNext = false;
+  let endPos = -1;
+
+  for (let i = 0; i < trimmed.length; i++) {
+    const char = trimmed[i];
+
+    if (escapeNext) {
+      escapeNext = false;
+      continue;
+    }
+
+    if (char === '\\' && inString) {
+      escapeNext = true;
+      continue;
+    }
+
+    if (char === '"') {
+      inString = !inString;
+      continue;
+    }
+
+    if (inString) {
+      continue;
+    }
+
+    if (char === openChar) {
+      depth++;
+    } else if (char === closeChar) {
+      depth--;
+      if (depth === 0) {
+        endPos = i + 1;
+        break;
+      }
+    }
+  }
+
+  if (endPos <= 0 || endPos >= trimmed.length) {
+    return null; // No complete JSON found, or JSON spans the entire string
+  }
+
+  // Check that there's actual non-whitespace trailing content
+  const remainder = trimmed.substring(endPos).trim();
+  if (remainder.length === 0) {
+    return null; // Only whitespace after JSON, no trailing content issue
+  }
+
+  // Try to parse the prefix
+  const prefix = trimmed.substring(0, endPos);
+  try {
+    const parsed = JSON.parse(prefix);
+
+    if (debug) {
+      console.log(`[DEBUG] tryExtractValidJsonPrefix: Extracted valid JSON prefix (${prefix.length} chars), stripped trailing content (${remainder.length} chars)`);
+    }
+
+    // If schema provided, validate against it
+    if (schema) {
+      const schemaValidation = validateJsonResponse(prefix, { debug, schema });
+      if (!schemaValidation.isValid) {
+        if (debug) {
+          console.log(`[DEBUG] tryExtractValidJsonPrefix: Prefix is valid JSON but fails schema validation: ${schemaValidation.error}`);
+        }
+        return null;
+      }
+    }
+
+    return { isValid: true, parsed, extracted: prefix };
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Validate that the cleaned response is valid XML if expected
  * @param {string} response - Cleaned response

package/build/agent/tools.js

@@ -87,7 +87,6 @@ export function createTools(configOptions) {
   if (configOptions.allowEdit && isToolAllowed('create')) {
     tools.createTool = createTool(configOptions);
   }
-
   return tools;
 }
 
@@ -132,33 +131,6 @@ export {
   parseXmlToolCall
 };
 
-// Define the implement tool XML definition
-export const implementToolDefinition = `
-## implement
-Description: Implement a given task. Can modify files. Can be used ONLY if task explicitly stated that something requires modification or implementation.
-
-Parameters:
-- task: (required) The task description. Should be as detailed as possible, ideally pointing to exact files which needs be modified or created.
-- autoCommits: (optional) Whether to enable auto-commits in aider. Default is false.
-
-Usage Example:
-
-<examples>
-
-User: Can you implement a function to calculate Fibonacci numbers in main.js?
-<implement>
-<task>Implement a recursive function to calculate the nth Fibonacci number in main.js</task>
-</implement>
-
-User: Can you implement a function to calculate Fibonacci numbers in main.js with auto-commits?
-<implement>
-<task>Implement a recursive function to calculate the nth Fibonacci number in main.js</task>
-<autoCommits>true</autoCommits>
-</implement>
-
-</examples>
-`;
-
 // Define the listFiles tool XML definition
 export const listFilesToolDefinition = `
 ## listFiles

package/build/delegate.js

@@ -346,6 +346,7 @@ const DEFAULT_DELEGATE_TIMEOUT = parseInt(process.env.DELEGATE_TIMEOUT, 10) || 3
  * @param {Object} [options.tracer=null] - Telemetry tracer instance
  * @param {boolean} [options.enableBash=false] - Enable bash tool (inherited from parent)
  * @param {Object} [options.bashConfig] - Bash configuration (inherited from parent)
+ * @param {boolean} [options.allowEdit=false] - Allow edit/create tools (inherited from parent)
  * @param {string} [options.architectureFileName] - Architecture context filename to embed from repo root
  * @param {string} [options.promptType='code-researcher'] - Prompt type for the subagent
  * @param {Array<string>|null} [options.allowedTools] - Allowed tools for the subagent (null = default)
@@ -373,6 +374,7 @@ export async function delegate({
 	model = null,
 	enableBash = false,
 	bashConfig = null,
+	allowEdit = false,
 	architectureFileName = null,
 	promptType = 'code-researcher',
 	allowedTools = null,
@@ -462,6 +464,7 @@ export async function delegate({
 			model,      // Inherit from parent
 			enableBash, // Inherit from parent
 			bashConfig, // Inherit from parent
+			allowEdit,  // Inherit from parent
 			architectureFileName,
 			allowedTools,
 			disableTools,

package/build/index.js

@@ -51,6 +51,7 @@ import { searchTool, queryTool, extractTool, delegateTool, analyzeAllTool } from
 import { createExecutePlanTool, getExecutePlanToolDefinition, createCleanupExecutePlanTool, getCleanupExecutePlanToolDefinition } from './tools/executePlan.js';
 import { bashTool } from './tools/bash.js';
 import { editTool, createTool } from './tools/edit.js';
+import { FileTracker } from './tools/fileTracker.js';
 import { ProbeAgent } from './agent/ProbeAgent.js';
 import { SimpleTelemetry, SimpleAppTracer, initializeSimpleTelemetryFromOptions } from './agent/simpleTelemetry.js';
 import { listFilesToolInstance, searchFilesToolInstance } from './agent/probeTool.js';
@@ -98,6 +99,7 @@ export {
 	bashTool,
 	editTool,
 	createTool,
+	FileTracker,
 	// Export tool instances
 	listFilesToolInstance,
 	searchFilesToolInstance,

package/build/tools/common.js

@@ -269,6 +269,8 @@ User: Read file inside the dependency
 </extract>
 
 </examples>
+
+**Edit Integration:** The line numbers shown in extract output (e.g. "42 | code") can be used directly with the edit tool's start_line/end_line parameters for precise line-targeted editing. To edit inside a large function: extract it by symbol name first (e.g. "file.js#myFunction"), then use the line numbers from the output to make surgical edits with start_line/end_line.
 `;
 
 export const delegateToolDefinition = `
@@ -431,7 +433,7 @@ Capabilities:
 
 export const searchDescription = 'Search code in the repository. Free-form questions are accepted, but Elasticsearch-style keyword queries work best. Use this tool first for any code-related questions.';
 export const queryDescription = 'Search code using ast-grep structural pattern matching. Use this tool to find specific code structures like functions, classes, or methods.';
-export const extractDescription = 'Extract code blocks from files based on file paths and optional line numbers. Use this tool to see complete context after finding relevant files.';
+export const extractDescription = 'Extract code blocks from files based on file paths and optional line numbers. Use this tool to see complete context after finding relevant files. Line numbers from output can be used with edit start_line/end_line for precise editing.';
 export const delegateDescription = 'Automatically delegate big distinct tasks to specialized probe subagents within the agentic loop. Used by AI agents to break down complex requests into focused, parallel tasks.';
 export const bashDescription = 'Execute bash commands for system exploration and development tasks. Secure by default with built-in allow/deny lists.';
 export const analyzeAllDescription = 'Answer questions that require analyzing ALL matching data in the codebase. Use for aggregate questions like "What features exist?", "List all API endpoints", "Count TODO comments". The AI automatically plans the search strategy, processes all results via map-reduce, and synthesizes a comprehensive answer. WARNING: Slower than search - only use when you need complete coverage.';
@@ -450,7 +452,6 @@ export const DEFAULT_VALID_TOOLS = [
 	'useSkill',
 	'listFiles',
 	'searchFiles',
-	'implement',
 	'bash',
 	'task',
 	'attempt_completion'
@@ -496,9 +497,9 @@ function getValidParamsForTool(toolName) {
 
 	const schema = schemaMap[toolName];
 	if (!schema) {
-		// For tools without schema (listFiles, searchFiles, implement), return common params
+		// For tools without schema (listFiles, searchFiles), return common params
 		// These are the shared params that appear across multiple tools
-		return ['path', 'directory', 'pattern', 'recursive', 'includeHidden', 'task', 'files', 'autoCommits', 'result'];
+		return ['path', 'directory', 'pattern', 'recursive', 'includeHidden', 'task', 'files', 'result'];
 	}
 
 	// For attempt_completion, it has custom validation, just return 'result'
@@ -520,6 +521,23 @@ function getValidParamsForTool(toolName) {
 	return [];
 }
 
+/**
+ * Unescape standard XML entities in a string value.
+ * Order matters: &amp; must be decoded LAST to avoid double-decoding
+ * (e.g., &amp;lt; should become &lt;, not <).
+ * @param {string} str - The string to unescape
+ * @returns {string} The unescaped string
+ */
+export function unescapeXmlEntities(str) {
+	if (typeof str !== 'string') return str;
+	return str
+		.replace(/&lt;/g, '<')
+		.replace(/&gt;/g, '>')
+		.replace(/&quot;/g, '"')
+		.replace(/&apos;/g, "'")
+		.replace(/&amp;/g, '&');
+}
+
 // Simple XML parser helper - safer string-based approach
 export function parseXmlToolCall(xmlString, validTools = DEFAULT_VALID_TOOLS) {
 	// Find the tool that appears EARLIEST in the string
@@ -608,10 +626,10 @@ export function parseXmlToolCall(xmlString, validTools = DEFAULT_VALID_TOOLS) {
 			paramCloseIndex = nextTagIndex;
 		}
 
-		let paramValue = innerContent.substring(
+		let paramValue = unescapeXmlEntities(innerContent.substring(
 			paramOpenIndex + paramOpenTag.length,
 			paramCloseIndex
-		).trim();
+		).trim());
 
 		// Basic type inference (can be improved)
 		if (paramValue.toLowerCase() === 'true') {
@@ -632,7 +650,7 @@ export function parseXmlToolCall(xmlString, validTools = DEFAULT_VALID_TOOLS) {
 
 	// Special handling for attempt_completion - use entire inner content as result
 	if (toolName === 'attempt_completion') {
-		params['result'] = innerContent.trim();
+		params['result'] = unescapeXmlEntities(innerContent.trim());
 		// Remove command parameter if it was parsed by generic logic above (legacy compatibility)
 		if (params.command) {
 			delete params.command;
@@ -688,7 +706,7 @@ export function detectUnrecognizedToolCall(xmlString, validTools) {
 	// Common tool names that AI might try to use (these should appear as top-level tags)
 	const knownToolNames = [
 		'search', 'query', 'extract', 'listFiles', 'searchFiles',
-		'listSkills', 'useSkill', 'readImage', 'implement', 'edit',
+		'listSkills', 'useSkill', 'readImage', 'edit',
 		'create', 'delegate', 'bash', 'task', 'attempt_completion',
 		'attempt_complete', 'read_file', 'write_file', 'run_command',
 		'grep', 'find', 'cat', 'list_directory'