@gguf/coder 0.3.0 → 0.3.1

package/source/tool-calling/xml-parser.ts

@@ -0,0 +1,289 @@
+import type {ToolCall} from '@/types/index';
+
+interface ParsedToolCall {
+	toolName: string;
+	parameters: Record<string, unknown>;
+}
+
+/**
+ * Parses XML-formatted tool calls from non-function-calling models
+ * Expected format: <tool_name><param1>value1</param1><param2>value2</param2></tool_name>
+ */
+export class XMLToolCallParser {
+	private static readonly TOOL_CALL_REGEX = /<(\w+)>(.*?)<\/\1>/gs;
+	private static readonly PARAMETER_REGEX = /<(\w+)>(.*?)<\/\1>/gs;
+
+	/**
+	 * Extracts tool calls from text content containing XML-formatted tool calls
+	 */
+	static parseToolCalls(content: string): ParsedToolCall[] {
+		const toolCalls: ParsedToolCall[] = [];
+		let match;
+
+		// Handle content that might be wrapped in markdown code blocks
+		let processedContent = content;
+		const codeBlockMatch = content.match(/```(?:\w+)?\s*\n?([\s\S]*?)\n?```/);
+		if (codeBlockMatch && codeBlockMatch[1]) {
+			processedContent = codeBlockMatch[1].trim();
+		}
+
+		// Remove <tool_call> wrapper tags if present
+		processedContent = processedContent.replace(/<\/?tool_call>/g, '').trim();
+
+		// Find all tool call blocks
+		this.TOOL_CALL_REGEX.lastIndex = 0; // Reset regex state
+		while ((match = this.TOOL_CALL_REGEX.exec(processedContent)) !== null) {
+			const [fullMatch, toolName, innerXml] = match;
+
+			// Skip if this is a generic "tool_call" tag that slipped through
+			if (toolName === 'tool_call') {
+				continue;
+			}
+
+			// Validate that this is a properly formed tool call
+			if (!this.isValidToolCall(fullMatch, toolName)) {
+				continue;
+			}
+
+			const parameters = this.parseParameters(innerXml);
+
+			toolCalls.push({
+				toolName,
+				parameters,
+			});
+		}
+
+		return toolCalls;
+	}
+
+	/**
+	 * Validates that a matched string is a proper tool call
+	 * Rejects partial matches, malformed syntax, and invalid structures
+	 */
+	private static isValidToolCall(fullMatch: string, toolName: string): boolean {
+		// Reject common HTML tags that might be in model output
+		const htmlTags = [
+			'div',
+			'span',
+			'p',
+			'a',
+			'ul',
+			'ol',
+			'li',
+			'table',
+			'tr',
+			'td',
+			'th',
+			'thead',
+			'tbody',
+			'h1',
+			'h2',
+			'h3',
+			'h4',
+			'h5',
+			'h6',
+			'br',
+			'hr',
+			'strong',
+			'em',
+			'code',
+			'pre',
+			'blockquote',
+			'img',
+			'section',
+			'article',
+			'header',
+			'footer',
+			'nav',
+			'aside',
+		];
+		if (htmlTags.includes(toolName.toLowerCase())) {
+			return false;
+		}
+
+		// Check for malformed attribute-style syntax like <function=name> or <parameter=name>
+		if (fullMatch.includes('=')) {
+			return false;
+		}
+
+		// Check if the closing tag is properly formed
+		if (!fullMatch.endsWith(`</${toolName}>`)) {
+			return false;
+		}
+
+		// Extract inner content between opening and closing tags
+		const innerContent = fullMatch.substring(
+			toolName.length + 2,
+			fullMatch.length - (toolName.length + 3),
+		);
+
+		// Valid tool calls should contain parameter tags (other XML tags inside)
+		// This prevents matching on standalone parameter tags from malformed XML
+		const hasNestedTags = /<\w+>/.test(innerContent);
+
+		// Tool names should contain underscores (snake_case convention for tools)
+		// or be reasonably long compound words
+		const hasUnderscore = toolName.includes('_');
+
+		// Allow if it has nested tags OR follows naming convention
+		if (!hasNestedTags && !hasUnderscore) {
+			return false;
+		}
+
+		return true;
+	}
+
+	/**
+	 * Parses parameters from inner XML content
+	 */
+	private static parseParameters(innerXml: string): Record<string, unknown> {
+		const parameters: Record<string, unknown> = {};
+		let match;
+
+		// Reset regex state
+		this.PARAMETER_REGEX.lastIndex = 0;
+
+		while ((match = this.PARAMETER_REGEX.exec(innerXml)) !== null) {
+			const [, paramName, paramValue] = match;
+
+			// Trim whitespace from parameter value
+			const trimmedValue = paramValue.trim();
+
+			// Try to parse as JSON for complex objects/arrays
+			try {
+				parameters[paramName] = JSON.parse(trimmedValue) as unknown;
+			} catch {
+				// If not valid JSON, use as string (preserving internal whitespace/newlines)
+				parameters[paramName] = trimmedValue;
+			}
+		}
+
+		return parameters;
+	}
+
+	/**
+	 * Converts parsed tool calls to the standard ToolCall format
+	 */
+	static convertToToolCalls(parsedCalls: ParsedToolCall[]): ToolCall[] {
+		return parsedCalls.map((call, index) => ({
+			id: `xml_call_${index}`,
+			function: {
+				name: call.toolName,
+				arguments: call.parameters,
+			},
+		}));
+	}
+
+	/**
+	 * Removes XML tool call blocks from content, leaving only the text
+	 */
+	static removeToolCallsFromContent(content: string): string {
+		let cleanedContent = content;
+
+		// Remove all markdown code blocks that contain XML tool calls (using global flag)
+		cleanedContent = cleanedContent.replace(
+			/```(?:\w+)?\s*\n?([\s\S]*?)\n?```/g,
+			(match, blockContent: string) => {
+				if (blockContent) {
+					// Reset regex and check if this block contains XML tool calls
+					this.TOOL_CALL_REGEX.lastIndex = 0;
+					if (this.TOOL_CALL_REGEX.test(blockContent)) {
+						// This code block contains XML tool calls, remove it entirely
+						return '';
+					}
+				}
+				// Keep blocks that don't contain XML tool calls
+				return match;
+			},
+		);
+
+		// Remove XML tool calls that aren't in code blocks
+		this.TOOL_CALL_REGEX.lastIndex = 0;
+		cleanedContent = cleanedContent.replace(this.TOOL_CALL_REGEX, '');
+
+		// Remove any <tool_call> wrapper tags that may be left behind
+		cleanedContent = cleanedContent.replace(/<\/?tool_call>/g, '');
+
+		// Clean up whitespace artifacts left by removed tool calls
+		cleanedContent = cleanedContent
+			// Remove trailing whitespace from each line
+			.replace(/[ \t]+$/gm, '')
+			// Collapse multiple spaces (but not at start of line for indentation)
+			.replace(/([^ \t\n]) {2,}/g, '$1 ')
+			// Remove lines that are only whitespace
+			.replace(/^[ \t]+$/gm, '')
+			// Collapse 2+ consecutive blank lines to a single blank line
+			.replace(/\n{3,}/g, '\n\n')
+			.trim();
+
+		return cleanedContent;
+	}
+
+	/**
+	 * Checks if content contains XML-formatted tool calls
+	 */
+	static hasToolCalls(content: string): boolean {
+		// Use parseToolCalls with validation to ensure we only detect valid tool calls
+		const toolCalls = this.parseToolCalls(content);
+		return toolCalls.length > 0;
+	}
+
+	/**
+	 * Detects malformed XML tool call attempts and returns error details
+	 * Returns null if no malformed tool calls detected
+	 */
+	static detectMalformedToolCall(
+		content: string,
+	): {error: string; examples: string} | null {
+		// Common malformed patterns
+		const patterns = [
+			{
+				// [tool_use: name] or [Tool: name] syntax (common with some models like GLM)
+				regex: /\[(?:tool_use|Tool):\s*(\w+)\]/i,
+				error:
+					'Invalid syntax: [tool_use: name] or [Tool: name] format is not supported',
+			},
+			{
+				// <function=name> syntax
+				regex: /<function=(\w+)>/,
+				error: 'Invalid syntax: <function=name> is not supported',
+			},
+			{
+				// <parameter=name> syntax
+				regex: /<parameter=(\w+)>/,
+				error: 'Invalid syntax: <parameter=name> is not supported',
+			},
+			{
+				// Generic closing </parameter> without proper name
+				regex: /<parameter=\w+>[\s\S]*?<\/parameter>/,
+				error:
+					'Invalid parameter syntax: parameters must use named tags, not generic <parameter> wrapper',
+			},
+			{
+				// Generic closing </function> when <function=name> was used
+				regex: /<function=\w+>[\s\S]*?<\/function>/,
+				error:
+					'Invalid function syntax: use simple named tags, not <function=name> wrapper',
+			},
+		];
+
+		for (const pattern of patterns) {
+			const match = content.match(pattern.regex);
+			if (match) {
+				return {
+					error: pattern.error,
+					examples: this.getCorrectFormatExamples(),
+				};
+			}
+		}
+
+		return null;
+	}
+
+	/**
+	 * Generates correct format examples for error messages
+	 */
+	private static getCorrectFormatExamples(): string {
+		return `Please use the native tool calling format provided by the system. The tools are already available to you - call them directly using the function calling interface.`;
+	}
+}

package/source/tools/execute-bash.spec.tsx

@@ -0,0 +1,353 @@
+import test from 'ava';
+import {render} from 'ink-testing-library';
+import React from 'react';
+import {themes} from '../config/themes';
+import {ThemeContext} from '../hooks/useTheme';
+import {executeBashTool} from './execute-bash';
+
+// ============================================================================
+// Test Helpers
+// ============================================================================
+
+console.log(`\nexecute-bash.spec.tsx – ${React.version}`);
+
+// Create a mock theme provider for tests
+function TestThemeProvider({children}: {children: React.ReactNode}) {
+	const themeContextValue = {
+		currentTheme: 'tokyo-night' as const,
+		colors: themes['tokyo-night'].colors,
+		setCurrentTheme: () => {},
+	};
+
+	return (
+		<ThemeContext.Provider value={themeContextValue}>
+			{children}
+		</ThemeContext.Provider>
+	);
+}
+
+// ============================================================================
+// Tests for ExecuteBashFormatter Component Rendering
+// ============================================================================
+
+test('ExecuteBashFormatter renders with command', t => {
+	const formatter = executeBashTool.formatter;
+	if (!formatter) {
+		t.fail('Formatter is not defined');
+		return;
+	}
+
+	const element = formatter({command: 'echo "hello"'}, 'hello');
+	const {lastFrame} = render(<TestThemeProvider>{element}</TestThemeProvider>);
+
+	const output = lastFrame();
+	t.truthy(output);
+	t.regex(output!, /execute_bash/);
+	t.regex(output!, /echo "hello"/);
+});
+
+test('ExecuteBashFormatter displays output size', t => {
+	const formatter = executeBashTool.formatter;
+	if (!formatter) {
+		t.fail('Formatter is not defined');
+		return;
+	}
+
+	const result = 'test output';
+	const element = formatter({command: 'echo test'}, result);
+	const {lastFrame} = render(<TestThemeProvider>{element}</TestThemeProvider>);
+
+	const output = lastFrame();
+	t.truthy(output);
+	t.regex(output!, /characters/);
+	t.regex(output!, /tokens/);
+});
+
+test('ExecuteBashFormatter renders without result', t => {
+	const formatter = executeBashTool.formatter;
+	if (!formatter) {
+		t.fail('Formatter is not defined');
+		return;
+	}
+
+	const element = formatter({command: 'ls'});
+	const {lastFrame} = render(<TestThemeProvider>{element}</TestThemeProvider>);
+
+	const output = lastFrame();
+	t.truthy(output);
+	t.regex(output!, /execute_bash/);
+	t.regex(output!, /ls/);
+});
+
+test('ExecuteBashFormatter handles complex commands', t => {
+	const formatter = executeBashTool.formatter;
+	if (!formatter) {
+		t.fail('Formatter is not defined');
+		return;
+	}
+
+	const command = 'find . -name "*.ts" | grep -v node_modules';
+	const element = formatter({command}, 'file1.ts\nfile2.ts');
+	const {lastFrame} = render(<TestThemeProvider>{element}</TestThemeProvider>);
+
+	const output = lastFrame();
+	t.truthy(output);
+	t.regex(output!, /find/);
+});
+
+// ============================================================================
+// Tests for execute_bash Tool Handler - Basic Functionality
+// ============================================================================
+
+test('execute_bash runs simple echo command', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{command: 'echo "test output"'},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.true(result.includes('test output'));
+});
+
+test('execute_bash returns output from ls command', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{command: 'ls'},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.is(typeof result, 'string');
+});
+
+test('execute_bash handles command with pipes', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'echo "line1\nline2\nline3" | grep line2',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.true(result.includes('line2'));
+	t.false(result.includes('line1'));
+});
+
+test('execute_bash handles command with redirects', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'echo "test" 2>&1',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.true(result.includes('test'));
+});
+
+test('execute_bash preserves multiline output', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'echo "line1"; echo "line2"; echo "line3"',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.true(result.includes('line1'));
+	t.true(result.includes('line2'));
+	t.true(result.includes('line3'));
+});
+
+// ============================================================================
+// Tests for execute_bash Tool Handler - Error Handling
+// ============================================================================
+
+test('execute_bash captures stderr output', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'echo "error message" >&2',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	// Should include STDERR label when stderr is present
+	t.true(result.includes('STDERR') || result.includes('error message'));
+});
+
+test('execute_bash handles command not found', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'nonexistentcommand12345',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	// Should capture the error output
+	t.true(
+		result.includes('not found') ||
+			result.includes('command not found') ||
+			result.includes('STDERR'),
+	);
+});
+
+test('execute_bash handles syntax errors', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'echo "unclosed quote',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	// Should capture the syntax error
+	t.is(typeof result, 'string');
+});
+
+// ============================================================================
+// Tests for execute_bash Tool Handler - Output Truncation
+// ============================================================================
+
+test('execute_bash truncates long output to 2000 characters', async t => {
+	// Generate output longer than 2000 characters
+	// Use POSIX-compatible syntax (seq instead of bash brace expansion)
+	const longCommand =
+		'seq 1 100 | while read i; do echo "This is a long line of text that repeats many times"; done';
+	const result = await executeBashTool.tool.execute!(
+		{command: longCommand},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	// Should be truncated to around 2000 characters
+	t.true(
+		result.length <= 2100,
+		`Output length ${result.length} should be <= 2100`,
+	);
+	// Should include truncation message
+	t.true(result.includes('[Output truncated'));
+});
+
+test('execute_bash does not truncate short output', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'echo "short output"',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.false(result.includes('[Output truncated'));
+	t.true(result.includes('short output'));
+});
+
+test('execute_bash returns plain string not JSON', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{command: 'echo "test"'},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.is(typeof result, 'string');
+	// Should NOT be JSON with fullOutput and llmContext
+	t.false(result.includes('fullOutput'));
+	t.false(result.includes('llmContext'));
+});
+
+// ============================================================================
+// Tests for execute_bash Tool Handler - Special Characters
+// ============================================================================
+
+test('execute_bash handles special characters in output', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'echo "special: $@#%^&*()"',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.true(result.includes('special'));
+});
+
+test('execute_bash handles quotes in commands', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'echo "He said \\"hello\\""',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.true(result.includes('said'));
+});
+
+test('execute_bash handles newlines in command', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{
+			command: 'echo "line1\nline2"',
+		},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.is(typeof result, 'string');
+});
+
+// ============================================================================
+// Tests for execute_bash Tool Configuration
+// ============================================================================
+
+test('execute_bash tool has correct name', t => {
+	t.is(executeBashTool.name, 'execute_bash');
+});
+
+test('execute_bash tool requires confirmation', t => {
+	// Execute bash should require confirmation for security
+	t.not(executeBashTool.tool.needsApproval, false);
+});
+
+test('execute_bash tool has handler function', t => {
+	t.is(typeof executeBashTool.tool.execute, 'function');
+});
+
+test('execute_bash tool has formatter function', t => {
+	t.is(typeof executeBashTool.formatter, 'function');
+});
+
+// ============================================================================
+// Tests for execute_bash Tool Handler - Edge Cases
+// ============================================================================
+
+test('execute_bash handles empty command output', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{command: 'true'},
+		{toolCallId: 'test', messages: []},
+	);
+
+	// Empty output returns empty string, which is falsy but valid
+	t.is(typeof result, 'string');
+	// Empty output is still a valid string
+	t.true(result.length >= 0);
+});
+
+test('execute_bash handles commands with no output', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{command: ':'},
+		{toolCallId: 'test', messages: []},
+	);
+
+	// Empty output returns empty string, which is falsy but valid
+	t.is(typeof result, 'string');
+});
+
+test('execute_bash handles whitespace-only output', async t => {
+	const result = await executeBashTool.tool.execute!(
+		{command: 'echo "   "'},
+		{toolCallId: 'test', messages: []},
+	);
+
+	t.truthy(result);
+	t.is(typeof result, 'string');
+});