@gguf/coder 0.3.1 → 0.3.2

package/source/tool-calling/json-parser.ts

@@ -1,247 +0,0 @@
-import type {ToolCall} from '@/types/index';
-
-/**
- * Internal JSON tool call parser
- * Note: This is now an internal utility. Use tool-parser.ts for public API.
- */
-
-/**
- * Detects malformed JSON tool call attempts and returns error details
- * Returns null if no malformed tool calls detected
- */
-export function detectMalformedJSONToolCall(
-	content: string,
-): {error: string; examples: string} | null {
-	// Check for incomplete JSON structures
-	const patterns = [
-		{
-			// Incomplete JSON with name but missing arguments
-			regex: /\{\s*"name"\s*:\s*"[^"]+"\s*,?\s*\}/,
-			error: 'Incomplete tool call: missing "arguments" field',
-			hint: 'Tool calls must include both "name" and "arguments" fields',
-		},
-		{
-			// Incomplete JSON with arguments but missing name
-			regex: /\{\s*"arguments"\s*:\s*\{[^}]*\}\s*\}/,
-			error: 'Incomplete tool call: missing "name" field',
-			hint: 'Tool calls must include both "name" and "arguments" fields',
-		},
-		{
-			// Malformed arguments (not an object)
-			regex: /\{\s*"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*"[^"]*"\s*\}/,
-			error: 'Invalid tool call: "arguments" must be an object, not a string',
-			hint: 'Use {"name": "tool_name", "arguments": {...}} format',
-		},
-	];
-
-	for (const pattern of patterns) {
-		const match = content.match(pattern.regex);
-		if (match) {
-			return {
-				error: pattern.error,
-				examples: getCorrectJSONFormatExamples(pattern.hint),
-			};
-		}
-	}
-
-	return null;
-}
-
-/**
- * Generates correct format examples for JSON error messages
- */
-function getCorrectJSONFormatExamples(_specificHint: string): string {
-	return `Please use the native tool calling format provided by the system. The tools are already available to you - call them directly using the function calling interface.`;
-}
-
-/**
- * Parses JSON-formatted tool calls from content
- * This is an internal function - use tool-parser.ts for public API
- */
-export function parseJSONToolCalls(content: string): ToolCall[] {
-	const extractedCalls: ToolCall[] = [];
-	let trimmedContent = content.trim();
-
-	// Handle markdown code blocks
-	const codeBlockMatch = trimmedContent.match(
-		/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/,
-	);
-	if (codeBlockMatch && codeBlockMatch[1]) {
-		trimmedContent = codeBlockMatch[1].trim();
-	}
-
-	// Try to parse entire content as single JSON tool call
-	if (trimmedContent.startsWith('{') && trimmedContent.endsWith('}')) {
-		// Skip empty or nearly empty JSON objects
-		if (trimmedContent === '{}' || trimmedContent.replace(/\s/g, '') === '{}') {
-			return extractedCalls;
-		}
-
-		try {
-			const parsed = JSON.parse(trimmedContent) as {
-				name?: string;
-				arguments?: Record<string, unknown>;
-			};
-
-			if (parsed.name && parsed.arguments !== undefined) {
-				const toolCall = {
-					id: `call_${Date.now()}`,
-					function: {
-						name: parsed.name || '',
-						arguments: parsed.arguments || {},
-					},
-				};
-				extractedCalls.push(toolCall);
-				return extractedCalls;
-			}
-		} catch {
-			// Failed to parse - will be caught by malformed detection
-		}
-	}
-
-	// Look for standalone JSON blocks in the content (multiline without code blocks)
-	const jsonBlockRegex =
-		/\{\s*\n\s*"name":\s*"([^"]+)",\s*\n\s*"arguments":\s*\{[\s\S]*?\}\s*\n\s*\}/g;
-	let jsonMatch;
-	while ((jsonMatch = jsonBlockRegex.exec(content)) !== null) {
-		try {
-			const parsed = JSON.parse(jsonMatch[0]) as {
-				name?: string;
-				arguments?: Record<string, unknown>;
-			};
-			if (parsed.name && parsed.arguments !== undefined) {
-				const toolCall = {
-					id: `call_${Date.now()}_${extractedCalls.length}`,
-					function: {
-						name: parsed.name || '',
-						arguments: parsed.arguments || {},
-					},
-				};
-				extractedCalls.push(toolCall);
-			}
-		} catch {
-			// Failed to parse - will be caught by malformed detection
-		}
-	}
-
-	// Look for embedded tool calls using regex patterns
-	const toolCallPatterns = [
-		/\{"name":\s*"([^"]+)",\s*"arguments":\s*(\{[^}]*\})\}/g,
-		/\{"name":\s*"([^"]+)",\s*"arguments":\s*(\{[^}]+\})\}/g,
-		/\{"name":\s*"([^"]+)",\s*"arguments":\s*"([^"]+)"\}/g,
-	];
-
-	for (const pattern of toolCallPatterns) {
-		let match;
-		while ((match = pattern.exec(content)) !== null) {
-			const [, name, argsStr] = match;
-			try {
-				let args: Record<string, unknown> | string;
-				if (argsStr && argsStr.startsWith('{')) {
-					args = JSON.parse(argsStr || '{}') as Record<string, unknown>;
-				} else {
-					args = argsStr || '';
-				}
-				extractedCalls.push({
-					id: `call_${Date.now()}_${extractedCalls.length}`,
-					function: {
-						name: name || '',
-						arguments: args as Record<string, unknown>,
-					},
-				});
-			} catch {
-				// Failed to parse - will be caught by malformed detection
-			}
-		}
-	}
-
-	// Deduplicate identical tool calls
-	const uniqueCalls = deduplicateToolCalls(extractedCalls);
-
-	return uniqueCalls;
-}
-
-function deduplicateToolCalls(toolCalls: ToolCall[]): ToolCall[] {
-	const seen = new Set<string>();
-	const unique: ToolCall[] = [];
-
-	for (const call of toolCalls) {
-		// Create a hash of the function name and arguments for comparison
-		const hash = `${call.function.name}:${JSON.stringify(
-			call.function.arguments,
-		)}`;
-
-		if (!seen.has(hash)) {
-			seen.add(hash);
-			unique.push(call);
-		}
-		// Else: duplicate, skip it
-	}
-
-	return unique;
-}
-
-/**
- * Cleans content by removing tool call JSON blocks
- * This is an internal function - use tool-parser.ts for public API
- */
-export function cleanJSONToolCalls(
-	content: string,
-	toolCalls: ToolCall[],
-): string {
-	if (toolCalls.length === 0) return content;
-
-	let cleanedContent = content;
-
-	// Handle markdown code blocks that contain only tool calls
-	const codeBlockRegex = /```(?:json)?\s*\n?([\s\S]*?)\n?```/g;
-	cleanedContent = cleanedContent.replace(
-		codeBlockRegex,
-		(match, blockContent: string) => {
-			const trimmedBlock = blockContent.trim();
-
-			// Check if this block contains a tool call that we parsed
-			try {
-				const parsed = JSON.parse(trimmedBlock) as {
-					name?: string;
-					arguments?: unknown;
-				};
-				if (parsed.name && parsed.arguments !== undefined) {
-					// This code block contains only a tool call, remove the entire block
-					return '';
-				}
-			} catch {
-				// Not valid JSON, keep the code block
-			}
-
-			// Keep the code block as-is if it doesn't contain a tool call
-			return match;
-		},
-	);
-
-	// Remove JSON blocks that were parsed as tool calls (for non-code-block cases)
-	const toolCallPatterns = [
-		/\{\s*\n\s*"name":\s*"([^"]+)",\s*\n\s*"arguments":\s*\{[\s\S]*?\}\s*\n\s*\}/g, // Multiline JSON blocks
-		/\{"name":\s*"([^"]+)",\s*"arguments":\s*(\{[^}]*\})\}/g,
-		/\{"name":\s*"([^"]+)",\s*"arguments":\s*(\{[^}]+\})\}/g,
-		/\{"name":\s*"([^"]+)",\s*"arguments":\s*"([^"]+)"\}/g,
-	];
-
-	for (const pattern of toolCallPatterns) {
-		cleanedContent = cleanedContent.replace(pattern, '');
-	}
-
-	// Clean up whitespace artifacts left by removed tool calls
-	cleanedContent = cleanedContent
-		// Remove trailing whitespace from each line
-		.replace(/[ \t]+$/gm, '')
-		// Collapse multiple spaces (but not at start of line for indentation)
-		.replace(/([^ \t\n]) {2,}/g, '$1 ')
-		// Remove lines that are only whitespace
-		.replace(/^[ \t]+$/gm, '')
-		// Collapse 2+ consecutive blank lines to a single blank line
-		.replace(/\n{3,}/g, '\n\n')
-		.trim();
-
-	return cleanedContent;
-}

package/source/tool-calling/tool-parser.spec.ts

@@ -1,395 +0,0 @@
-import test from 'ava';
-import {parseToolCalls} from './tool-parser';
-
-console.log(`\ntool-parser.spec.ts`);
-
-// XML Parser Tests
-
-test('parseToolCalls: successfully parses valid XML tool call', t => {
-	const content = `
-<read_file>
-  <path>/path/to/file.txt</path>
-</read_file>
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 1);
-		t.is(result.toolCalls[0].function.name, 'read_file');
-		t.deepEqual(result.toolCalls[0].function.arguments, {
-			path: '/path/to/file.txt',
-		});
-	}
-});
-
-test('parseToolCalls: detects malformed XML with attribute syntax', t => {
-	const content = `
-<function=read_file>
-  <parameter=path>/path/to/file.txt</parameter>
-</function>
-
-I want to read the file at /path/to/file.txt
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.false(result.success);
-	if (!result.success) {
-		t.regex(result.error, /Invalid syntax/i);
-		t.regex(result.examples, /native tool calling/i);
-	}
-});
-
-test('parseToolCalls: handles multiple valid XML tool calls', t => {
-	const content = `
-<read_file>
-  <path>/path/to/file1.txt</path>
-</read_file>
-
-<create_file>
-  <path>/path/to/file2.txt</path>
-  <content>Hello world</content>
-</create_file>
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 2);
-		t.is(result.toolCalls[0].function.name, 'read_file');
-		t.is(result.toolCalls[1].function.name, 'create_file');
-	}
-});
-
-test('parseToolCalls: cleans XML tool calls from content', t => {
-	const content = `
-Here is some text before the tool call.
-
-<read_file>
-  <path>/path/to/file.txt</path>
-</read_file>
-
-And some text after.
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 1);
-		t.regex(result.cleanedContent, /Here is some text before/);
-		t.regex(result.cleanedContent, /And some text after/);
-		t.notRegex(result.cleanedContent, /<read_file>/);
-	}
-});
-
-// JSON Parser Tests
-
-test('parseToolCalls: successfully parses valid JSON tool call', t => {
-	const content = `
-{
-  "name": "read_file",
-  "arguments": {
-    "path": "/path/to/file.txt"
-  }
-}
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 1);
-		t.is(result.toolCalls[0].function.name, 'read_file');
-		t.deepEqual(result.toolCalls[0].function.arguments, {
-			path: '/path/to/file.txt',
-		});
-	}
-});
-
-test('parseToolCalls: detects malformed JSON missing arguments field', t => {
-	const content = `
-{
-  "name": "read_file"
-}
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.false(result.success);
-	if (!result.success) {
-		t.regex(result.error, /missing "arguments" field/i);
-		t.regex(result.examples, /native tool calling format/i);
-	}
-});
-
-test('parseToolCalls: detects malformed JSON missing name field', t => {
-	const content = `
-{
-  "arguments": {
-    "path": "/path/to/file.txt"
-  }
-}
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.false(result.success);
-	if (!result.success) {
-		t.regex(result.error, /missing "name" field/i);
-		t.regex(result.examples, /native tool calling format/i);
-	}
-});
-
-test('parseToolCalls: detects malformed JSON with string arguments', t => {
-	const content = `
-{
-  "name": "read_file",
-  "arguments": "/path/to/file.txt"
-}
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.false(result.success);
-	if (!result.success) {
-		t.regex(result.error, /"arguments" must be an object/i);
-		t.regex(result.examples, /native tool calling format/i);
-	}
-});
-
-test('parseToolCalls: handles JSON in markdown code blocks', t => {
-	const content = `
-\`\`\`json
-{
-  "name": "read_file",
-  "arguments": {
-    "path": "/path/to/file.txt"
-  }
-}
-\`\`\`
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 1);
-		t.is(result.toolCalls[0].function.name, 'read_file');
-	}
-});
-
-test('parseToolCalls: cleans JSON tool calls from content', t => {
-	const content = `
-Here is some text before.
-
-{
-  "name": "read_file",
-  "arguments": {
-    "path": "/path/to/file.txt"
-  }
-}
-
-And some text after.
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 1);
-		t.regex(result.cleanedContent, /Here is some text before/);
-		t.regex(result.cleanedContent, /And some text after/);
-		t.notRegex(result.cleanedContent, /"name":\s*"read_file"/);
-	}
-});
-
-// Priority Tests (XML should be tried first)
-
-test('parseToolCalls: prioritizes XML over JSON when both present', t => {
-	const content = `
-<read_file>
-  <path>/xml/path.txt</path>
-</read_file>
-
-{
-  "name": "read_file",
-  "arguments": {
-    "path": "/json/path.txt"
-  }
-}
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		// Should parse XML first and return immediately
-		t.is(result.toolCalls.length, 1);
-		t.is(result.toolCalls[0].function.name, 'read_file');
-		// Verify it's the XML call (xml_call_ prefix)
-		t.regex(result.toolCalls[0].id, /xml_call/);
-		t.is(
-			(result.toolCalls[0].function.arguments as {path: string}).path,
-			'/xml/path.txt',
-		);
-	}
-});
-
-// Edge Cases
-
-test('parseToolCalls: handles empty content', t => {
-	const result = parseToolCalls('');
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 0);
-		t.is(result.cleanedContent, '');
-	}
-});
-
-test('parseToolCalls: handles content with no tool calls', t => {
-	const content = 'Just some plain text without any tool calls.';
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 0);
-		t.is(result.cleanedContent, content);
-	}
-});
-
-test('parseToolCalls: handles empty JSON object', t => {
-	const content = '{}';
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 0);
-	}
-});
-
-test('parseToolCalls: deduplicates identical tool calls', t => {
-	const content = `
-{
-  "name": "read_file",
-  "arguments": {
-    "path": "/path/to/file.txt"
-  }
-}
-
-{
-  "name": "read_file",
-  "arguments": {
-    "path": "/path/to/file.txt"
-  }
-}
-  `;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		// Should deduplicate to single call
-		t.is(result.toolCalls.length, 1);
-	}
-});
-
-// Think Tag Tests (models like GLM-4 emit these for chain-of-thought)
-
-test('parseToolCalls: strips complete <think>...</think> tags', t => {
-	const content = `<think>
-Let me think about this...
-I should read the file first.
-</think>
-
-Here is my response to your question.`;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 0);
-		t.notRegex(result.cleanedContent, /<think>/);
-		t.notRegex(result.cleanedContent, /<\/think>/);
-		t.regex(result.cleanedContent, /Here is my response/);
-	}
-});
-
-test('parseToolCalls: strips orphaned closing </think> tags', t => {
-	const content = `</think>
-
-Here is my response after some thinking.`;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.notRegex(result.cleanedContent, /<\/think>/);
-		t.regex(result.cleanedContent, /Here is my response/);
-	}
-});
-
-test('parseToolCalls: strips incomplete opening <think> tags (streaming)', t => {
-	const content = `Here is my response.
-
-<think>
-I'm still thinking about this...`;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.notRegex(result.cleanedContent, /<think>/);
-		t.regex(result.cleanedContent, /Here is my response/);
-		// The incomplete thinking content should be removed
-		t.notRegex(result.cleanedContent, /still thinking/);
-	}
-});
-
-test('parseToolCalls: handles think tags with tool calls', t => {
-	const content = `<think>
-Let me analyze this request...
-I'll need to read the file first.
-</think>
-
-<read_file>
-  <path>/path/to/file.txt</path>
-</read_file>`;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.is(result.toolCalls.length, 1);
-		t.is(result.toolCalls[0].function.name, 'read_file');
-		t.notRegex(result.cleanedContent, /<think>/);
-		t.notRegex(result.cleanedContent, /<\/think>/);
-	}
-});
-
-test('parseToolCalls: handles case-insensitive think tags', t => {
-	const content = `<THINK>
-Some thinking...
-</THINK>
-
-<Think>
-More thinking...
-</Think>
-
-The actual response.`;
-
-	const result = parseToolCalls(content);
-
-	t.true(result.success);
-	if (result.success) {
-		t.notRegex(result.cleanedContent, /<think>/i);
-		t.notRegex(result.cleanedContent, /<\/think>/i);
-		t.regex(result.cleanedContent, /The actual response/);
-	}
-});