@vybestack/llxprt-code-core 0.5.0-nightly.251102.f115237d → 0.5.0-nightly.251104.319bfefc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -14
- package/dist/src/core/client.js +17 -3
- package/dist/src/core/client.js.map +1 -1
- package/dist/src/core/geminiChat.d.ts +19 -2
- package/dist/src/core/geminiChat.js +153 -56
- package/dist/src/core/geminiChat.js.map +1 -1
- package/dist/src/core/subagentOrchestrator.js +8 -4
- package/dist/src/core/subagentOrchestrator.js.map +1 -1
- package/dist/src/core/turn.d.ts +6 -2
- package/dist/src/core/turn.js +6 -0
- package/dist/src/core/turn.js.map +1 -1
- package/dist/src/debug/ConfigurationManager.js +6 -0
- package/dist/src/debug/ConfigurationManager.js.map +1 -1
- package/dist/src/parsers/TextToolCallParser.d.ts +17 -1
- package/dist/src/parsers/TextToolCallParser.js +540 -186
- package/dist/src/parsers/TextToolCallParser.js.map +1 -1
- package/dist/src/providers/LoggingProviderWrapper.d.ts +1 -0
- package/dist/src/providers/LoggingProviderWrapper.js +89 -4
- package/dist/src/providers/LoggingProviderWrapper.js.map +1 -1
- package/dist/src/providers/gemini/GeminiProvider.js +5 -6
- package/dist/src/providers/gemini/GeminiProvider.js.map +1 -1
- package/dist/src/providers/openai/OpenAIProvider.js +18 -4
- package/dist/src/providers/openai/OpenAIProvider.js.map +1 -1
- package/dist/src/runtime/AgentRuntimeContext.d.ts +1 -1
- package/dist/src/runtime/createAgentRuntimeContext.js +8 -1
- package/dist/src/runtime/createAgentRuntimeContext.js.map +1 -1
- package/dist/src/services/complexity-analyzer.d.ts +4 -1
- package/dist/src/services/complexity-analyzer.js +73 -21
- package/dist/src/services/complexity-analyzer.js.map +1 -1
- package/dist/src/services/shellExecutionService.js +22 -12
- package/dist/src/services/shellExecutionService.js.map +1 -1
- package/dist/src/settings/SettingsService.js +7 -0
- package/dist/src/settings/SettingsService.js.map +1 -1
- package/dist/src/telemetry/sdk.js +2 -2
- package/dist/src/telemetry/sdk.js.map +1 -1
- package/dist/src/tools/edit.js +27 -7
- package/dist/src/tools/edit.js.map +1 -1
- package/dist/src/tools/fuzzy-replacer.d.ts +61 -0
- package/dist/src/tools/fuzzy-replacer.js +450 -0
- package/dist/src/tools/fuzzy-replacer.js.map +1 -0
- package/dist/src/tools/mcp-tool.d.ts +1 -1
- package/dist/src/tools/mcp-tool.js +1 -1
- package/dist/src/tools/tool-registry.js +1 -1
- package/dist/src/tools/tool-registry.js.map +1 -1
- package/dist/src/utils/editor.js +10 -8
- package/dist/src/utils/editor.js.map +1 -1
- package/package.json +1 -1
|
@@ -4,29 +4,8 @@
|
|
|
4
4
|
* SPDX-License-Identifier: Apache-2.0
|
|
5
5
|
*/
|
|
6
6
|
export class GemmaToolCallParser {
|
|
7
|
-
|
|
8
|
-
patterns = [
|
|
9
|
-
// Format 1: [TOOL_REQUEST] toolName {args} [TOOL_REQUEST_END]
|
|
10
|
-
/\[TOOL_REQUEST\]\s*(\w+)\s+({.*?})\s*\[TOOL_REQUEST_END\]/gs,
|
|
11
|
-
// Format 2: ✦ tool_call: toolName for key value pairs (more specific to avoid false positives)
|
|
12
|
-
/✦\s*tool_call:\s*(\w+)\s+for\s+(.+?)(?=\n|✦|$)/gs,
|
|
13
|
-
// Format 3: JSON object with name/arguments followed by [END_TOOL_REQUEST]
|
|
14
|
-
/(\d+\s+)?{"name":\s*"(\w+)",\s*"arguments":\s*({.*?})}\s*(?:\n\s*\d+\s+)?\[END_TOOL_REQUEST\]/gs,
|
|
15
|
-
// Format 4: Hermes format with <tool_call> tags
|
|
16
|
-
/<tool_call>\s*({.*?"name":\s*"(\w+)".*?})\s*<\/tool_call>/gs,
|
|
17
|
-
// Format 5: XML with <invoke> tags (Claude-style)
|
|
18
|
-
/<invoke\s+name="(\w+)">(.*?)<\/invoke>/gs,
|
|
19
|
-
// Format 6: Generic XML tool format
|
|
20
|
-
/<tool>\s*<name>(\w+)<\/name>\s*<arguments>(.*?)<\/arguments>\s*<\/tool>/gs,
|
|
21
|
-
// Format 7: <use toolName with key="value" ...>
|
|
22
|
-
/<use\s+([a-zA-Z0-9_.-]+)([^>]*)>/gs,
|
|
23
|
-
// Format 8: <use_toolName with key="value" ...>
|
|
24
|
-
/<use_([a-zA-Z0-9_.-]+)([^>]*)>/gs,
|
|
25
|
-
];
|
|
7
|
+
keyValuePattern = /✦\s*tool_call:\s*([A-Za-z0-9_.-]+)\s+for\s+([^\n✦]*)/g;
|
|
26
8
|
parse(content) {
|
|
27
|
-
const toolCalls = [];
|
|
28
|
-
let cleanedContent = content;
|
|
29
|
-
const matches = [];
|
|
30
9
|
// Quick check: if content doesn't contain any tool call markers, return early
|
|
31
10
|
if (!content.includes('[TOOL_REQUEST') &&
|
|
32
11
|
!content.includes('tool_call:') &&
|
|
@@ -38,186 +17,445 @@ export class GemmaToolCallParser {
|
|
|
38
17
|
!content.includes('<use ')) {
|
|
39
18
|
return { cleanedContent: content, toolCalls: [] };
|
|
40
19
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
const args = this.parseKeyValuePairs(argsStr);
|
|
49
|
-
matches.push({ fullMatch, toolName, args });
|
|
50
|
-
}
|
|
51
|
-
else if (pattern === this.patterns[2]) {
|
|
52
|
-
// Format 3: JSON object format {"name": "tool", "arguments": {...}}
|
|
53
|
-
const [fullMatch, , toolName, jsonArgs] = match;
|
|
54
|
-
matches.push({ fullMatch, toolName, args: jsonArgs });
|
|
55
|
-
}
|
|
56
|
-
else if (pattern === this.patterns[3]) {
|
|
57
|
-
// Format 4: Hermes format <tool_call>{"arguments": {...}, "name": "tool"}</tool_call>
|
|
58
|
-
const [fullMatch, hermesJson, toolName] = match;
|
|
59
|
-
try {
|
|
60
|
-
const parsed = JSON.parse(hermesJson);
|
|
61
|
-
matches.push({
|
|
62
|
-
fullMatch,
|
|
63
|
-
toolName,
|
|
64
|
-
args: JSON.stringify(parsed.arguments || {}),
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
catch (error) {
|
|
68
|
-
console.error(`[GemmaToolCallParser] Failed to parse Hermes format: ${error}`);
|
|
69
|
-
// Still need to track the match to remove it from content
|
|
70
|
-
matches.push({ fullMatch, toolName: '', args: '' });
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
else if (pattern === this.patterns[4]) {
|
|
74
|
-
// Format 5: XML with <invoke> tags (Claude-style)
|
|
75
|
-
const [fullMatch, toolName, xmlContent] = match;
|
|
76
|
-
matches.push({ fullMatch, toolName, args: xmlContent });
|
|
77
|
-
}
|
|
78
|
-
else if (pattern === this.patterns[5]) {
|
|
79
|
-
// Format 6: Generic XML tool format
|
|
80
|
-
const [fullMatch, toolName, xmlArgs] = match;
|
|
81
|
-
matches.push({ fullMatch, toolName, args: xmlArgs });
|
|
82
|
-
}
|
|
83
|
-
else if (pattern === this.patterns[6] ||
|
|
84
|
-
pattern === this.patterns[7]) {
|
|
85
|
-
// Format 7: <use toolName ...>
|
|
86
|
-
const [fullMatch, toolNameRaw, attributeText] = match;
|
|
87
|
-
const toolName = toolNameRaw?.trim();
|
|
88
|
-
const args = {};
|
|
89
|
-
const attributeMatches = Array.from(attributeText.matchAll(/([a-zA-Z0-9_.-]+)\s*=\s*"(.*?)"/g));
|
|
90
|
-
for (const [, key, rawValue] of attributeMatches) {
|
|
91
|
-
if (!key)
|
|
92
|
-
continue;
|
|
93
|
-
const trimmedValue = rawValue.trim();
|
|
94
|
-
if (trimmedValue.startsWith('{') || trimmedValue.startsWith('[')) {
|
|
95
|
-
try {
|
|
96
|
-
args[key] = JSON.parse(trimmedValue);
|
|
97
|
-
continue;
|
|
98
|
-
}
|
|
99
|
-
catch (_error) {
|
|
100
|
-
// fall back to string
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
if (/^-?\d+(\.\d+)?$/.test(trimmedValue)) {
|
|
104
|
-
args[key] = Number(trimmedValue);
|
|
105
|
-
}
|
|
106
|
-
else if (trimmedValue.toLowerCase() === 'true' ||
|
|
107
|
-
trimmedValue.toLowerCase() === 'false') {
|
|
108
|
-
args[key] = trimmedValue.toLowerCase() === 'true';
|
|
109
|
-
}
|
|
110
|
-
else {
|
|
111
|
-
args[key] = trimmedValue;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
matches.push({ fullMatch, toolName, args });
|
|
115
|
-
}
|
|
116
|
-
else {
|
|
117
|
-
// Format 1: tool name followed by JSON arguments
|
|
118
|
-
const [fullMatch, toolName, jsonArgs] = match;
|
|
119
|
-
matches.push({ fullMatch, toolName, args: jsonArgs });
|
|
120
|
-
}
|
|
20
|
+
const matches = this.collectMatches(content);
|
|
21
|
+
const toolCalls = [];
|
|
22
|
+
const ranges = [];
|
|
23
|
+
for (const match of matches) {
|
|
24
|
+
ranges.push({ start: match.start, end: match.end });
|
|
25
|
+
if (!match.toolName) {
|
|
26
|
+
continue;
|
|
121
27
|
}
|
|
122
|
-
|
|
123
|
-
|
|
28
|
+
const parsedArgs = this.normalizeArguments(match.rawArgs, match.toolName, match.fullMatch);
|
|
29
|
+
if (!parsedArgs) {
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
toolCalls.push({
|
|
33
|
+
name: match.toolName,
|
|
34
|
+
arguments: parsedArgs,
|
|
35
|
+
});
|
|
124
36
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
37
|
+
const withoutMatches = this.removeMatchedRanges(content, ranges);
|
|
38
|
+
const cleanedContent = this.postProcessCleanedContent(withoutMatches);
|
|
39
|
+
return { cleanedContent, toolCalls };
|
|
40
|
+
}
|
|
41
|
+
collectMatches(content) {
|
|
42
|
+
const matches = [
|
|
43
|
+
...this.findBracketToolRequests(content),
|
|
44
|
+
...this.findJsonToolRequests(content),
|
|
45
|
+
...this.findHermesToolRequests(content),
|
|
46
|
+
...this.findInvokeToolRequests(content),
|
|
47
|
+
...this.findGenericXmlToolRequests(content),
|
|
48
|
+
...this.findUseToolRequests(content),
|
|
49
|
+
...this.findUseUnderscoreToolRequests(content),
|
|
50
|
+
...this.findKeyValueToolRequests(content),
|
|
51
|
+
];
|
|
52
|
+
return matches.sort((a, b) => a.start - b.start);
|
|
53
|
+
}
|
|
54
|
+
findBracketToolRequests(content) {
|
|
55
|
+
const matches = [];
|
|
56
|
+
const startMarker = '[TOOL_REQUEST]';
|
|
57
|
+
const endMarker = '[TOOL_REQUEST_END]';
|
|
58
|
+
let searchIndex = 0;
|
|
59
|
+
while (searchIndex < content.length) {
|
|
60
|
+
const start = content.indexOf(startMarker, searchIndex);
|
|
61
|
+
if (start === -1)
|
|
62
|
+
break;
|
|
63
|
+
const afterStart = start + startMarker.length;
|
|
64
|
+
const endMarkerIndex = content.indexOf(endMarker, afterStart);
|
|
65
|
+
if (endMarkerIndex === -1)
|
|
66
|
+
break;
|
|
67
|
+
const segment = content.slice(afterStart, endMarkerIndex);
|
|
68
|
+
const toolNameMatch = segment.match(/^\s*([^\s{]+)\s+/);
|
|
69
|
+
if (!toolNameMatch) {
|
|
70
|
+
searchIndex = endMarkerIndex + endMarker.length;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const toolName = toolNameMatch[1];
|
|
74
|
+
const braceOffset = segment.indexOf('{', toolNameMatch[0].length);
|
|
75
|
+
if (braceOffset === -1) {
|
|
76
|
+
searchIndex = endMarkerIndex + endMarker.length;
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
const jsonStart = afterStart + braceOffset;
|
|
80
|
+
const jsonSegment = this.extractBalancedSegment(content, jsonStart, '{', '}');
|
|
81
|
+
if (!jsonSegment || jsonSegment.endIndex > endMarkerIndex) {
|
|
82
|
+
searchIndex = endMarkerIndex + endMarker.length;
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
const fullEnd = endMarkerIndex + endMarker.length;
|
|
86
|
+
matches.push({
|
|
87
|
+
start,
|
|
88
|
+
end: fullEnd,
|
|
89
|
+
toolName,
|
|
90
|
+
rawArgs: jsonSegment.segment,
|
|
91
|
+
fullMatch: content.slice(start, fullEnd),
|
|
92
|
+
});
|
|
93
|
+
searchIndex = fullEnd;
|
|
94
|
+
}
|
|
95
|
+
return matches;
|
|
96
|
+
}
|
|
97
|
+
findJsonToolRequests(content) {
|
|
98
|
+
const matches = [];
|
|
99
|
+
const marker = '{"name":';
|
|
100
|
+
const endMarker = '[END_TOOL_REQUEST]';
|
|
101
|
+
let searchIndex = 0;
|
|
102
|
+
while (searchIndex < content.length) {
|
|
103
|
+
const candidateIndex = content.indexOf(marker, searchIndex);
|
|
104
|
+
if (candidateIndex === -1) {
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
107
|
+
let startIndex = candidateIndex;
|
|
108
|
+
let backPointer = candidateIndex;
|
|
109
|
+
while (backPointer > 0 && /\s/.test(content.charAt(backPointer - 1))) {
|
|
110
|
+
backPointer--;
|
|
111
|
+
}
|
|
112
|
+
let digitPointer = backPointer;
|
|
113
|
+
while (digitPointer > 0 && /\d/.test(content.charAt(digitPointer - 1))) {
|
|
114
|
+
digitPointer--;
|
|
115
|
+
}
|
|
116
|
+
if (digitPointer < backPointer) {
|
|
117
|
+
startIndex = digitPointer;
|
|
118
|
+
}
|
|
119
|
+
const jsonSegment = this.extractBalancedSegment(content, candidateIndex, '{', '}');
|
|
120
|
+
if (!jsonSegment) {
|
|
121
|
+
searchIndex = candidateIndex + marker.length;
|
|
131
122
|
continue;
|
|
132
123
|
}
|
|
133
124
|
try {
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
125
|
+
const parsed = JSON.parse(jsonSegment.segment);
|
|
126
|
+
const toolName = String(parsed.name ?? '');
|
|
127
|
+
const argsText = JSON.stringify(parsed.arguments ?? {});
|
|
128
|
+
const endMarkerIndex = content.indexOf(endMarker, jsonSegment.endIndex);
|
|
129
|
+
if (toolName && endMarkerIndex !== -1) {
|
|
130
|
+
const fullEnd = endMarkerIndex + endMarker.length;
|
|
131
|
+
matches.push({
|
|
132
|
+
start: startIndex,
|
|
133
|
+
end: fullEnd,
|
|
134
|
+
toolName,
|
|
135
|
+
rawArgs: argsText,
|
|
136
|
+
fullMatch: content.slice(startIndex, fullEnd),
|
|
137
|
+
});
|
|
138
|
+
searchIndex = fullEnd;
|
|
139
|
+
continue;
|
|
149
140
|
}
|
|
150
|
-
toolCalls.push({
|
|
151
|
-
name: toolName,
|
|
152
|
-
arguments: parsedArgs,
|
|
153
|
-
});
|
|
154
141
|
}
|
|
155
142
|
catch (error) {
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
143
|
+
console.error(`[GemmaToolCallParser] Failed to parse structured tool call JSON: ${error}`);
|
|
144
|
+
}
|
|
145
|
+
searchIndex = candidateIndex + marker.length;
|
|
146
|
+
}
|
|
147
|
+
return matches;
|
|
148
|
+
}
|
|
149
|
+
findHermesToolRequests(content) {
|
|
150
|
+
const matches = [];
|
|
151
|
+
const startTag = '<tool_call>';
|
|
152
|
+
const endTag = '</tool_call>';
|
|
153
|
+
let searchIndex = 0;
|
|
154
|
+
while (searchIndex < content.length) {
|
|
155
|
+
const start = content.indexOf(startTag, searchIndex);
|
|
156
|
+
if (start === -1)
|
|
157
|
+
break;
|
|
158
|
+
const end = content.indexOf(endTag, start + startTag.length);
|
|
159
|
+
if (end === -1)
|
|
160
|
+
break;
|
|
161
|
+
const jsonText = content.slice(start + startTag.length, end).trim();
|
|
162
|
+
let toolName = '';
|
|
163
|
+
let args = '{}';
|
|
164
|
+
try {
|
|
165
|
+
const parsed = JSON.parse(jsonText);
|
|
166
|
+
toolName = String(parsed.name ?? '');
|
|
167
|
+
args = JSON.stringify(parsed.arguments ?? {});
|
|
168
|
+
}
|
|
169
|
+
catch (error) {
|
|
170
|
+
console.error(`[GemmaToolCallParser] Failed to parse Hermes format: ${error}`);
|
|
171
|
+
}
|
|
172
|
+
const fullEnd = end + endTag.length;
|
|
173
|
+
matches.push({
|
|
174
|
+
start,
|
|
175
|
+
end: fullEnd,
|
|
176
|
+
toolName,
|
|
177
|
+
rawArgs: args,
|
|
178
|
+
fullMatch: content.slice(start, fullEnd),
|
|
179
|
+
});
|
|
180
|
+
searchIndex = fullEnd;
|
|
181
|
+
}
|
|
182
|
+
return matches;
|
|
183
|
+
}
|
|
184
|
+
findInvokeToolRequests(content) {
|
|
185
|
+
const matches = [];
|
|
186
|
+
const tagPrefix = '<invoke';
|
|
187
|
+
const closing = '</invoke>';
|
|
188
|
+
let searchIndex = 0;
|
|
189
|
+
while (searchIndex < content.length) {
|
|
190
|
+
const start = content.indexOf(tagPrefix, searchIndex);
|
|
191
|
+
if (start === -1) {
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
const tagEnd = this.findTagClose(content, start + tagPrefix.length);
|
|
195
|
+
if (tagEnd === -1) {
|
|
196
|
+
break;
|
|
197
|
+
}
|
|
198
|
+
const header = content.slice(start + tagPrefix.length, tagEnd);
|
|
199
|
+
const attributes = this.parseAttributeArguments(header);
|
|
200
|
+
const toolNameValue = attributes.name;
|
|
201
|
+
const toolName = typeof toolNameValue === 'string' ? toolNameValue.trim() : '';
|
|
202
|
+
const bodyStart = tagEnd + 1;
|
|
203
|
+
const closingIndex = content.indexOf(closing, bodyStart);
|
|
204
|
+
if (!toolName || closingIndex === -1) {
|
|
205
|
+
searchIndex = bodyStart;
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
const fullEnd = closingIndex + closing.length;
|
|
209
|
+
matches.push({
|
|
210
|
+
start,
|
|
211
|
+
end: fullEnd,
|
|
212
|
+
toolName,
|
|
213
|
+
rawArgs: content.slice(bodyStart, closingIndex),
|
|
214
|
+
fullMatch: content.slice(start, fullEnd),
|
|
215
|
+
});
|
|
216
|
+
searchIndex = fullEnd;
|
|
217
|
+
}
|
|
218
|
+
return matches;
|
|
219
|
+
}
|
|
220
|
+
findGenericXmlToolRequests(content) {
|
|
221
|
+
const matches = [];
|
|
222
|
+
const startTag = '<tool>';
|
|
223
|
+
const endTag = '</tool>';
|
|
224
|
+
let searchIndex = 0;
|
|
225
|
+
while (searchIndex < content.length) {
|
|
226
|
+
const start = content.indexOf(startTag, searchIndex);
|
|
227
|
+
if (start === -1)
|
|
228
|
+
break;
|
|
229
|
+
const end = content.indexOf(endTag, start + startTag.length);
|
|
230
|
+
if (end === -1)
|
|
231
|
+
break;
|
|
232
|
+
const inner = content.slice(start + startTag.length, end);
|
|
233
|
+
const nameMatch = inner.match(/<name>([^<]+)<\/name>/i);
|
|
234
|
+
const argsMatch = inner.match(/<arguments>([\s\S]*?)<\/arguments>/i);
|
|
235
|
+
if (!nameMatch || !argsMatch) {
|
|
236
|
+
searchIndex = end + endTag.length;
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
const fullEnd = end + endTag.length;
|
|
240
|
+
matches.push({
|
|
241
|
+
start,
|
|
242
|
+
end: fullEnd,
|
|
243
|
+
toolName: nameMatch[1].trim(),
|
|
244
|
+
rawArgs: argsMatch[1],
|
|
245
|
+
fullMatch: content.slice(start, fullEnd),
|
|
246
|
+
});
|
|
247
|
+
searchIndex = fullEnd;
|
|
248
|
+
}
|
|
249
|
+
return matches;
|
|
250
|
+
}
|
|
251
|
+
findUseToolRequests(content) {
|
|
252
|
+
const matches = [];
|
|
253
|
+
const prefix = '<use';
|
|
254
|
+
const closing = '</use>';
|
|
255
|
+
let searchIndex = 0;
|
|
256
|
+
while (searchIndex < content.length) {
|
|
257
|
+
const start = content.indexOf(prefix, searchIndex);
|
|
258
|
+
if (start === -1)
|
|
259
|
+
break;
|
|
260
|
+
const tagEnd = this.findTagClose(content, start + prefix.length);
|
|
261
|
+
if (tagEnd === -1)
|
|
262
|
+
break;
|
|
263
|
+
const header = content.slice(start + prefix.length, tagEnd);
|
|
264
|
+
const { toolName, attributeText } = this.extractToolNameAndAttributes(header);
|
|
265
|
+
const closingIndex = content.startsWith(closing, tagEnd + 1)
|
|
266
|
+
? tagEnd + 1 + closing.length
|
|
267
|
+
: tagEnd + 1;
|
|
268
|
+
if (!toolName) {
|
|
269
|
+
searchIndex = tagEnd + 1;
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
matches.push({
|
|
273
|
+
start,
|
|
274
|
+
end: closingIndex,
|
|
275
|
+
toolName,
|
|
276
|
+
rawArgs: this.parseAttributeArguments(attributeText),
|
|
277
|
+
fullMatch: content.slice(start, closingIndex),
|
|
278
|
+
});
|
|
279
|
+
searchIndex = closingIndex;
|
|
280
|
+
}
|
|
281
|
+
return matches;
|
|
282
|
+
}
|
|
283
|
+
findUseUnderscoreToolRequests(content) {
|
|
284
|
+
const matches = [];
|
|
285
|
+
const prefix = '<use_';
|
|
286
|
+
let searchIndex = 0;
|
|
287
|
+
while (searchIndex < content.length) {
|
|
288
|
+
const start = content.indexOf(prefix, searchIndex);
|
|
289
|
+
if (start === -1)
|
|
290
|
+
break;
|
|
291
|
+
let nameEnd = start + prefix.length;
|
|
292
|
+
while (nameEnd < content.length &&
|
|
293
|
+
/[A-Za-z0-9_.-]/.test(content[nameEnd])) {
|
|
294
|
+
nameEnd++;
|
|
295
|
+
}
|
|
296
|
+
const toolName = content.slice(start + prefix.length, nameEnd);
|
|
297
|
+
const tagEnd = this.findTagClose(content, nameEnd);
|
|
298
|
+
if (tagEnd === -1)
|
|
299
|
+
break;
|
|
300
|
+
const attributeText = content.slice(nameEnd, tagEnd);
|
|
301
|
+
const closingTag = `</use_${toolName}>`;
|
|
302
|
+
const bodyEnd = tagEnd + 1;
|
|
303
|
+
const closingIndex = content.startsWith(closingTag, bodyEnd)
|
|
304
|
+
? bodyEnd + closingTag.length
|
|
305
|
+
: bodyEnd;
|
|
306
|
+
if (!toolName) {
|
|
307
|
+
searchIndex = bodyEnd;
|
|
308
|
+
continue;
|
|
194
309
|
}
|
|
310
|
+
matches.push({
|
|
311
|
+
start,
|
|
312
|
+
end: closingIndex,
|
|
313
|
+
toolName,
|
|
314
|
+
rawArgs: this.parseAttributeArguments(attributeText),
|
|
315
|
+
fullMatch: content.slice(start, closingIndex),
|
|
316
|
+
});
|
|
317
|
+
searchIndex = closingIndex;
|
|
318
|
+
}
|
|
319
|
+
return matches;
|
|
320
|
+
}
|
|
321
|
+
findKeyValueToolRequests(content) {
|
|
322
|
+
const matches = [];
|
|
323
|
+
let match;
|
|
324
|
+
while ((match = this.keyValuePattern.exec(content)) !== null) {
|
|
325
|
+
const fullMatch = match[0];
|
|
326
|
+
matches.push({
|
|
327
|
+
start: match.index,
|
|
328
|
+
end: match.index + fullMatch.length,
|
|
329
|
+
toolName: match[1],
|
|
330
|
+
rawArgs: this.parseKeyValuePairs(match[2]),
|
|
331
|
+
fullMatch,
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
this.keyValuePattern.lastIndex = 0;
|
|
335
|
+
return matches;
|
|
336
|
+
}
|
|
337
|
+
removeMatchedRanges(content, ranges) {
|
|
338
|
+
if (ranges.length === 0) {
|
|
339
|
+
return content;
|
|
340
|
+
}
|
|
341
|
+
const sorted = ranges
|
|
342
|
+
.filter(({ start, end }) => start < end)
|
|
343
|
+
.sort((a, b) => a.start - b.start);
|
|
344
|
+
const merged = [];
|
|
345
|
+
for (const range of sorted) {
|
|
346
|
+
const last = merged[merged.length - 1];
|
|
347
|
+
if (last && range.start <= last.end) {
|
|
348
|
+
last.end = Math.max(last.end, range.end);
|
|
349
|
+
}
|
|
350
|
+
else {
|
|
351
|
+
merged.push({ ...range });
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
let cursor = 0;
|
|
355
|
+
const pieces = [];
|
|
356
|
+
for (const range of merged) {
|
|
357
|
+
if (cursor < range.start) {
|
|
358
|
+
pieces.push(content.slice(cursor, range.start));
|
|
359
|
+
}
|
|
360
|
+
cursor = Math.max(cursor, range.end);
|
|
195
361
|
}
|
|
196
|
-
|
|
197
|
-
|
|
362
|
+
pieces.push(content.slice(cursor));
|
|
363
|
+
return pieces.join('');
|
|
364
|
+
}
|
|
365
|
+
postProcessCleanedContent(content) {
|
|
366
|
+
return content
|
|
198
367
|
.replace(/\[TOOL_REQUEST(?:_END)?]/g, '')
|
|
199
368
|
.replace(/<\|im_start\|>assistant/g, '')
|
|
200
369
|
.replace(/<\|im_end\|>/g, '')
|
|
201
|
-
.replace(/<tool_call>[\s\S]*?<\/tool_call>/g, '')
|
|
202
|
-
.replace(/<function_calls>[\s\S]*?<\/function_calls>/g, '')
|
|
203
|
-
.replace(/<invoke[\s\S]*?<\/invoke>/g, '')
|
|
204
|
-
.replace(/<tool>[\s\S]*?<\/tool>/g, '')
|
|
205
|
-
.replace(/<\/use_[
|
|
206
|
-
.replace(/<\/use>/g, '')
|
|
207
|
-
|
|
208
|
-
.replace(
|
|
209
|
-
.replace(/\{"name"\s*:\s*"[^"]*"\s*,?\s*"arguments"\s*:\s*\{[^}]*$/gm, '') // Remove incomplete JSON tool calls
|
|
370
|
+
.replace(/<tool_call>[\s\S]*?<\/tool_call>/g, '')
|
|
371
|
+
.replace(/<function_calls>[\s\S]*?<\/function_calls>/g, '')
|
|
372
|
+
.replace(/<invoke[\s\S]*?<\/invoke>/g, '')
|
|
373
|
+
.replace(/<tool>[\s\S]*?<\/tool>/g, '')
|
|
374
|
+
.replace(/<\/use_[A-Za-z0-9_.-]+>/g, '')
|
|
375
|
+
.replace(/<\/use>/g, '')
|
|
376
|
+
.replace(/<tool_call>\s*\{[^}]*$/gm, '')
|
|
377
|
+
.replace(/\{"name"\s*:\s*"[^"]*"\s*,?\s*"arguments"\s*:\s*\{[^}]*$/gm, '')
|
|
210
378
|
.replace(/✦\s*<think>/g, '')
|
|
211
|
-
.trim();
|
|
212
|
-
// Collapse multiple consecutive newlines/spaces after removing tool calls
|
|
213
|
-
cleanedContent = cleanedContent
|
|
214
379
|
.replace(/\n\s*\n/g, '\n')
|
|
215
380
|
.replace(/\n/g, ' ')
|
|
216
381
|
.trim();
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
382
|
+
}
|
|
383
|
+
normalizeArguments(args, toolName, fullMatch) {
|
|
384
|
+
if (typeof args !== 'string') {
|
|
385
|
+
return args;
|
|
386
|
+
}
|
|
387
|
+
try {
|
|
388
|
+
if (args.includes('<parameter') ||
|
|
389
|
+
(args.includes('<') && args.includes('>'))) {
|
|
390
|
+
return this.parseXMLParameters(args);
|
|
391
|
+
}
|
|
392
|
+
return JSON.parse(args);
|
|
393
|
+
}
|
|
394
|
+
catch (error) {
|
|
395
|
+
const repaired = this.tryRepairJson(args);
|
|
396
|
+
if (repaired) {
|
|
397
|
+
try {
|
|
398
|
+
return JSON.parse(repaired);
|
|
399
|
+
}
|
|
400
|
+
catch {
|
|
401
|
+
// ignore and fall through
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
const simpleJsonMatch = args.match(/^{[^{]*}$/);
|
|
405
|
+
if (simpleJsonMatch) {
|
|
406
|
+
try {
|
|
407
|
+
return JSON.parse(simpleJsonMatch[0]);
|
|
408
|
+
}
|
|
409
|
+
catch {
|
|
410
|
+
// fall through to logging
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
console.error(`[GemmaToolCallParser] Failed to parse tool arguments for ${toolName}:`, error);
|
|
414
|
+
console.error(`[GemmaToolCallParser] Raw arguments excerpt: ${fullMatch.slice(0, 200)}`);
|
|
415
|
+
return null;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
extractBalancedSegment(content, startIndex, openChar, closeChar) {
|
|
419
|
+
if (content[startIndex] !== openChar) {
|
|
420
|
+
return null;
|
|
421
|
+
}
|
|
422
|
+
let depth = 0;
|
|
423
|
+
let inString = null;
|
|
424
|
+
let escapeNext = false;
|
|
425
|
+
for (let i = startIndex; i < content.length; i++) {
|
|
426
|
+
const char = content[i];
|
|
427
|
+
if (escapeNext) {
|
|
428
|
+
escapeNext = false;
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
if (char === '\\' && inString) {
|
|
432
|
+
escapeNext = true;
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
if (inString) {
|
|
436
|
+
if (char === inString) {
|
|
437
|
+
inString = null;
|
|
438
|
+
}
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
if (char === '"' || char === "'") {
|
|
442
|
+
inString = char;
|
|
443
|
+
continue;
|
|
444
|
+
}
|
|
445
|
+
if (char === openChar) {
|
|
446
|
+
depth++;
|
|
447
|
+
}
|
|
448
|
+
else if (char === closeChar) {
|
|
449
|
+
depth--;
|
|
450
|
+
if (depth === 0) {
|
|
451
|
+
return {
|
|
452
|
+
segment: content.slice(startIndex, i + 1),
|
|
453
|
+
endIndex: i + 1,
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
return null;
|
|
221
459
|
}
|
|
222
460
|
// Best-effort repair for JSON with unescaped inner quotes in string values.
|
|
223
461
|
tryRepairJson(args) {
|
|
@@ -228,7 +466,7 @@ export class GemmaToolCallParser {
|
|
|
228
466
|
catch {
|
|
229
467
|
// Target only inner quotes within JSON string values, preserving multibyte and spacing
|
|
230
468
|
// e.g., { "command": "printf "ありがとう 世界"" } -> { "command": "printf \"ありがとう 世界\"" }
|
|
231
|
-
const repaired = args.replace(/:(\s)
|
|
469
|
+
const repaired = args.replace(/:(\s*)"((?:\\.|[^"\\])*)"(\s*)([,}])/g, (_m, s1, val, s2, tail) => {
|
|
232
470
|
// Escape only unescaped quotes inside the value
|
|
233
471
|
const fixed = val.replace(/(?<!\\)"/g, '\\"');
|
|
234
472
|
return `:${s1}"${fixed}"${s2}${tail}`;
|
|
@@ -279,6 +517,88 @@ export class GemmaToolCallParser {
|
|
|
279
517
|
}
|
|
280
518
|
return args;
|
|
281
519
|
}
|
|
520
|
+
parseAttributeArguments(attributeText) {
|
|
521
|
+
const args = {};
|
|
522
|
+
const text = attributeText ?? '';
|
|
523
|
+
const length = text.length;
|
|
524
|
+
let index = 0;
|
|
525
|
+
const readIdentifier = () => {
|
|
526
|
+
const start = index;
|
|
527
|
+
while (index < length && /[A-Za-z0-9_.-]/.test(text.charAt(index))) {
|
|
528
|
+
index++;
|
|
529
|
+
}
|
|
530
|
+
return text.slice(start, index);
|
|
531
|
+
};
|
|
532
|
+
const skipWhitespace = () => {
|
|
533
|
+
while (index < length && /\s/.test(text.charAt(index))) {
|
|
534
|
+
index++;
|
|
535
|
+
}
|
|
536
|
+
};
|
|
537
|
+
while (index < length) {
|
|
538
|
+
skipWhitespace();
|
|
539
|
+
const key = readIdentifier();
|
|
540
|
+
if (!key) {
|
|
541
|
+
index++;
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
544
|
+
skipWhitespace();
|
|
545
|
+
if (text.charAt(index) !== '=') {
|
|
546
|
+
index++;
|
|
547
|
+
continue;
|
|
548
|
+
}
|
|
549
|
+
index++;
|
|
550
|
+
skipWhitespace();
|
|
551
|
+
const quote = text.charAt(index);
|
|
552
|
+
if (quote !== '"' && quote !== "'") {
|
|
553
|
+
index++;
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
index++;
|
|
557
|
+
let value = '';
|
|
558
|
+
let escaped = false;
|
|
559
|
+
while (index < length) {
|
|
560
|
+
const char = text.charAt(index);
|
|
561
|
+
index++;
|
|
562
|
+
if (escaped) {
|
|
563
|
+
value += char;
|
|
564
|
+
escaped = false;
|
|
565
|
+
continue;
|
|
566
|
+
}
|
|
567
|
+
if (char === '\\') {
|
|
568
|
+
escaped = true;
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
if (char === quote) {
|
|
572
|
+
break;
|
|
573
|
+
}
|
|
574
|
+
value += char;
|
|
575
|
+
}
|
|
576
|
+
const unescaped = value.trim();
|
|
577
|
+
if (!key) {
|
|
578
|
+
continue;
|
|
579
|
+
}
|
|
580
|
+
if (unescaped.startsWith('{') || unescaped.startsWith('[')) {
|
|
581
|
+
try {
|
|
582
|
+
args[key] = JSON.parse(unescaped);
|
|
583
|
+
continue;
|
|
584
|
+
}
|
|
585
|
+
catch {
|
|
586
|
+
// fall through to scalar handling
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
if (/^-?\d+(\.\d+)?$/.test(unescaped)) {
|
|
590
|
+
args[key] = Number(unescaped);
|
|
591
|
+
}
|
|
592
|
+
else if (unescaped.toLowerCase() === 'true' ||
|
|
593
|
+
unescaped.toLowerCase() === 'false') {
|
|
594
|
+
args[key] = unescaped.toLowerCase() === 'true';
|
|
595
|
+
}
|
|
596
|
+
else {
|
|
597
|
+
args[key] = unescaped;
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
return args;
|
|
601
|
+
}
|
|
282
602
|
parseXMLParameters(xmlContent) {
|
|
283
603
|
const args = {};
|
|
284
604
|
// Parse Claude-style <parameter name="key">value</parameter>
|
|
@@ -316,5 +636,39 @@ export class GemmaToolCallParser {
|
|
|
316
636
|
.replace(/"/g, '"')
|
|
317
637
|
.replace(/'/g, "'");
|
|
318
638
|
}
|
|
639
|
+
findTagClose(content, fromIndex) {
|
|
640
|
+
let inQuote = null;
|
|
641
|
+
for (let i = fromIndex; i < content.length; i++) {
|
|
642
|
+
const char = content[i];
|
|
643
|
+
if (inQuote) {
|
|
644
|
+
if (char === inQuote && content[i - 1] !== '\\') {
|
|
645
|
+
inQuote = null;
|
|
646
|
+
}
|
|
647
|
+
continue;
|
|
648
|
+
}
|
|
649
|
+
if (char === '"' || char === "'") {
|
|
650
|
+
inQuote = char;
|
|
651
|
+
continue;
|
|
652
|
+
}
|
|
653
|
+
if (char === '>') {
|
|
654
|
+
return i;
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
return -1;
|
|
658
|
+
}
|
|
659
|
+
extractToolNameAndAttributes(header) {
|
|
660
|
+
let index = 0;
|
|
661
|
+
const length = header.length;
|
|
662
|
+
while (index < length && /\s/.test(header.charAt(index))) {
|
|
663
|
+
index++;
|
|
664
|
+
}
|
|
665
|
+
const nameStart = index;
|
|
666
|
+
while (index < length && /[A-Za-z0-9_.-]/.test(header.charAt(index))) {
|
|
667
|
+
index++;
|
|
668
|
+
}
|
|
669
|
+
const toolName = header.slice(nameStart, index).trim();
|
|
670
|
+
const attributeText = header.slice(index);
|
|
671
|
+
return { toolName, attributeText };
|
|
672
|
+
}
|
|
319
673
|
}
|
|
320
674
|
//# sourceMappingURL=TextToolCallParser.js.map
|