@bolt-cli/cli 0.0.0-stage → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +75 -0
- package/README.md +155 -2
- package/bin/bolt.js +21 -0
- package/lib/agent.js +361 -0
- package/lib/api.js +246 -0
- package/lib/bootstrap.js +292 -0
- package/lib/browser.js +152 -0
- package/lib/cancel.js +153 -0
- package/lib/checkpoints.js +67 -0
- package/lib/cli.js +2637 -0
- package/lib/completion.js +77 -0
- package/lib/config.js +80 -0
- package/lib/doctor.js +110 -0
- package/lib/export.js +77 -0
- package/lib/history.js +78 -0
- package/lib/memory.js +80 -0
- package/lib/output.js +88 -0
- package/lib/permissions.js +167 -0
- package/lib/present.js +181 -0
- package/lib/redact.js +56 -0
- package/lib/session.js +153 -0
- package/lib/toolparse.js +752 -0
- package/lib/tools.js +606 -0
- package/lib/ui.js +783 -0
- package/lib/updatecheck.js +114 -0
- package/lib/watch.js +698 -0
- package/package.json +36 -6
- package/templates/README.md +44 -0
- package/templates/aslr-offset-probe.sh +96 -0
- package/templates/crash-offset-pattern.sh +81 -0
package/lib/toolparse.js
ADDED
|
@@ -0,0 +1,752 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Model-agnostic tool-call extraction.
|
|
3
|
+
//
|
|
4
|
+
// The agent loop is client-driven (text-in/text-out relay), so the CLI must
|
|
5
|
+
// read tool calls whatever shape the model emits them in. Models drift from
|
|
6
|
+
// the instructed ```tool fences to chat-template syntax: <tool>,
|
|
7
|
+
// <tool_call>, <function_calls>, <invoke>, bare JSON objects, unclosed
|
|
8
|
+
// final fences. A parser that knows one shape makes those runs "reply with
|
|
9
|
+
// nothing" or stream raw markup to the operator. This module recovers the
|
|
10
|
+
// calls from every common shape, returns the cleaned prose (markup removed),
|
|
11
|
+
// and provides the streaming filter that keeps machine syntax off the
|
|
12
|
+
// operator's screen while tokens arrive.
|
|
13
|
+
//
|
|
14
|
+
// Shapes parsed (overlaps resolved by position, earliest start wins):
|
|
15
|
+
// - ```tool / ```ask and ~~~tilde twins, closed or unclosed at end of text
|
|
16
|
+
// - <tool>, <tool_call>, <tool_use>, <function>, <function_call>,
|
|
17
|
+
// <invoke> and the <function_calls> wrapper — with or without the
|
|
18
|
+
// antml: namespace, JSON bodies or <parameter name=...> children
|
|
19
|
+
// - plain or ```json fences and bare line-leading JSON objects, claimed
|
|
20
|
+
// only when they normalize to a known tool (ordinary JSON stays prose)
|
|
21
|
+
// - 自知-family blocks are stripped as reasoning, never executed
|
|
22
|
+
//
|
|
23
|
+
// Zero deps. toolparse must not require tools.js (tools.js requires this).
|
|
24
|
+
|
|
25
|
+
const TOOL_NAMES = ["read", "glob", "grep", "list", "write", "edit", "bash", "ask"];
|
|
26
|
+
const MAX_TOOL_BLOCKS = 6;
|
|
27
|
+
|
|
28
|
+
const MAX_DEPTH = 3;
|
|
29
|
+
const RAW_EXCERPT = 200;
|
|
30
|
+
// Stream filter: an open tool tag with no close stops holding after this
|
|
31
|
+
// many chars; the tag is dropped and the rest flows visibly again, so an
|
|
32
|
+
// unclosed tag can never eat the operator's final answer.
|
|
33
|
+
const FILTER_HOLD_LIMIT = 6000;
|
|
34
|
+
|
|
35
|
+
// Keys consumed by normalization, never copied into call arguments.
|
|
36
|
+
const RESERVED_KEYS = new Set([
|
|
37
|
+
"tool", "name", "tool_name", "toolName", "function",
|
|
38
|
+
"arguments", "parameters", "params", "input", "args",
|
|
39
|
+
"id", "type", "index",
|
|
40
|
+
]);
|
|
41
|
+
// Generic XML children with these names carry identity, not arguments.
|
|
42
|
+
const GENERIC_SKIP = new Set(["name", "tool", "id", "type", "function", "index"]);
|
|
43
|
+
// Attribute keys that never become arguments (call bookkeeping).
|
|
44
|
+
const RESERVED_ATTR_KEYS = new Set(["id", "type", "index", "session"]);
|
|
45
|
+
|
|
46
|
+
// Reasoning tags: stripped everywhere, their content is never executed —
|
|
47
|
+
// a model that thinks inside these tags has not made a tool call yet.
|
|
48
|
+
const THINK_TAGS = ["thinking", "thought", "scratchpad", "reasoning", "think"];
|
|
49
|
+
|
|
50
|
+
// XML-family tags that carry one call each. "function_calls" is a wrapper
|
|
51
|
+
// whose children are parsed recursively.
|
|
52
|
+
const XML_TAGS = ["function_calls", "tool_call", "tool_use", "function_call", "function", "tool", "invoke"];
|
|
53
|
+
|
|
54
|
+
const str = (v) => (typeof v === "string" && v.trim() ? v.trim() : null);
|
|
55
|
+
const firstOf = (obj, keys) => {
|
|
56
|
+
for (const k of keys) if (obj[k] != null) return obj[k];
|
|
57
|
+
return null;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Normalizes any call shape into {tool, ...args}:
|
|
62
|
+
* {"tool":"read",...} | {"name":"read","arguments":{...}} |
|
|
63
|
+
* {"function":{"name":"read","arguments":{...}}} | {"function":"read",...} |
|
|
64
|
+
* {"toolName":"read","input":{...}}. String args are JSON-parsed.
|
|
65
|
+
* Returns {ok, call} or {ok:false, error}.
|
|
66
|
+
*/
|
|
67
|
+
function normalizeCall(obj, fallbackAttrs = null) {
|
|
68
|
+
if (!obj || typeof obj !== "object" || Array.isArray(obj)) {
|
|
69
|
+
return { ok: false, error: "tool call is not a JSON object" };
|
|
70
|
+
}
|
|
71
|
+
let tool = str(obj.tool) || str(obj.name) || str(obj.tool_name) || str(obj.toolName);
|
|
72
|
+
let args = null;
|
|
73
|
+
const ARG_KEYS = ["arguments", "parameters", "params", "input", "args"];
|
|
74
|
+
if (obj.function != null) {
|
|
75
|
+
if (typeof obj.function === "object" && !Array.isArray(obj.function)) {
|
|
76
|
+
tool = tool || str(obj.function.name);
|
|
77
|
+
args = args || firstOf(obj.function, ARG_KEYS);
|
|
78
|
+
} else {
|
|
79
|
+
tool = tool || str(obj.function);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if (!tool && fallbackAttrs) tool = str(fallbackAttrs.name) || str(fallbackAttrs.tool);
|
|
83
|
+
args = args || firstOf(obj, ARG_KEYS);
|
|
84
|
+
if (!tool) return { ok: false, error: "no tool name in call" };
|
|
85
|
+
tool = String(tool).trim().toLowerCase();
|
|
86
|
+
|
|
87
|
+
const call = {};
|
|
88
|
+
if (args != null) {
|
|
89
|
+
if (typeof args === "string") {
|
|
90
|
+
try {
|
|
91
|
+
args = JSON.parse(args);
|
|
92
|
+
} catch {
|
|
93
|
+
return { ok: false, error: "arguments are not valid JSON" };
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (args && typeof args === "object" && !Array.isArray(args)) Object.assign(call, args);
|
|
97
|
+
}
|
|
98
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
99
|
+
if (RESERVED_KEYS.has(k)) continue;
|
|
100
|
+
call[k] = v;
|
|
101
|
+
}
|
|
102
|
+
if (fallbackAttrs) {
|
|
103
|
+
for (const [k, v] of Object.entries(fallbackAttrs)) {
|
|
104
|
+
if (RESERVED_ATTR_KEYS.has(k) || RESERVED_KEYS.has(k)) continue;
|
|
105
|
+
if (call[k] == null) call[k] = v;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
call.tool = tool;
|
|
109
|
+
return { ok: true, call };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Balanced-brace span for a JSON object starting at openIdx. Returns the
|
|
113
|
+
// index of the closing "}", or -1. String- and escape-aware.
|
|
114
|
+
function jsonSpan(s, openIdx) {
|
|
115
|
+
let depth = 0;
|
|
116
|
+
let inStr = false;
|
|
117
|
+
let esc = false;
|
|
118
|
+
for (let i = openIdx; i < s.length; i += 1) {
|
|
119
|
+
const c = s[i];
|
|
120
|
+
if (inStr) {
|
|
121
|
+
if (esc) esc = false;
|
|
122
|
+
else if (c === "\\") esc = true;
|
|
123
|
+
else if (c === '"') inStr = false;
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (c === '"') inStr = true;
|
|
127
|
+
else if (c === "{") depth += 1;
|
|
128
|
+
else if (c === "}") {
|
|
129
|
+
depth -= 1;
|
|
130
|
+
if (depth === 0) return i;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return -1;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// ``` or ~~~ fence close: a bare marker line (newline, optional spaces,
|
|
137
|
+
// marker, then newline or end of text). The trailing newline is NOT
|
|
138
|
+
// consumed — the next fence's opener must stay findable.
|
|
139
|
+
//
|
|
140
|
+
// A body whose JSON contains fence markers at line starts (the write tool
|
|
141
|
+
// carrying markdown is the common case) defeats the FIRST close, so the
|
|
142
|
+
// search continues to the first close whose body actually parses. Returns
|
|
143
|
+
// {start, end, parseable} — the first close when nothing parses (error
|
|
144
|
+
// path) — or null while no close exists in view.
|
|
145
|
+
function chooseFenceClose(s, from, marker) {
|
|
146
|
+
const re = new RegExp(`\\n[ \\t]*${marker}[ \\t]*(?=\\r?\\n|$)`, "g");
|
|
147
|
+
re.lastIndex = from;
|
|
148
|
+
let first = null;
|
|
149
|
+
let m;
|
|
150
|
+
while ((m = re.exec(s)) !== null) {
|
|
151
|
+
const cand = { start: m.index, end: m.index + m[0].length, parseable: false };
|
|
152
|
+
if (!first) first = cand;
|
|
153
|
+
const raw = s.slice(from, cand.start).trim();
|
|
154
|
+
if (raw.startsWith("{") || raw.startsWith("[")) {
|
|
155
|
+
try {
|
|
156
|
+
JSON.parse(raw);
|
|
157
|
+
cand.parseable = true;
|
|
158
|
+
return cand;
|
|
159
|
+
} catch {
|
|
160
|
+
// Content contains a fence marker — try the next close.
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return first;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Parse attrs of an open tag like <function name="read" id="x">.
|
|
168
|
+
function parseAttrs(openTag) {
|
|
169
|
+
const attrs = {};
|
|
170
|
+
const re = /([a-zA-Z_][\w.:-]*)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s">]+))/g;
|
|
171
|
+
let m;
|
|
172
|
+
while ((m = re.exec(openTag)) !== null) {
|
|
173
|
+
attrs[m[1]] = m[2] != null ? m[2] : m[3] != null ? m[3] : m[4] != null ? m[4] : "";
|
|
174
|
+
}
|
|
175
|
+
return attrs;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Find the matching close for an XML open whose tag ends at openEnd, with
|
|
179
|
+
// base name `base` (spelling without any antml: prefix). Accepts both
|
|
180
|
+
// spellings for the close and handles same-tag nesting. Returns the end
|
|
181
|
+
// index after ">", or -1 while no close exists in view.
|
|
182
|
+
function findXmlClose(s, openEnd, base) {
|
|
183
|
+
const openRe = new RegExp(`<(?:antml:)?${base}(?=[\\s>/])`, "g");
|
|
184
|
+
const closeRe = new RegExp(`</(?:antml:)?${base}\\s*>`, "g");
|
|
185
|
+
openRe.lastIndex = openEnd;
|
|
186
|
+
closeRe.lastIndex = openEnd;
|
|
187
|
+
const events = [];
|
|
188
|
+
let mo = openRe.exec(s);
|
|
189
|
+
while (mo) {
|
|
190
|
+
events.push({ index: mo.index, open: true, end: mo.index + mo[0].length });
|
|
191
|
+
mo = openRe.exec(s);
|
|
192
|
+
}
|
|
193
|
+
let mc = closeRe.exec(s);
|
|
194
|
+
while (mc) {
|
|
195
|
+
events.push({ index: mc.index, open: false, end: mc.index + mc[0].length });
|
|
196
|
+
mc = closeRe.exec(s);
|
|
197
|
+
}
|
|
198
|
+
events.sort((a, b) => a.index - b.index);
|
|
199
|
+
let depth = 1;
|
|
200
|
+
for (const ev of events) {
|
|
201
|
+
if (ev.index < openEnd) continue;
|
|
202
|
+
if (ev.open) depth += 1;
|
|
203
|
+
else {
|
|
204
|
+
depth -= 1;
|
|
205
|
+
if (depth === 0) return ev.end;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return -1;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// Start index of the close tag whose end is closeEnd, so a region body
|
|
212
|
+
// excludes the closing tag itself.
|
|
213
|
+
function closeStartOf(s, from, base, closeEnd) {
|
|
214
|
+
const closeRe = new RegExp(`</(?:antml:)?${base}\\s*>`, "g");
|
|
215
|
+
closeRe.lastIndex = from;
|
|
216
|
+
let m;
|
|
217
|
+
while ((m = closeRe.exec(s)) !== null) {
|
|
218
|
+
if (m.index + m[0].length === closeEnd) return m.index;
|
|
219
|
+
if (m.index >= closeEnd) break;
|
|
220
|
+
}
|
|
221
|
+
return closeEnd;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const PARAM_RE = /<(?:antml:)?(?:parameter|param)(?:\s+name\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+)))?[^>]*>([\s\S]*?)<\/(?:antml:)?(?:parameter|param)>/g;
|
|
225
|
+
|
|
226
|
+
function childText(body, name) {
|
|
227
|
+
const re = new RegExp(`<(?:antml:)?${name}\\b[^>]*>([\\s\\S]*?)</(?:antml:)?${name}>`);
|
|
228
|
+
const m = re.exec(body);
|
|
229
|
+
return m ? m[1].trim() : null;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// Top-level children of an XML block body as {tag, body} pairs. A proper
|
|
233
|
+
// scanner, not a lazy regex: a lazy match swallows sibling children when a
|
|
234
|
+
// nested block shares the body (e.g. <name> inside <function>), losing
|
|
235
|
+
// their parameters.
|
|
236
|
+
function xmlChildren(body) {
|
|
237
|
+
const out = [];
|
|
238
|
+
const openRe = /<([a-zA-Z_][\w.:-]*)\b[^>]*?(\/?)>/g;
|
|
239
|
+
let m;
|
|
240
|
+
while ((m = openRe.exec(body)) !== null) {
|
|
241
|
+
const tag = m[1];
|
|
242
|
+
if (m[2]) {
|
|
243
|
+
out.push({ tag, body: "" });
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
const start = m.index + m[0].length;
|
|
247
|
+
const closeIdx = findXmlClose(body, start, tag);
|
|
248
|
+
if (closeIdx === -1) continue;
|
|
249
|
+
out.push({ tag, body: body.slice(start, closeStartOf(body, start, tag, closeIdx)) });
|
|
250
|
+
openRe.lastIndex = closeIdx;
|
|
251
|
+
}
|
|
252
|
+
return out;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Parses an XML block body into calls. Tries: JSON body, antml parameter
|
|
257
|
+
* children, generic XML children. Returns {ok, calls} | {ok:false, error} |
|
|
258
|
+
* {recurse:true} when the body holds nested blocks (a function_calls
|
|
259
|
+
* wrapper) instead of a directly-parseable call.
|
|
260
|
+
*/
|
|
261
|
+
function xmlToCall(attrs, body) {
|
|
262
|
+
const trimmed = String(body || "").trim();
|
|
263
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
264
|
+
try {
|
|
265
|
+
const obj = JSON.parse(trimmed);
|
|
266
|
+
if (obj && typeof obj === "object") {
|
|
267
|
+
if (Array.isArray(obj)) {
|
|
268
|
+
const calls = [];
|
|
269
|
+
for (const item of obj) {
|
|
270
|
+
const n = normalizeCall(item, attrs);
|
|
271
|
+
if (n.ok) calls.push(n.call);
|
|
272
|
+
}
|
|
273
|
+
if (calls.length) return { ok: true, calls };
|
|
274
|
+
return { ok: false, error: "no tool call in JSON array" };
|
|
275
|
+
}
|
|
276
|
+
const n = normalizeCall(obj, attrs);
|
|
277
|
+
if (n.ok) return { ok: true, calls: [n.call] };
|
|
278
|
+
return { ok: false, error: n.error };
|
|
279
|
+
}
|
|
280
|
+
} catch {
|
|
281
|
+
// Not JSON — fall through to XML children.
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
const name = str(attrs && attrs.name) || str(attrs && attrs.tool) || childText(trimmed, "name") || childText(trimmed, "tool");
|
|
285
|
+
const params = {};
|
|
286
|
+
let found = false;
|
|
287
|
+
PARAM_RE.lastIndex = 0;
|
|
288
|
+
let m;
|
|
289
|
+
while ((m = PARAM_RE.exec(trimmed)) !== null) {
|
|
290
|
+
const key = m[1] != null ? m[1] : m[2] != null ? m[2] : m[3];
|
|
291
|
+
if (!key) continue;
|
|
292
|
+
params[key.trim()] = (m[4] || "").trim();
|
|
293
|
+
found = true;
|
|
294
|
+
}
|
|
295
|
+
if (!found) {
|
|
296
|
+
for (const child of xmlChildren(trimmed)) {
|
|
297
|
+
const key = child.tag.replace(/^antml:/, "");
|
|
298
|
+
if (GENERIC_SKIP.has(key) || THINK_TAGS.includes(key)) continue;
|
|
299
|
+
params[key] = child.body.trim();
|
|
300
|
+
found = true;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
// No name but a nested call block (or nothing at all): recurse — the
|
|
304
|
+
// parameters collected here belong to the inner call, parsed whole.
|
|
305
|
+
const hasNestedCall = /<(?:antml:)?(?:function_calls|tool_call|tool_use|function_call|function|tool|invoke)\b/.test(trimmed);
|
|
306
|
+
if (!name && (hasNestedCall || !found)) return { recurse: true };
|
|
307
|
+
if (!name) return { ok: false, error: "no tool name in block" };
|
|
308
|
+
// Name without parameters while a nested call block sits inside: the real
|
|
309
|
+
// call lives in the child (wrapper naming the tool, inner block carrying
|
|
310
|
+
// the arguments) — recurse; otherwise emit the bare call so executeCall's
|
|
311
|
+
// own feedback fires.
|
|
312
|
+
if (!found && hasNestedCall) return { recurse: true };
|
|
313
|
+
if (!found) return { ok: true, calls: [normalizeCall({ tool: name }, attrs).call] };
|
|
314
|
+
return { ok: true, calls: [normalizeCall({ tool: name, ...params }, null).call] };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
function excerpt(raw) {
|
|
318
|
+
return String(raw).trim().slice(0, RAW_EXCERPT);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// Would an unclosed XML region parse as a tool call? Decides whether to
|
|
322
|
+
// claim an unclosed block (never emits errors, never executes on its own).
|
|
323
|
+
function probeUnclosedXml(region, depth) {
|
|
324
|
+
const r = xmlToCall(region.attrs || {}, region.body);
|
|
325
|
+
if (r && r.ok) return true;
|
|
326
|
+
if (r && r.recurse && depth < MAX_DEPTH) {
|
|
327
|
+
const inner = extractFromText(region.body, depth + 1);
|
|
328
|
+
return inner.calls.length > 0;
|
|
329
|
+
}
|
|
330
|
+
return false;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Extracts every tool region in `s` — fences, XML blocks, bare JSON,
|
|
335
|
+
* think tags — as {start, end, kind, body, attrs?, tag?}, with overlaps
|
|
336
|
+
* resolved (earliest start, longest first). Depth guards recursion.
|
|
337
|
+
*/
|
|
338
|
+
function collectRegions(s, depth) {
|
|
339
|
+
const regions = [];
|
|
340
|
+
if (depth > MAX_DEPTH) return regions;
|
|
341
|
+
|
|
342
|
+
// Fences: canonical (tool/ask) always claim; plain/json fences claim
|
|
343
|
+
// only when the body normalizes to a known tool call.
|
|
344
|
+
const fenceOpen = /(^|\n)[ \t]*(```|~~~)[ \t]*([a-zA-Z0-9+#._-]*)(?=[\s]|$)/g;
|
|
345
|
+
let m;
|
|
346
|
+
while ((m = fenceOpen.exec(s)) !== null) {
|
|
347
|
+
const marker = m[2] === "```" ? "```" : "~~~";
|
|
348
|
+
const lang = m[3].toLowerCase();
|
|
349
|
+
if (lang && lang !== "tool" && lang !== "ask" && lang !== "json") continue;
|
|
350
|
+
// Body starts after the opening line's newline (the match itself begins
|
|
351
|
+
// with that newline or text start — skip past it).
|
|
352
|
+
const nl = s.indexOf("\n", m.index + m[0].length);
|
|
353
|
+
const bodyStart = nl === -1 ? s.length : nl + 1;
|
|
354
|
+
const chosen = chooseFenceClose(s, bodyStart, marker);
|
|
355
|
+
const end = chosen ? chosen.end : s.length;
|
|
356
|
+
const body = chosen ? s.slice(bodyStart, chosen.start) : s.slice(bodyStart);
|
|
357
|
+
if (lang === "tool" || lang === "ask") {
|
|
358
|
+
regions.push({ start: m.index, end, kind: lang === "ask" ? "askfence" : "fence", body });
|
|
359
|
+
if (chosen) fenceOpen.lastIndex = end;
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
// plain/json fence: claim only when the body parses as a tool call.
|
|
363
|
+
if (chosen && chosen.parseable) {
|
|
364
|
+
try {
|
|
365
|
+
const n = normalizeCall(JSON.parse(body.trim()));
|
|
366
|
+
if (n.ok && TOOL_NAMES.includes(n.call.tool)) {
|
|
367
|
+
regions.push({ start: m.index, end, kind: "jsonfence", body });
|
|
368
|
+
fenceOpen.lastIndex = end;
|
|
369
|
+
}
|
|
370
|
+
} catch {
|
|
371
|
+
// Ordinary code/JSON block — leave it as prose.
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// XML family. Unclosed blocks claim to end of text only when they
|
|
377
|
+
// validate as a call (drift case: the stream ends mid-call).
|
|
378
|
+
for (const base of XML_TAGS) {
|
|
379
|
+
const openRe = new RegExp(`<(antml:)?${base}(?=[\\s>/])`, "g");
|
|
380
|
+
while ((m = openRe.exec(s)) !== null) {
|
|
381
|
+
const gt = s.indexOf(">", m.index);
|
|
382
|
+
if (gt === -1) break;
|
|
383
|
+
const openTag = s.slice(m.index, gt + 1);
|
|
384
|
+
if (/\/\s*>$/.test(openTag)) {
|
|
385
|
+
// Self-closing: <tool name="bash" command="ls"/>.
|
|
386
|
+
const attrs = parseAttrs(openTag);
|
|
387
|
+
const n = normalizeCall({}, attrs);
|
|
388
|
+
if (n.ok && TOOL_NAMES.includes(n.call.tool)) {
|
|
389
|
+
regions.push({ start: m.index, end: gt + 1, kind: "xml", tag: base, attrs, body: "" });
|
|
390
|
+
openRe.lastIndex = gt + 1;
|
|
391
|
+
}
|
|
392
|
+
continue;
|
|
393
|
+
}
|
|
394
|
+
const bodyStart = gt + 1;
|
|
395
|
+
const closeIdx = findXmlClose(s, bodyStart, base);
|
|
396
|
+
if (closeIdx !== -1) {
|
|
397
|
+
const attrs = parseAttrs(openTag);
|
|
398
|
+
regions.push({
|
|
399
|
+
start: m.index,
|
|
400
|
+
end: closeIdx,
|
|
401
|
+
kind: "xml",
|
|
402
|
+
tag: base,
|
|
403
|
+
attrs,
|
|
404
|
+
body: s.slice(bodyStart, closeStartOf(s, bodyStart, base, closeIdx)),
|
|
405
|
+
});
|
|
406
|
+
openRe.lastIndex = closeIdx;
|
|
407
|
+
} else {
|
|
408
|
+
const attrs = parseAttrs(openTag);
|
|
409
|
+
const region = {
|
|
410
|
+
start: m.index,
|
|
411
|
+
end: s.length,
|
|
412
|
+
kind: "xml",
|
|
413
|
+
tag: base,
|
|
414
|
+
attrs,
|
|
415
|
+
body: s.slice(bodyStart),
|
|
416
|
+
unclosed: true,
|
|
417
|
+
};
|
|
418
|
+
if (probeUnclosedXml(region, depth)) {
|
|
419
|
+
regions.push(region);
|
|
420
|
+
openRe.lastIndex = s.length;
|
|
421
|
+
} else {
|
|
422
|
+
openRe.lastIndex = m.index + m[0].length;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
// Reasoning tags: always claim (stripped from prose, never executed).
|
|
429
|
+
for (const base of THINK_TAGS) {
|
|
430
|
+
const openRe = new RegExp(`<(?:antml:)?${base}(?=[\\s>/])`, "g");
|
|
431
|
+
while ((m = openRe.exec(s)) !== null) {
|
|
432
|
+
const gt = s.indexOf(">", m.index);
|
|
433
|
+
if (gt === -1) break;
|
|
434
|
+
const bodyStart = gt + 1;
|
|
435
|
+
const closeIdx = findXmlClose(s, bodyStart, base);
|
|
436
|
+
const end = closeIdx === -1 ? s.length : closeIdx;
|
|
437
|
+
regions.push({
|
|
438
|
+
start: m.index,
|
|
439
|
+
end,
|
|
440
|
+
kind: "think",
|
|
441
|
+
body: closeIdx === -1 ? s.slice(bodyStart) : s.slice(bodyStart, closeStartOf(s, bodyStart, base, closeIdx)),
|
|
442
|
+
});
|
|
443
|
+
openRe.lastIndex = end;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
// Bare JSON objects on their own line(s), claimed only when they
|
|
448
|
+
// normalize to a known tool. JSON embedded inside a sentence stays prose.
|
|
449
|
+
const bareOpen = /(^|\n)([ \t]*)\{/g;
|
|
450
|
+
while ((m = bareOpen.exec(s)) !== null) {
|
|
451
|
+
const openIdx = m.index + m[0].length - 1;
|
|
452
|
+
const closeIdx = jsonSpan(s, openIdx);
|
|
453
|
+
if (closeIdx === -1) continue;
|
|
454
|
+
// Trailing spaces allowed, but the newline itself is left in the text
|
|
455
|
+
// so the next scan still sees line starts.
|
|
456
|
+
const after = s.slice(closeIdx + 1).match(/^[ \t]*(?=\r?\n|$)/);
|
|
457
|
+
if (!after) continue;
|
|
458
|
+
const span = s.slice(openIdx, closeIdx + 1);
|
|
459
|
+
try {
|
|
460
|
+
const n = normalizeCall(JSON.parse(span));
|
|
461
|
+
if (n.ok && TOOL_NAMES.includes(n.call.tool)) {
|
|
462
|
+
regions.push({ start: m.index, end: closeIdx + 1 + after[0].length, kind: "barejson", body: span });
|
|
463
|
+
bareOpen.lastIndex = closeIdx + 1;
|
|
464
|
+
}
|
|
465
|
+
} catch {
|
|
466
|
+
// Not JSON, or not a tool call — prose.
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// Resolve overlaps: earliest start wins, longest first on ties. A wrapper
|
|
471
|
+
// (function_calls, think) starts before its children, so children inside
|
|
472
|
+
// an accepted wrapper are skipped here and parsed during wrapper parsing.
|
|
473
|
+
regions.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
474
|
+
const accepted = [];
|
|
475
|
+
let lastEnd = -1;
|
|
476
|
+
for (const r of regions) {
|
|
477
|
+
if (r.start >= lastEnd) {
|
|
478
|
+
accepted.push(r);
|
|
479
|
+
lastEnd = r.end;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
return accepted;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* Parses one region into result items: {call, via} | {error, raw}.
|
|
487
|
+
* kind "think" yields nothing (stripped reasoning).
|
|
488
|
+
*/
|
|
489
|
+
function parseRegion(region, depth) {
|
|
490
|
+
if (region.kind === "think") return [];
|
|
491
|
+
if (region.kind === "fence" || region.kind === "askfence") {
|
|
492
|
+
const raw = region.body.trim();
|
|
493
|
+
if (!raw) return [];
|
|
494
|
+
let parsed;
|
|
495
|
+
try {
|
|
496
|
+
parsed = JSON.parse(raw);
|
|
497
|
+
} catch {
|
|
498
|
+
return [{ error: "tool block is not valid JSON", raw: excerpt(region.body) }];
|
|
499
|
+
}
|
|
500
|
+
if (region.kind === "askfence" && parsed && typeof parsed === "object" && !Array.isArray(parsed) && !parsed.tool) {
|
|
501
|
+
parsed.tool = "ask";
|
|
502
|
+
}
|
|
503
|
+
if (Array.isArray(parsed)) {
|
|
504
|
+
const items = [];
|
|
505
|
+
for (const item of parsed) {
|
|
506
|
+
const n = normalizeCall(item);
|
|
507
|
+
if (!n.ok) items.push({ error: n.error, raw: excerpt(JSON.stringify(item)) });
|
|
508
|
+
else if (!TOOL_NAMES.includes(n.call.tool)) items.push({ error: `unknown tool (want one of: ${TOOL_NAMES.join(", ")})`, raw: excerpt(JSON.stringify(item)) });
|
|
509
|
+
else items.push({ call: n.call, via: "fence" });
|
|
510
|
+
}
|
|
511
|
+
return items;
|
|
512
|
+
}
|
|
513
|
+
const n = normalizeCall(parsed);
|
|
514
|
+
if (!n.ok) return [{ error: n.error, raw: excerpt(region.body) }];
|
|
515
|
+
if (!TOOL_NAMES.includes(n.call.tool)) {
|
|
516
|
+
return [{ error: `unknown tool (want one of: ${TOOL_NAMES.join(", ")})`, raw: excerpt(region.body) }];
|
|
517
|
+
}
|
|
518
|
+
return [{ call: n.call, via: "fence" }];
|
|
519
|
+
}
|
|
520
|
+
if (region.kind === "jsonfence" || region.kind === "barejson") {
|
|
521
|
+
let parsed;
|
|
522
|
+
try {
|
|
523
|
+
parsed = JSON.parse(region.body.trim());
|
|
524
|
+
} catch {
|
|
525
|
+
return [{ error: "tool call is not valid JSON", raw: excerpt(region.body) }];
|
|
526
|
+
}
|
|
527
|
+
const n = normalizeCall(parsed);
|
|
528
|
+
if (!n.ok) return [{ error: n.error, raw: excerpt(region.body) }];
|
|
529
|
+
if (!TOOL_NAMES.includes(n.call.tool)) {
|
|
530
|
+
return [{ error: `unknown tool (want one of: ${TOOL_NAMES.join(", ")})`, raw: excerpt(region.body) }];
|
|
531
|
+
}
|
|
532
|
+
return [{ call: n.call, via: "json" }];
|
|
533
|
+
}
|
|
534
|
+
// XML block.
|
|
535
|
+
const r = xmlToCall(region.attrs || {}, region.body || "");
|
|
536
|
+
if (r && r.ok) return r.calls.map((call) => ({ call, via: "xml" }));
|
|
537
|
+
if (r && r.recurse && depth < MAX_DEPTH) {
|
|
538
|
+
const inner = extractFromText(region.body || "", depth + 1);
|
|
539
|
+
if (inner.calls.length || inner.errors.length) return [...inner.calls, ...inner.errors];
|
|
540
|
+
return [{ error: "wrapper block held no tool calls", raw: excerpt(region.body) }];
|
|
541
|
+
}
|
|
542
|
+
return [{ error: (r && r.error) || "unparseable tool block", raw: excerpt(region.body) }];
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
function extractFromText(s, depth = 0) {
|
|
546
|
+
const regions = collectRegions(s, depth);
|
|
547
|
+
const calls = [];
|
|
548
|
+
const errors = [];
|
|
549
|
+
for (const region of regions) {
|
|
550
|
+
if (calls.length + errors.length >= MAX_TOOL_BLOCKS) break;
|
|
551
|
+
for (const item of parseRegion(region, depth)) {
|
|
552
|
+
if (calls.length + errors.length >= MAX_TOOL_BLOCKS) break;
|
|
553
|
+
if (item.call) calls.push(item);
|
|
554
|
+
else errors.push({ raw: item.raw || "", error: item.error || "bad tool block" });
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
const cleaned = cleanProse(s, regions);
|
|
558
|
+
return { calls, errors, cleaned, regions };
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
// Prose with every claimed region (tool markup, think blocks) removed.
|
|
562
|
+
function cleanProse(s, regions) {
|
|
563
|
+
if (!regions.length) return s.trim();
|
|
564
|
+
let out = "";
|
|
565
|
+
let pos = 0;
|
|
566
|
+
for (const r of regions) {
|
|
567
|
+
if (r.start > pos) out += s.slice(pos, r.start);
|
|
568
|
+
pos = Math.max(pos, r.end);
|
|
569
|
+
}
|
|
570
|
+
if (pos < s.length) out += s.slice(pos);
|
|
571
|
+
return out.replace(/\n{3,}/g, "\n\n").trim();
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Public API. Returns:
|
|
576
|
+
* { calls: [{tool, ..., via}], errors: [{raw, error}], cleaned,
|
|
577
|
+
* via: {fence, xml, json}, usedNonFence }
|
|
578
|
+
* `via` marks how each call arrived so the loop can coach the model back
|
|
579
|
+
* to canonical fences.
|
|
580
|
+
*/
|
|
581
|
+
function extractToolCalls(text) {
|
|
582
|
+
const { calls, errors, cleaned } = extractFromText(String(text ?? ""));
|
|
583
|
+
const via = { fence: 0, xml: 0, json: 0 };
|
|
584
|
+
for (const call of calls) {
|
|
585
|
+
const v = call.via || "fence";
|
|
586
|
+
if (via[v] != null) via[v] += 1;
|
|
587
|
+
}
|
|
588
|
+
return {
|
|
589
|
+
calls: calls.map((c) => c.call),
|
|
590
|
+
errors,
|
|
591
|
+
cleaned,
|
|
592
|
+
via,
|
|
593
|
+
usedNonFence: via.xml + via.json > 0,
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
// ---------------------------------------------------------------------------
|
|
598
|
+
// Streaming filter: keeps machine syntax (tool blocks, XML calls, think
|
|
599
|
+
// tags) off the operator's screen while tokens stream in. Chunk-boundary
|
|
600
|
+
// safe: holds back any suffix that could be the start of a trigger, swallows
|
|
601
|
+
// whole regions once their close arrives, and drops a held-open tag after
|
|
602
|
+
// FILTER_HOLD_LIMIT chars so an unclosed tag can never eat the answer.
|
|
603
|
+
|
|
604
|
+
const FENCE_TRIGGERS = ["```tool", "```ask", "~~~tool", "~~~ask"];
|
|
605
|
+
const FILTER_TAGS = [...XML_TAGS, ...THINK_TAGS];
|
|
606
|
+
const TAG_TRIGGERS = FILTER_TAGS.flatMap((b) => [`<${b}`, `<antml:${b}`]);
|
|
607
|
+
|
|
608
|
+
function holdBackLen(buf) {
|
|
609
|
+
const max = Math.min(buf.length, 24);
|
|
610
|
+
for (let k = max; k >= 1; k -= 1) {
|
|
611
|
+
const tail = buf.slice(buf.length - k);
|
|
612
|
+
for (const list of [FENCE_TRIGGERS, TAG_TRIGGERS]) {
|
|
613
|
+
for (const t of list) {
|
|
614
|
+
if (t.startsWith(tail)) return tail.length;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
return 0;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
// Longest-first alternation; the lookahead rejects longer names sharing a
|
|
622
|
+
// prefix (e.g. <tool_name> is not <tool).
|
|
623
|
+
const FENCE_TRIGGER_RE = /(?:^|\n)[ \t]*(```|~~~)[ \t]*(tool|ask)(?=[\s]|$)/;
|
|
624
|
+
const TAG_TRIGGER_RE = new RegExp(
|
|
625
|
+
`<(antml:)?(${FILTER_TAGS.join("|")})(?=[\\s>/])`,
|
|
626
|
+
);
|
|
627
|
+
|
|
628
|
+
function findTrigger(buf) {
|
|
629
|
+
const mf = FENCE_TRIGGER_RE.exec(buf);
|
|
630
|
+
const mt = TAG_TRIGGER_RE.exec(buf);
|
|
631
|
+
if (mf && (!mt || mf.index <= mt.index)) {
|
|
632
|
+
return { index: mf.index, marker: mf[1], fence: mf[2], end: mf.index + mf[0].length };
|
|
633
|
+
}
|
|
634
|
+
if (mt) return { index: mt.index, tag: mt[2], spell: mt[1] || "" };
|
|
635
|
+
return null;
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// Resolve the close for a trigger in buf. Returns {state:"closed", end},
|
|
639
|
+
// or {state:"wait", openEnd} while the close has not arrived (openEnd =
|
|
640
|
+
// index after the open tag / open line, for the give-up drop).
|
|
641
|
+
function findClose(buf, trig) {
|
|
642
|
+
if (trig.fence) {
|
|
643
|
+
// Match the parser's body-start rule: skip past the open line.
|
|
644
|
+
const nl = buf.indexOf("\n", trig.end);
|
|
645
|
+
const bodyFrom = nl === -1 ? buf.length : nl + 1;
|
|
646
|
+
const chosen = chooseFenceClose(buf, bodyFrom, trig.marker);
|
|
647
|
+
if (!chosen) return { state: "wait", openEnd: trig.end - trig.index };
|
|
648
|
+
return { state: "closed", end: chosen.end };
|
|
649
|
+
}
|
|
650
|
+
const gt = buf.indexOf(">", trig.index);
|
|
651
|
+
const openEnd = gt === -1 ? buf.length : gt + 1;
|
|
652
|
+
if (gt === -1) return { state: "wait", openEnd };
|
|
653
|
+
if (/\/\s*>$/.test(buf.slice(trig.index, gt + 1))) return { state: "closed", end: gt + 1 };
|
|
654
|
+
const closeIdx = findXmlClose(buf, gt + 1, trig.tag);
|
|
655
|
+
if (closeIdx === -1) return { state: "wait", openEnd };
|
|
656
|
+
return { state: "closed", end: closeIdx };
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/**
|
|
660
|
+
* Creates a chunk-boundary-safe markup filter. `emit(text)` receives only
|
|
661
|
+
* operator-visible text. write(chunk) and end() stream through it.
|
|
662
|
+
*/
|
|
663
|
+
function createMarkupFilter(emit) {
|
|
664
|
+
let buf = "";
|
|
665
|
+
let pendingNl = false; // separate text after a swallowed block
|
|
666
|
+
let holdOpenEnd = -1; // open-tag end while waiting for a close
|
|
667
|
+
|
|
668
|
+
const out = (t) => {
|
|
669
|
+
if (!t) return;
|
|
670
|
+
if (pendingNl && !t.startsWith("\n")) out2("\n");
|
|
671
|
+
pendingNl = false;
|
|
672
|
+
out2(t);
|
|
673
|
+
};
|
|
674
|
+
const out2 = (t) => {
|
|
675
|
+
if (t) emit(t);
|
|
676
|
+
};
|
|
677
|
+
|
|
678
|
+
function giveUpHold() {
|
|
679
|
+
if (holdOpenEnd < 0 || holdOpenEnd >= buf.length) return false;
|
|
680
|
+
// Only give up when the close is genuinely never coming: the buffer has
|
|
681
|
+
// grown well past the open tag with no close in sight. Until then the
|
|
682
|
+
// held region is machine syntax that must stay invisible.
|
|
683
|
+
if (buf.length <= holdOpenEnd + FILTER_HOLD_LIMIT) return false;
|
|
684
|
+
buf = buf.slice(holdOpenEnd);
|
|
685
|
+
holdOpenEnd = -1;
|
|
686
|
+
pendingNl = true;
|
|
687
|
+
return true;
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
function process(final) {
|
|
691
|
+
if (!final && giveUpHold()) {
|
|
692
|
+
// Open tag dropped; keep scanning what follows.
|
|
693
|
+
}
|
|
694
|
+
for (;;) {
|
|
695
|
+
const trig = findTrigger(buf);
|
|
696
|
+
if (!trig) break;
|
|
697
|
+
const before = buf.slice(0, trig.index);
|
|
698
|
+
if (before) out(before);
|
|
699
|
+
buf = buf.slice(trig.index);
|
|
700
|
+
const close = findClose(buf, trig);
|
|
701
|
+
if (close.state === "closed") {
|
|
702
|
+
buf = buf.slice(close.end);
|
|
703
|
+
holdOpenEnd = -1;
|
|
704
|
+
pendingNl = true;
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
if (!final) {
|
|
708
|
+
holdOpenEnd = close.openEnd;
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
// Unclosed at end of stream: swallow the rest — machine syntax.
|
|
712
|
+
buf = "";
|
|
713
|
+
holdOpenEnd = -1;
|
|
714
|
+
break;
|
|
715
|
+
}
|
|
716
|
+
if (final) {
|
|
717
|
+
if (buf) out(buf);
|
|
718
|
+
buf = "";
|
|
719
|
+
holdOpenEnd = -1;
|
|
720
|
+
return;
|
|
721
|
+
}
|
|
722
|
+
const hold = holdBackLen(buf);
|
|
723
|
+
if (hold && hold < buf.length) {
|
|
724
|
+
out(buf.slice(0, buf.length - hold));
|
|
725
|
+
buf = buf.slice(buf.length - hold);
|
|
726
|
+
} else if (!hold) {
|
|
727
|
+
out(buf);
|
|
728
|
+
buf = "";
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
return {
|
|
733
|
+
write(chunk) {
|
|
734
|
+
if (!chunk) return;
|
|
735
|
+
buf += chunk;
|
|
736
|
+
process(false);
|
|
737
|
+
},
|
|
738
|
+
end() {
|
|
739
|
+
process(true);
|
|
740
|
+
buf = "";
|
|
741
|
+
holdOpenEnd = -1;
|
|
742
|
+
},
|
|
743
|
+
};
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
module.exports = {
|
|
747
|
+
TOOL_NAMES,
|
|
748
|
+
MAX_TOOL_BLOCKS,
|
|
749
|
+
extractToolCalls,
|
|
750
|
+
normalizeCall,
|
|
751
|
+
createMarkupFilter,
|
|
752
|
+
};
|