@possumtech/rummy 2.0.1 → 2.1.0

package/src/agent/XmlParser.js

@@ -1,4 +1,3 @@
-import { Parser } from "htmlparser2";
 import { parseEditContent } from "../plugins/hedberg/edits.js";
 import { parseJsonEdit } from "../plugins/hedberg/normalize.js";
 import { parseSed } from "../plugins/hedberg/sed.js";
@@ -13,15 +12,11 @@ export const ALL_TOOLS = new Set([
 	"think",
 ]);
 
-/**
- * Resolve the competing attr-vs-body philosophies per tool.
- * If the canonical attribute is missing, the body fills it. Silent.
- */
+// Per-tool resolution: missing canonical attribute is filled silently from the body.
 function resolveCommand(name, a, rawBody) {
 	const trimmed = rawBody.trim();
 
 	if (name === "set") {
-		// Structured edit detection — merge conflict, udiff, Claude XML
 		const hasEdit =
 			/<{3,12} SEARCH/.test(trimmed) ||
 			/>{3,12} REPLACE/.test(trimmed) ||
@@ -35,17 +30,15 @@ function resolveCommand(name, a, rawBody) {
 					name,
 					path: a.path,
 					body: a.body,
-					preview: a.preview,
+					manifest: a.manifest,
 					blocks,
 				};
 			}
 		}
-		// JSON-style { search, replace }
 		const jsonEdit = parseJsonEdit(trimmed);
 		if (jsonEdit) {
 			return { name, path: a.path, ...jsonEdit };
 		}
-		// Sed syntax: s/search/replace/flags — supports chained commands
 		if (trimmed.startsWith("s/")) {
 			const blocks = parseSed(trimmed);
 			if (blocks?.length === 1) {
@@ -62,29 +55,26 @@ function resolveCommand(name, a, rawBody) {
 				return { name, path: a.path, blocks };
 			}
 		}
-		// search+replace attrs → attribute edit mode
 		if (a.search) {
 			const replace = a.replace ?? trimmed;
 			return {
 				name,
 				path: a.path,
 				body: a.body,
-				preview: a.preview,
+				manifest: a.manifest,
 				search: a.search,
 				replace,
 			};
 		}
-		// Body attr + body content → search/replace (attr is search, body is replace)
 		if (trimmed && a.body) {
 			return {
 				name,
 				path: a.path,
 				search: a.body,
 				replace: trimmed,
-				preview: a.preview,
+				manifest: a.manifest,
 			};
 		}
-		// Plain write or visibility change
 		const body = trimmed || a.body || "";
 		return { name, ...a, body };
 	}
@@ -96,10 +86,6 @@ function resolveCommand(name, a, rawBody) {
 	}
 
 	if (name === "get" || name === "rm") {
-		// Spread `a` so `line`, `limit`, `visibility`, and future attrs
-		// reach the handler. Earlier narrow extraction silently dropped
-		// `line=/limit=` and stranded the partial-read path advertised
-		// in getDoc.
 		return { name, ...a, path: a.path || trimmed || null };
 	}
 
@@ -110,10 +96,6 @@ function resolveCommand(name, a, rawBody) {
 	}
 
 	if (name === "mv" || name === "cp") {
-		// Spread `a` so `visibility` reaches the handler. mvDoc
-		// advertises `<mv path="known://..." visibility="summarized"/>`
-		// for batch visibility changes and was silently stripping that
-		// attr before.
 		return { name, ...a, path: a.path, to: a.to || trimmed || null };
 	}
 
@@ -131,176 +113,102 @@ function resolveCommand(name, a, rawBody) {
 	return { name, ...a, body: trimmed || a.body };
 }
 
+const NAME_CHAR = /[a-zA-Z0-9_]/;
+const ATTR_KEY_CHAR = /[a-zA-Z0-9_:-]/;
+const WS = /\s/;
+
+// Recovery-tolerant tokenizer for rummy's closed set of tool tags.
+//
+// Design contract:
+//   - Tool tags (<get>, <set>, <sh>, ...) are the only syntactic special tags.
+//     Any other "<...>" sequence in OUTER text is treated as literal text.
+//   - Inside a tool tag's body, content is OPAQUE: only the matching close
+//     tag is recognized. Body may contain regex (`(?<!`), generics (`Vec<u8>`),
+//     HTML, XML, heredocs, comparison operators — none of it affects parsing.
+//   - Backtick spans (`...`) and triple-backtick fences (```...```) at the
+//     OUTER level neutralize tag-like content, mirroring the markdown
+//     convention that documentation about a tool isn't a tool call.
+//     Inside tool bodies this tracking does NOT apply (body opacity wins).
+//   - Same-name nesting (`<set>...<set/>...</set>`) is depth-counted so
+//     nested examples don't prematurely close the outer.
+//   - Recovery: unclosed openers capture body to EOF + emit a warning.
+//     Orphan closes at outer level become text, no warning (body opacity
+//     means models legitimately write `</set>` in prose / summaries).
 export default class XmlParser {
-	/**
-	 * Parse tool commands from model content using htmlparser2.
-	 * Handles malformed XML gracefully — unclosed tags, missing slashes, etc.
-	 * Every tool can appear as self-closing (attrs only) or with body content.
-	 * Competing attr-vs-body philosophies are resolved silently.
-	 * @param {string} content - Raw model response text
-	 * @returns {{ commands: Array, warnings: string[], unparsed: string }}
-	 */
 	static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS);
 
 	static parse(content) {
 		if (!content) return { commands: [], warnings: [], unparsed: "" };
 
-		// Normalize native tool call formats to rummy XML
 		const normalized = XmlParser.#normalizeToolCalls(content);
+		return XmlParser.#tokenize(normalized, []);
+	}
 
+	static #tokenize(s, warnings) {
 		const commands = [];
-		const warnings = [];
-		const textChunks = [];
-
-		// Pre-flight: neutralize tool tags inside markdown code spans.
-		// Models quote instructions containing `<get/>` etc. — the parser
-		// would treat them as real tool calls. Replace the angle brackets
-		// inside backtick spans so htmlparser2 ignores them.
-		const codeNeutralized = XmlParser.#neutralizeCodeSpans(normalized);
-
-		// Pre-flight: fix mismatched close tags that htmlparser2 silently
-		// drops (making our onclosetag recovery code unreachable). Must run
-		// before balanceAttrQuotes since the mismatch scan needs clean tags.
-		const mismatchFixed = XmlParser.#correctMismatchedCloses(
-			codeNeutralized,
-			warnings,
-		);
-
-		// Pre-flight: balance unclosed attribute quotes that would otherwise
-		// cause htmlparser2 to consume the rest of input as a single attribute
-		// value, silently dropping every subsequent tool call.
-		const balanced = XmlParser.#balanceAttrQuotes(mismatchFixed, warnings);
-		let current = null;
-		let ended = false;
+		const text = [];
+		let i = 0;
+		let inSingleBacktick = false;
+		let inTripleFence = false;
 		let capped = false;
 
-		const parser = new Parser(
-			{
-				onopentag(name, attrs) {
-					if (capped) return;
-
-					if (current) {
-						// Empty-body case: current tool opened but got no text
-						// content before a new tag. The model likely meant current
-						// to self-close but typed it in paired form, or emitted a
-						// mismatched close tag that htmlparser2 silently dropped.
-						// Close current, open new.
-						const hasBody = current.rawBody.trim() !== "";
-						const hasNestedOpens = (current.nested || []).length > 0;
-						if (!hasBody && !hasNestedOpens && ALL_TOOLS.has(name)) {
-							warnings.push(
-								`Unclosed <${current.name}> before <${name}> — recovered`,
-							);
-							commands.push(
-								resolveCommand(current.name, current.attrs, current.rawBody),
-							);
-							current = null;
-						} else {
-							// Nested tag inside a body with content — treat as body
-							// text. Tool bodies are opaque: the model writing a plan
-							// with <get/> in it, SEARCH/REPLACE in <set>, or XML
-							// examples in <known> all need to survive intact. Track
-							// nested opens on a stack so matching closes pop off and
-							// orphan closes (typos) still trigger recovery.
-							const attrStr = Object.entries(attrs)
-								.map(([k, v]) => (v === "" ? k : `${k}="${v}"`))
-								.join(" ");
-							current.rawBody += attrStr ? `<${name} ${attrStr}>` : `<${name}>`;
-							current.nested ||= [];
-							current.nested.push(name);
-							return;
-						}
-					}
-
-					if (!ALL_TOOLS.has(name)) return;
+		while (i < s.length) {
+			if (commands.length >= XmlParser.MAX_COMMANDS) {
+				capped = true;
+				break;
+			}
 
-					if (commands.length >= XmlParser.MAX_COMMANDS) {
-						capped = true;
-						return;
-					}
+			// Triple-backtick fence toggles take precedence over single backtick
+			// because ``` overlaps `.
+			if (s[i] === "`" && s[i + 1] === "`" && s[i + 2] === "`") {
+				inTripleFence = !inTripleFence;
+				text.push("```");
+				i += 3;
+				continue;
+			}
+			if (s[i] === "`" && !inTripleFence) {
+				inSingleBacktick = !inSingleBacktick;
+				text.push("`");
+				i++;
+				continue;
+			}
 
-					current = { name, attrs, rawBody: "", nested: [] };
-				},
+			if (inSingleBacktick || inTripleFence || s[i] !== "<") {
+				text.push(s[i]);
+				i++;
+				continue;
+			}
 
-				ontext(text) {
-					if (capped) return;
-					if (current) {
-						current.rawBody += text;
-					} else {
-						textChunks.push(text);
-					}
-				},
-
-				onclosetag(name, isImplied) {
-					if (capped) return;
-
-					if (current) {
-						// Matching nested close — pop stack, keep as text.
-						const nested = current.nested;
-						if (nested.length > 0 && nested[nested.length - 1] === name) {
-							nested.pop();
-							current.rawBody += `</${name}>`;
-							return;
-						}
-
-						// Matching close for outer tool — finalize.
-						if (name === current.name && nested.length === 0) {
-							if (ended) {
-								warnings.push(
-									`Unclosed <${name}> tag — content captured anyway`,
-								);
-							}
-							commands.push(
-								resolveCommand(current.name, current.attrs, current.rawBody),
-							);
-							current = null;
-							return;
-						}
-
-						// Orphan close for a known tool (likely typo) — recover.
-						if (ALL_TOOLS.has(name)) {
-							warnings.push(
-								`Mismatched </${name}> closing <${current.name}> — recovered`,
-							);
-							commands.push(
-								resolveCommand(current.name, current.attrs, current.rawBody),
-							);
-							current = null;
-							return;
-						}
-
-						// Unknown orphan close — text.
-						current.rawBody += `</${name}>`;
-						return;
-					}
+			const opener = XmlParser.#matchOpener(s, i);
+			if (!opener) {
+				text.push(s[i]);
+				i++;
+				continue;
+			}
 
-					if (isImplied && ALL_TOOLS.has(name)) {
-						// Self-closing tag that htmlparser2 auto-closed at top level
-					}
-				},
+			const { name, attrs, selfClose, end: openerEnd } = opener;
 
-				onerror(err) {
-					warnings.push(`Parse error: ${err.message}`);
-				},
-			},
-			{
-				recognizeSelfClosing: true,
-				lowerCaseTags: true,
-				lowerCaseAttributeNames: true,
-			},
-		);
+			if (selfClose) {
+				commands.push(resolveCommand(name, attrs, ""));
+				i = openerEnd;
+				continue;
+			}
 
-		parser.write(balanced);
-		ended = true;
-		parser.end();
+			const result = XmlParser.#findBodyEnd(s, name, openerEnd);
+			const body = s.slice(openerEnd, result.bodyEnd);
+			if (result.unclosed) {
+				warnings.push(`Unclosed <${name}> tag — content captured anyway`);
+			} else if (result.mismatchedCloseName) {
+				warnings.push(
+					`Mismatched </${result.mismatchedCloseName}> closing <${name}> — corrected to </${name}>`,
+				);
+			}
+			commands.push(resolveCommand(name, attrs, body));
+			i = result.afterClose;
 
-		// Flush any unclosed tool tag
-		if (current && !capped) {
-			warnings.push(`Unclosed <${current.name}> tag — content captured anyway`);
-			commands.push(
-				resolveCommand(current.name, current.attrs, current.rawBody),
-			);
-			current = null;
+			// Body terminated; reset outer-text fence tracking.
+			inSingleBacktick = false;
+			inTripleFence = false;
 		}
 
 		if (capped) {
@@ -309,109 +217,187 @@ export default class XmlParser {
 			);
 		}
 
-		const unparsed = textChunks.join("").trim();
-		return { commands, warnings, unparsed };
+		return {
+			commands,
+			warnings,
+			unparsed: text.join("").trim(),
+		};
 	}
 
-	/**
-	 * Repair a specific malformed-tag pattern: an attribute value opened with
-	 * `="` that never closes before the next tag. Without repair, htmlparser2
-	 * consumes the rest of input as one giant attribute value and silently
-	 * drops every subsequent tool call.
-	 *
-	 * Pattern matched:  <TAG ... ATTR="text-with-no-quote</NEXT>
-	 * Repair:           <TAG ... ATTR="text-with-no-quote"></NEXT>
-	 *
-	 * Conservative — only triggers when the value contains no quote, no `>`,
-	 * and is followed by another tag opening or close. Well-formed input is
-	 * untouched.
-	 */
-	static #balanceAttrQuotes(content, warnings) {
-		let fixes = 0;
-		const repaired = content.replace(
-			/(<\w+\s[^<>]*?\w+=")([^"<>]*?)(<\/?\w+)/g,
-			(_, opening, value, nextTag) => {
-				fixes++;
-				return `${opening}${value}">${nextTag}`;
-			},
-		);
-		if (fixes > 0) {
-			warnings.push(
-				`Repaired ${fixes} malformed attribute(s) — close all attribute values with a quote.`,
-			);
+	// Returns { name, attrs, selfClose, end } if `s[pos..]` opens a known tool,
+	// else null. `end` is the index after the closing `>` (or `/>`).
+	static #matchOpener(s, pos) {
+		if (s[pos] !== "<") return null;
+		let i = pos + 1;
+
+		const nameStart = i;
+		while (i < s.length && NAME_CHAR.test(s[i])) i++;
+		const name = s.slice(nameStart, i).toLowerCase();
+		if (!ALL_TOOLS.has(name)) return null;
+
+		// Char after the name must end the name token cleanly.
+		if (i < s.length && !WS.test(s[i]) && s[i] !== "/" && s[i] !== ">") {
+			return null;
+		}
+
+		const attrsStart = i;
+		let inQuote = null;
+
+		while (i < s.length) {
+			const c = s[i];
+			if (inQuote) {
+				if (c === inQuote) inQuote = null;
+				i++;
+				continue;
+			}
+			if (c === '"' || c === "'") {
+				inQuote = c;
+				i++;
+				continue;
+			}
+			if (c === "/") {
+				let k = i + 1;
+				while (k < s.length && WS.test(s[k])) k++;
+				if (s[k] === ">") {
+					return {
+						name,
+						attrs: XmlParser.#parseAttrs(s.slice(attrsStart, i)),
+						selfClose: true,
+						end: k + 1,
+					};
+				}
+				i++;
+				continue;
+			}
+			if (c === ">") {
+				return {
+					name,
+					attrs: XmlParser.#parseAttrs(s.slice(attrsStart, i)),
+					selfClose: false,
+					end: i + 1,
+				};
+			}
+			i++;
 		}
-		return repaired;
+
+		// Hit EOF without closing — not a parseable opener.
+		return null;
 	}
 
-	/**
-	 * Correct mismatched close tags before htmlparser2 sees them.
-	 *
-	 * htmlparser2 silently drops close tags that don't match the currently
-	 * open element (e.g. `<set>body</known>` — `</known>` vanishes). This
-	 * makes the explicit mismatch recovery in onclosetag unreachable and
-	 * causes all subsequent sibling commands to be absorbed as body text.
-	 *
-	 * Conservative: only corrects when the mismatch is at the outermost
-	 * tool depth (stack.length === 1). Nested mismatches inside body text
-	 * are left for htmlparser2 + body opacity to handle normally.
-	 */
-	/**
-	 * Neutralize XML tags inside markdown code spans so the parser
-	 * doesn't treat quoted tool names as real commands.
-	 * `<get/>` → `&lt;get/&gt;`  (htmlparser2 ignores entities)
-	 */
-	static #neutralizeCodeSpans(content) {
-		return content.replace(/`([^`]*)`/g, (match, inner) => {
-			if (!/<\/?[\w]/.test(inner)) return match;
-			return `\`${inner.replace(/</g, "&lt;").replace(/>/g, "&gt;")}\``;
-		});
+	static #parseAttrs(raw) {
+		const attrs = {};
+		let i = 0;
+		while (i < raw.length) {
+			while (i < raw.length && WS.test(raw[i])) i++;
+			if (i >= raw.length) break;
+
+			const keyStart = i;
+			while (i < raw.length && ATTR_KEY_CHAR.test(raw[i])) i++;
+			if (i === keyStart) {
+				i++;
+				continue;
+			}
+			const key = raw.slice(keyStart, i).toLowerCase();
+
+			while (i < raw.length && WS.test(raw[i])) i++;
+
+			if (raw[i] !== "=") {
+				attrs[key] = "";
+				continue;
+			}
+			i++;
+
+			while (i < raw.length && WS.test(raw[i])) i++;
+
+			if (raw[i] === '"' || raw[i] === "'") {
+				const quote = raw[i];
+				i++;
+				const valStart = i;
+				while (i < raw.length && raw[i] !== quote) i++;
+				attrs[key] = raw.slice(valStart, i);
+				if (raw[i] === quote) i++;
+			} else {
+				const valStart = i;
+				while (i < raw.length && !WS.test(raw[i])) i++;
+				attrs[key] = raw.slice(valStart, i);
+			}
+		}
+		return attrs;
 	}
 
-	static #correctMismatchedCloses(content, warnings) {
-		const stack = [];
-		return content.replace(
-			/<(\/?)(\w+)([^>]*?)(\/?)>/g,
-			(match, slash, tag, _attrs, selfClose) => {
-				if (!ALL_TOOLS.has(tag)) return match;
-				if (selfClose === "/") return match;
-				if (slash === "/") {
-					if (stack.length === 0) return match;
-					if (stack[stack.length - 1] === tag) {
-						stack.pop();
-						return match;
+	// Scans body content from `fromPos` until the matching `</name>` closer,
+	// counting depth so same-name nested examples don't prematurely close.
+	// Returns { bodyEnd, afterClose, unclosed, mismatchedCloseName }.
+	//
+	// Mismatched-close recovery: if we encounter `</X>` where X != name and X
+	// is not a depth-counted nested tag, we use a balance heuristic to decide
+	// whether the orphan close was a typo (recover here) or legitimate body
+	// content (continue scanning). Specifically: count `</name>` minus
+	// `<name` in the rest of the string; if non-positive, no real close
+	// exists ahead and the orphan must be the intended close.
+	static #findBodyEnd(s, name, fromPos) {
+		let depth = 1;
+		let i = fromPos;
+		while (i < s.length) {
+			if (s[i] !== "<") {
+				i++;
+				continue;
+			}
+			if (s[i + 1] === "/") {
+				const nameStart = i + 2;
+				let nameEnd = nameStart;
+				while (nameEnd < s.length && NAME_CHAR.test(s[nameEnd])) nameEnd++;
+				const closeName = s.slice(nameStart, nameEnd).toLowerCase();
+				let k = nameEnd;
+				while (k < s.length && WS.test(s[k])) k++;
+				const isCloseTag = s[k] === ">";
+
+				if (isCloseTag && closeName === name) {
+					depth--;
+					if (depth === 0) {
+						return { bodyEnd: i, afterClose: k + 1, unclosed: false };
 					}
-					if (stack.length === 1) {
-						const top = stack.pop();
-						warnings.push(
-							`Mismatched </${tag}> closing <${top}> — corrected to </${top}>`,
-						);
-						return `</${top}>`;
+					i = k + 1;
+					continue;
+				}
+
+				if (isCloseTag && closeName.length > 0) {
+					const rest = s.slice(k + 1);
+					const closesAhead = (
+						rest.match(new RegExp(`<\\/${name}\\b\\s*>`, "g")) || []
+					).length;
+					const opensAhead = (rest.match(new RegExp(`<${name}\\b`, "g")) || [])
+						.length;
+					if (closesAhead - opensAhead < 1) {
+						return {
+							bodyEnd: i,
+							afterClose: k + 1,
+							unclosed: false,
+							mismatchedCloseName: closeName,
+						};
 					}
-					return match;
 				}
-				stack.push(tag);
-				return match;
-			},
-		);
+			}
+			const opener = XmlParser.#matchOpener(s, i);
+			if (opener && opener.name === name && !opener.selfClose) {
+				depth++;
+				i = opener.end;
+				continue;
+			}
+			i++;
+		}
+		return { bodyEnd: s.length, afterClose: s.length, unclosed: true };
 	}
 
-	/**
-	 * Normalize native tool call formats to rummy XML.
-	 * Models sometimes emit their training-format tool calls instead of
-	 * our XML tags. The intent is unambiguous — translate silently.
-	 */
+	// Translate native training-format tool calls into rummy XML silently.
 	static #normalizeToolCalls(content) {
-		// Gemma: ```tool_code\n<xml>...\n``` — strip code fences around valid XML
+		// Gemma code-fenced XML.
 		let result = content.replace(
 			/```(?:tool_code|tool_command|xml)\n([\s\S]*?)```/g,
 			(_, inner) => inner.trim(),
 		);
 
-		// Qwen/gemma: <|tool_call>call:NAME{key:"value"}<tool_call|>
-		// NAME may be namespaced with any of /, :, or . separators
-		// (e.g. `rummy.nvim/get`, `rummy:get`) — extract the trailing word
-		// sequence as the tool name. Value forms observed in the wild:
-		//   key="v" / key:"v" / key:v (unquoted) / key:<|"|>v<|"|> (gemma chat-quotes)
+		// Qwen/gemma <|tool_call>call:NAME{...}<tool_call|>; NAME may be namespaced.
 		result = result.replace(
 			/<\|tool_call>call:([\w.:/-]+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
 			(match, qualifiedName, params) => {
@@ -469,28 +455,16 @@ export default class XmlParser {
 			},
 		);
 
-		// Catch-all: any remaining <|tool_call> tokens are malformed native
-		// attempts (no {} block, missing close, wrong shape entirely). Replace
-		// each with an <error> so the model gets feedback on its next turn and
-		// learns to switch to XML. Lazy-match up to the next native close, the
-		// next XML close tag, or end of input — preserves any trailing valid XML.
-		// Error body must NOT contain literal <get>/<set>/etc. — those would
-		// re-enter the parser as phantom tool calls. Describe the format in
-		// prose instead and point at the tool docs above.
+		// Catch-all malformed <|tool_call> → <error> in prose (no literal tags or they'd re-parse).
 		result = result.replace(
 			/<\|tool_call>[\s\S]*?(?:<\|?tool_call\|?>|<\/\w+>|$)/g,
 			() =>
 				"<error>Native tool call format not supported. Use the XML commands listed above (e.g. a get tag with a path attribute, or a set tag with path and body).</error>",
 		);
 
-		// Strip any orphan chat-format quote tokens left after replacement.
 		result = result.replace(/<\|"\|>/g, '"');
 
-		// Gemma sometimes leaks OpenAI-harmony channel markers around its
-		// real XML output: `<|channel>thought\n<channel|>…<set path=…/>`.
-		// These aren't tool calls (handled above), they're role/channel
-		// tokens. Strip any remaining `<|name>` / `<name|>` pseudo-tags
-		// before the XML parser sees them.
+		// Strip OpenAI-harmony role/channel pseudo-tags (gemma leaks these).
 		result = result.replace(/<\|[\w:/-]+>/g, "");
 		result = result.replace(/<[\w:/-]+\|>/g, "");