@possumtech/rummy 2.0.1 → 2.2.0

package/src/agent/XmlParser.js

@@ -1,7 +1,47 @@
-import { Parser } from "htmlparser2";
-import { parseEditContent } from "../plugins/hedberg/edits.js";
-import { parseJsonEdit } from "../plugins/hedberg/normalize.js";
-import { parseSed } from "../plugins/hedberg/sed.js";
+import {
+	extractSingleHeredoc,
+	parseMarkerBody,
+} from "../lib/hedberg/marker.js";
+
+// Edit-marker body opacity. When `#findBodyEnd` is scanning a `<set>`
+// body and hits an opener, jump past the matching closer so tag-shaped
+// content inside the marker (`</set>`, `<get/>`, etc.) doesn't trigger
+// structural recovery.
+//
+// Two opener shapes are recognized for opacity:
+//   - `<<IDENT` — current edit syntax (parsed by marker.js).
+//   - `<<:::IDENT` — packet-rendering shape (engine emits via
+//     plugins/helpers.js). A model copy-pasting the packet shape into
+//     a `<set>` body should still get clean opacity even though
+//     marker.js routes such bodies to plain-body REPLACE.
+function skipBareMarker(s, pos) {
+	const m = s.slice(pos).match(/^<<([A-Z][A-Za-z0-9_]*)/);
+	if (!m) return null;
+	const ident = m[1];
+	const openerEnd = pos + m[0].length;
+	const escIdent = ident.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	const closerRe = new RegExp(`(?<=^|\\s)${escIdent}(?=[\\s<>]|$)`);
+	const cm = s.slice(openerEnd).match(closerRe);
+	if (!cm) return null;
+	return openerEnd + cm.index + cm[0].length;
+}
+
+function skipPacketMarker(s, pos) {
+	const m = s.slice(pos).match(/^<<:::([A-Za-z_][A-Za-z0-9_./-]*)/);
+	if (!m) return null;
+	const ident = m[1];
+	const openerEnd = pos + m[0].length;
+	const escIdent = ident.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	const closerRe = new RegExp(`:::${escIdent}(?![A-Za-z0-9_])`);
+	const cm = s.slice(openerEnd).match(closerRe);
+	if (!cm) return null;
+	return openerEnd + cm.index + cm[0].length;
+}
+
+function skipEditMarker(s, pos) {
+	if (s.startsWith("<<:::", pos)) return skipPacketMarker(s, pos);
+	return skipBareMarker(s, pos);
+}
 
 const STORE_TOOLS = new Set(["get", "rm", "set", "mv", "cp", "search"]);
 export const ALL_TOOLS = new Set([
@@ -13,80 +53,45 @@ export const ALL_TOOLS = new Set([
 	"think",
 ]);
 
-/**
- * Resolve the competing attr-vs-body philosophies per tool.
- * If the canonical attribute is missing, the body fills it. Silent.
- */
+// Per-tool resolution: missing canonical attribute is filled silently from the body.
 function resolveCommand(name, a, rawBody) {
+	// Generic heredoc affordance: any non-`<set>` plugin's body may be
+	// wrapped in a single `<<IDENT...IDENT` heredoc to opaquely contain
+	// multi-line scripts, tag-shaped prose, or content with special
+	// characters. Plugins consume the unwrapped inner body verbatim;
+	// the IDENT is exposed as `heredocIdent` on the command for plugins
+	// that want to act on the label. `<set>` is exempt because it does
+	// its own multi-op heredoc parsing via `parseMarkerBody`.
+	if (name !== "set") {
+		const heredoc = extractSingleHeredoc(rawBody);
+		if (heredoc) {
+			rawBody = heredoc.content;
+			a = { ...a, heredocIdent: heredoc.ident };
+		}
+	}
 	const trimmed = rawBody.trim();
 
 	if (name === "set") {
-		// Structured edit detection — merge conflict, udiff, Claude XML
-		const hasEdit =
-			/<{3,12} SEARCH/.test(trimmed) ||
-			/>{3,12} REPLACE/.test(trimmed) ||
-			(trimmed.includes("@@") &&
-				(trimmed.includes("\n-") || trimmed.includes("\n+"))) ||
-			trimmed.includes("<old_text>");
-		if (hasEdit) {
-			const blocks = parseEditContent(rawBody);
-			if (blocks.length > 0) {
-				return {
-					name,
-					path: a.path,
-					body: a.body,
-					preview: a.preview,
-					blocks,
-				};
-			}
-		}
-		// JSON-style { search, replace }
-		const jsonEdit = parseJsonEdit(trimmed);
-		if (jsonEdit) {
-			return { name, path: a.path, ...jsonEdit };
-		}
-		// Sed syntax: s/search/replace/flags — supports chained commands
-		if (trimmed.startsWith("s/")) {
-			const blocks = parseSed(trimmed);
-			if (blocks?.length === 1) {
-				return {
-					name,
-					path: a.path,
-					search: blocks[0].search,
-					replace: blocks[0].replace,
-					flags: blocks[0].flags,
-					sed: true,
-				};
-			}
-			if (blocks?.length > 1) {
-				return { name, path: a.path, blocks };
-			}
-		}
-		// search+replace attrs → attribute edit mode
-		if (a.search) {
-			const replace = a.replace ?? trimmed;
-			return {
-				name,
-				path: a.path,
-				body: a.body,
-				preview: a.preview,
-				search: a.search,
-				replace,
-			};
-		}
-		// Body attr + body content → search/replace (attr is search, body is replace)
-		if (trimmed && a.body) {
-			return {
-				name,
-				path: a.path,
-				search: a.body,
-				replace: trimmed,
-				preview: a.preview,
-			};
-		}
-		// Plain write or visibility change
-		const body = trimmed || a.body || "";
-		return { name, ...a, body };
+		// `search`/`replace` as attributes is no longer in the grammar;
+		// strip them so they can't sneak past via the attribute spread.
+		const { search: _s, replace: _r, ...rest } = a;
+		a = rest;
+
+		// Self-close / no-body: visibility/metadata op.
+		if (!trimmed) return { name, ...a, body: a.body || "" };
+
+		// Edit syntax (SPEC.md "Edit Syntax"): walks the body for
+		// `<<:::IDENT...:::IDENT` markers and returns an ordered op
+		// list. No markers → plain body, treated as full-replace.
+		// Non-keyword IDENTs (path-flavored, identifier-flavored)
+		// route to REPLACE so the model gets a working write whatever
+		// IDENT it picks.
+		const { ops, error } = parseMarkerBody(rawBody);
+		if (error) return { name, ...a, error };
+		if (ops) return { name, ...a, operations: ops };
+
+		// No markers — plain body, full-replace.
+		return { name, ...a, body: trimmed };
 	}
 
 	if (name === "update") {
@@ -95,212 +100,157 @@ function resolveCommand(name, a, rawBody) {
 		return { name, ...a, body, status };
 	}
 
+	// Body shorthand fallback: when the attribute is unset (undefined),
+	// fall back to the trimmed body. Empty-string attrs are preserved
+	// as-is — handlers validate. `||` would conflate the two cases.
+	const fromBody = trimmed === "" ? null : trimmed;
+
 	if (name === "get" || name === "rm") {
-		// Spread `a` so `line`, `limit`, `visibility`, and future attrs
-		// reach the handler. Earlier narrow extraction silently dropped
-		// `line=/limit=` and stranded the partial-read path advertised
-		// in getDoc.
-		return { name, ...a, path: a.path || trimmed || null };
+		return { name, ...a, path: a.path ?? fromBody };
 	}
 
 	if (name === "search") {
-		const path = a.path || trimmed || null;
+		const path = a.path ?? fromBody;
 		const results = a.results ? Number(a.results) : null;
 		return { name, ...a, path, results };
 	}
 
 	if (name === "mv" || name === "cp") {
-		// Spread `a` so `visibility` reaches the handler. mvDoc
-		// advertises `<mv path="known://..." visibility="summarized"/>`
-		// for batch visibility changes and was silently stripping that
-		// attr before.
-		return { name, ...a, path: a.path, to: a.to || trimmed || null };
+		return { name, ...a, path: a.path, to: a.to ?? fromBody };
 	}
 
 	if (name === "sh" || name === "env") {
-		const command = a.command || trimmed || null;
+		const command = a.command ?? fromBody;
 		return { name, ...a, command };
 	}
 
 	if (name === "ask_user") {
-		const question = a.question || null;
-		const options = a.options || trimmed || null;
+		const question = a.question ?? null;
+		const options = a.options ?? fromBody;
 		return { name, ...a, question, options };
 	}
 
-	return { name, ...a, body: trimmed || a.body };
+	return { name, ...a, body: trimmed === "" ? a.body : trimmed };
 }
 
+const NAME_CHAR = /[a-zA-Z0-9_]/;
+const ATTR_KEY_CHAR = /[a-zA-Z0-9_:-]/;
+const WS = /\s/;
+
+// Tokenizer for rummy's closed set of tool tags. Body opacity for closed
+// bodies; tail recovery for unclosed bodies.
+//
+// Design contract:
+//   - Tool tags (<get>, <set>, <sh>, ...) are the only syntactic special tags.
+//     Any other "<...>" sequence in OUTER text is treated as literal text.
+//   - Inside a tool tag's body, content is OPAQUE: only the matching
+//     `</tagname>` close (depth-counted for same-name nesting) ends the
+//     body. Mismatched closes of OTHER tag names — `</env>`, `</mv>`,
+//     `</foo>` inside a `<set>` body — are body content, not structural
+//     signals.
+//   - Backtick spans (`...`) and triple-backtick fences (```...```)
+//     suppress tag recognition AT THE OUTER LEVEL ONLY (between tool
+//     calls). Documentation prose with backticked tag examples doesn't
+//     get parsed as commands. Inside tool bodies backticks are content;
+//     bodies that need opacity for tag-like content use the edit-syntax
+//     marker family (see SPEC.md "Edit Syntax"), which has no
+//     false-positive failure modes (unlike inside-body backtick
+//     tracking, which would suppress closing tags on bodies with stray
+//     unbalanced backticks).
+//   - Edit-syntax marker opacity (set only): `<<:::IDENT...:::IDENT`
+//     spans inside a `<set>` body are skipped during tag detection so
+//     content with `</set>` literals or marker-shaped text stays as
+//     body. Multiple markers per body supported; see marker.js.
+//   - Same-name nesting (`<set>...<set/>...</set>`) is depth-counted so
+//     nested examples don't prematurely close the outer. Same-name
+//     nesting also disables tail recovery — the model's intent is clearly
+//     opaque body content.
+//   - Unclosed openers (no matching close, no same-name nesting) try
+//     tail recovery: scan the captured body for the leftmost position
+//     whose suffix tokenizes cleanly into ≥1 well-formed tool calls
+//     with zero leftover text. If found, end the unclosed body there
+//     and let the trailing tags parse as proper siblings. The warning
+//     surfaces "Unclosed <name> — recovered N trailing tool call(s)"
+//     so the model can see what happened. If recovery finds nothing,
+//     capture body to EOF and emit "Unclosed <name> — content captured
+//     anyway".
 export default class XmlParser {
-	/**
-	 * Parse tool commands from model content using htmlparser2.
-	 * Handles malformed XML gracefully — unclosed tags, missing slashes, etc.
-	 * Every tool can appear as self-closing (attrs only) or with body content.
-	 * Competing attr-vs-body philosophies are resolved silently.
-	 * @param {string} content - Raw model response text
-	 * @returns {{ commands: Array, warnings: string[], unparsed: string }}
-	 */
 	static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS);
 
 	static parse(content) {
 		if (!content) return { commands: [], warnings: [], unparsed: "" };
 
-		// Normalize native tool call formats to rummy XML
 		const normalized = XmlParser.#normalizeToolCalls(content);
+		return XmlParser.#tokenize(normalized, []);
+	}
 
+	static #tokenize(s, warnings) {
 		const commands = [];
-		const warnings = [];
-		const textChunks = [];
-
-		// Pre-flight: neutralize tool tags inside markdown code spans.
-		// Models quote instructions containing `<get/>` etc. — the parser
-		// would treat them as real tool calls. Replace the angle brackets
-		// inside backtick spans so htmlparser2 ignores them.
-		const codeNeutralized = XmlParser.#neutralizeCodeSpans(normalized);
-
-		// Pre-flight: fix mismatched close tags that htmlparser2 silently
-		// drops (making our onclosetag recovery code unreachable). Must run
-		// before balanceAttrQuotes since the mismatch scan needs clean tags.
-		const mismatchFixed = XmlParser.#correctMismatchedCloses(
-			codeNeutralized,
-			warnings,
-		);
-
-		// Pre-flight: balance unclosed attribute quotes that would otherwise
-		// cause htmlparser2 to consume the rest of input as a single attribute
-		// value, silently dropping every subsequent tool call.
-		const balanced = XmlParser.#balanceAttrQuotes(mismatchFixed, warnings);
-		let current = null;
-		let ended = false;
+		const text = [];
+		let i = 0;
+		let inSingleBacktick = false;
+		let inTripleFence = false;
 		let capped = false;
 
-		const parser = new Parser(
-			{
-				onopentag(name, attrs) {
-					if (capped) return;
-
-					if (current) {
-						// Empty-body case: current tool opened but got no text
-						// content before a new tag. The model likely meant current
-						// to self-close but typed it in paired form, or emitted a
-						// mismatched close tag that htmlparser2 silently dropped.
-						// Close current, open new.
-						const hasBody = current.rawBody.trim() !== "";
-						const hasNestedOpens = (current.nested || []).length > 0;
-						if (!hasBody && !hasNestedOpens && ALL_TOOLS.has(name)) {
-							warnings.push(
-								`Unclosed <${current.name}> before <${name}> — recovered`,
-							);
-							commands.push(
-								resolveCommand(current.name, current.attrs, current.rawBody),
-							);
-							current = null;
-						} else {
-							// Nested tag inside a body with content — treat as body
-							// text. Tool bodies are opaque: the model writing a plan
-							// with <get/> in it, SEARCH/REPLACE in <set>, or XML
-							// examples in <known> all need to survive intact. Track
-							// nested opens on a stack so matching closes pop off and
-							// orphan closes (typos) still trigger recovery.
-							const attrStr = Object.entries(attrs)
-								.map(([k, v]) => (v === "" ? k : `${k}="${v}"`))
-								.join(" ");
-							current.rawBody += attrStr ? `<${name} ${attrStr}>` : `<${name}>`;
-							current.nested ||= [];
-							current.nested.push(name);
-							return;
-						}
-					}
-
-					if (!ALL_TOOLS.has(name)) return;
+		while (i < s.length) {
+			if (commands.length >= XmlParser.MAX_COMMANDS) {
+				capped = true;
+				break;
+			}
 
-					if (commands.length >= XmlParser.MAX_COMMANDS) {
-						capped = true;
-						return;
-					}
+			// Triple-backtick fence toggles take precedence over single backtick
+			// because ``` overlaps `.
+			if (s[i] === "`" && s[i + 1] === "`" && s[i + 2] === "`") {
+				inTripleFence = !inTripleFence;
+				text.push("```");
+				i += 3;
+				continue;
+			}
+			if (s[i] === "`" && !inTripleFence) {
+				inSingleBacktick = !inSingleBacktick;
+				text.push("`");
+				i++;
+				continue;
+			}
 
-					current = { name, attrs, rawBody: "", nested: [] };
-				},
+			if (inSingleBacktick || inTripleFence || s[i] !== "<") {
+				text.push(s[i]);
+				i++;
+				continue;
+			}
 
-				ontext(text) {
-					if (capped) return;
-					if (current) {
-						current.rawBody += text;
-					} else {
-						textChunks.push(text);
-					}
-				},
-
-				onclosetag(name, isImplied) {
-					if (capped) return;
-
-					if (current) {
-						// Matching nested close — pop stack, keep as text.
-						const nested = current.nested;
-						if (nested.length > 0 && nested[nested.length - 1] === name) {
-							nested.pop();
-							current.rawBody += `</${name}>`;
-							return;
-						}
-
-						// Matching close for outer tool — finalize.
-						if (name === current.name && nested.length === 0) {
-							if (ended) {
-								warnings.push(
-									`Unclosed <${name}> tag — content captured anyway`,
-								);
-							}
-							commands.push(
-								resolveCommand(current.name, current.attrs, current.rawBody),
-							);
-							current = null;
-							return;
-						}
-
-						// Orphan close for a known tool (likely typo) — recover.
-						if (ALL_TOOLS.has(name)) {
-							warnings.push(
-								`Mismatched </${name}> closing <${current.name}> — recovered`,
-							);
-							commands.push(
-								resolveCommand(current.name, current.attrs, current.rawBody),
-							);
-							current = null;
-							return;
-						}
-
-						// Unknown orphan close — text.
-						current.rawBody += `</${name}>`;
-						return;
-					}
+			const opener = XmlParser.#matchOpener(s, i);
+			if (!opener) {
+				text.push(s[i]);
+				i++;
+				continue;
+			}
 
-					if (isImplied && ALL_TOOLS.has(name)) {
-						// Self-closing tag that htmlparser2 auto-closed at top level
-					}
-				},
+			const { name, attrs, selfClose, end: openerEnd } = opener;
 
-				onerror(err) {
-					warnings.push(`Parse error: ${err.message}`);
-				},
-			},
-			{
-				recognizeSelfClosing: true,
-				lowerCaseTags: true,
-				lowerCaseAttributeNames: true,
-			},
-		);
+			if (selfClose) {
+				commands.push(resolveCommand(name, attrs, ""));
+				i = openerEnd;
+				continue;
+			}
 
-		parser.write(balanced);
-		ended = true;
-		parser.end();
+			const result = XmlParser.#findBodyEnd(s, name, openerEnd);
+			const body = s.slice(openerEnd, result.bodyEnd);
+			if (result.unclosed) {
+				if (result.recoveredTailCount) {
+					warnings.push(
+						`Unclosed <${name}> tag — recovered ${result.recoveredTailCount} trailing tool call(s)`,
+					);
+				} else {
+					warnings.push(`Unclosed <${name}> tag — content captured anyway`);
+				}
+			}
+			commands.push(resolveCommand(name, attrs, body));
+			i = result.afterClose;
 
-		// Flush any unclosed tool tag
-		if (current && !capped) {
-			warnings.push(`Unclosed <${current.name}> tag — content captured anyway`);
-			commands.push(
-				resolveCommand(current.name, current.attrs, current.rawBody),
-			);
-			current = null;
+			// Body terminated; reset outer-text fence tracking.
+			inSingleBacktick = false;
+			inTripleFence = false;
 		}
 
 		if (capped) {
@@ -309,109 +259,240 @@ export default class XmlParser {
 			);
 		}
 
-		const unparsed = textChunks.join("").trim();
-		return { commands, warnings, unparsed };
+		return {
+			commands,
+			warnings,
+			unparsed: text.join("").trim(),
+		};
 	}
 
-	/**
-	 * Repair a specific malformed-tag pattern: an attribute value opened with
-	 * `="` that never closes before the next tag. Without repair, htmlparser2
-	 * consumes the rest of input as one giant attribute value and silently
-	 * drops every subsequent tool call.
-	 *
-	 * Pattern matched:  <TAG ... ATTR="text-with-no-quote</NEXT>
-	 * Repair:           <TAG ... ATTR="text-with-no-quote"></NEXT>
-	 *
-	 * Conservative — only triggers when the value contains no quote, no `>`,
-	 * and is followed by another tag opening or close. Well-formed input is
-	 * untouched.
-	 */
-	static #balanceAttrQuotes(content, warnings) {
-		let fixes = 0;
-		const repaired = content.replace(
-			/(<\w+\s[^<>]*?\w+=")([^"<>]*?)(<\/?\w+)/g,
-			(_, opening, value, nextTag) => {
-				fixes++;
-				return `${opening}${value}">${nextTag}`;
-			},
-		);
-		if (fixes > 0) {
-			warnings.push(
-				`Repaired ${fixes} malformed attribute(s) — close all attribute values with a quote.`,
-			);
+	// Returns { name, attrs, selfClose, end } if `s[pos..]` opens a known tool,
+	// else null. `end` is the index after the closing `>` (or `/>`).
+	static #matchOpener(s, pos) {
+		if (s[pos] !== "<") return null;
+		let i = pos + 1;
+
+		const nameStart = i;
+		while (i < s.length && NAME_CHAR.test(s[i])) i++;
+		const name = s.slice(nameStart, i).toLowerCase();
+		if (!ALL_TOOLS.has(name)) return null;
+
+		// Char after the name must end the name token cleanly.
+		if (i < s.length && !WS.test(s[i]) && s[i] !== "/" && s[i] !== ">") {
+			return null;
 		}
-		return repaired;
+
+		const attrsStart = i;
+		let inQuote = null;
+
+		while (i < s.length) {
+			const c = s[i];
+			if (inQuote) {
+				if (c === inQuote) inQuote = null;
+				i++;
+				continue;
+			}
+			if (c === '"' || c === "'") {
+				inQuote = c;
+				i++;
+				continue;
+			}
+			if (c === "/") {
+				let k = i + 1;
+				while (k < s.length && WS.test(s[k])) k++;
+				if (s[k] === ">") {
+					return {
+						name,
+						attrs: XmlParser.#parseAttrs(s.slice(attrsStart, i)),
+						selfClose: true,
+						end: k + 1,
+					};
+				}
+				i++;
+				continue;
+			}
+			if (c === ">") {
+				return {
+					name,
+					attrs: XmlParser.#parseAttrs(s.slice(attrsStart, i)),
+					selfClose: false,
+					end: i + 1,
+				};
+			}
+			i++;
+		}
+
+		// Hit EOF without closing — not a parseable opener.
+		return null;
 	}
 
-	/**
-	 * Correct mismatched close tags before htmlparser2 sees them.
-	 *
-	 * htmlparser2 silently drops close tags that don't match the currently
-	 * open element (e.g. `<set>body</known>` — `</known>` vanishes). This
-	 * makes the explicit mismatch recovery in onclosetag unreachable and
-	 * causes all subsequent sibling commands to be absorbed as body text.
-	 *
-	 * Conservative: only corrects when the mismatch is at the outermost
-	 * tool depth (stack.length === 1). Nested mismatches inside body text
-	 * are left for htmlparser2 + body opacity to handle normally.
-	 */
-	/**
-	 * Neutralize XML tags inside markdown code spans so the parser
-	 * doesn't treat quoted tool names as real commands.
-	 * `<get/>` → `&lt;get/&gt;`  (htmlparser2 ignores entities)
-	 */
-	static #neutralizeCodeSpans(content) {
-		return content.replace(/`([^`]*)`/g, (match, inner) => {
-			if (!/<\/?[\w]/.test(inner)) return match;
-			return `\`${inner.replace(/</g, "&lt;").replace(/>/g, "&gt;")}\``;
-		});
+	static #parseAttrs(raw) {
+		const attrs = {};
+		let i = 0;
+		while (i < raw.length) {
+			while (i < raw.length && WS.test(raw[i])) i++;
+			if (i >= raw.length) break;
+
+			const keyStart = i;
+			while (i < raw.length && ATTR_KEY_CHAR.test(raw[i])) i++;
+			if (i === keyStart) {
+				i++;
+				continue;
+			}
+			const key = raw.slice(keyStart, i).toLowerCase();
+
+			while (i < raw.length && WS.test(raw[i])) i++;
+
+			if (raw[i] !== "=") {
+				attrs[key] = "";
+				continue;
+			}
+			i++;
+
+			while (i < raw.length && WS.test(raw[i])) i++;
+
+			if (raw[i] === '"' || raw[i] === "'") {
+				const quote = raw[i];
+				i++;
+				const valStart = i;
+				while (i < raw.length && raw[i] !== quote) i++;
+				attrs[key] = raw.slice(valStart, i);
+				if (raw[i] === quote) i++;
+			} else {
+				const valStart = i;
+				while (i < raw.length && !WS.test(raw[i])) i++;
+				attrs[key] = raw.slice(valStart, i);
+			}
+		}
+		return attrs;
 	}
 
-	static #correctMismatchedCloses(content, warnings) {
-		const stack = [];
-		return content.replace(
-			/<(\/?)(\w+)([^>]*?)(\/?)>/g,
-			(match, slash, tag, _attrs, selfClose) => {
-				if (!ALL_TOOLS.has(tag)) return match;
-				if (selfClose === "/") return match;
-				if (slash === "/") {
-					if (stack.length === 0) return match;
-					if (stack[stack.length - 1] === tag) {
-						stack.pop();
-						return match;
-					}
-					if (stack.length === 1) {
-						const top = stack.pop();
-						warnings.push(
-							`Mismatched </${tag}> closing <${top}> — corrected to </${top}>`,
-						);
-						return `</${top}>`;
+	// Scans body content from `fromPos` until the matching `</name>` closer,
+	// counting depth so same-name nested examples don't prematurely close.
+	// Returns { bodyEnd, afterClose, unclosed }.
+	//
+	// Strict body opacity: only `</name>` (matching the open) and same-name
+	// nested opens affect parsing. Mismatched closes of OTHER tag names are
+	// body content, period.
+	//
+	// Backtick fences (`…`, ```…```) inside the body suppress all tag
+	// recognition — a markdown table cell containing `<set>` examples
+	// stays as content, not interpreted as a nested tag. This matches
+	// the outer-level convention and is the load-bearing reason a model
+	// can write documentation about rummy commands inside a deliverable
+	// body without breaking parsing.
+	//
+	// If the matching close never arrives, emit "Unclosed" so the model
+	// sees a clear failure and corrects on the next turn.
+	static #findBodyEnd(s, name, fromPos) {
+		let depth = 1;
+		let sameNameNested = false;
+		let i = fromPos;
+		while (i < s.length) {
+			// Edit-syntax marker opacity: marker spans (bare `<<IDENT` or
+			// packet-shaped `<<:::IDENT`) are opaque — tag detection
+			// skips them so inner `</set>` and other tag-shaped content
+			// stays as body. Multiple markers per `<set>` body are
+			// supported; check on every iteration.
+			if (
+				name === "set" &&
+				(s.startsWith("<<:::", i) ||
+					(s.startsWith("<<", i) && /^[A-Z]/.test(s[i + 2] ?? "")))
+			) {
+				const skipTo = skipEditMarker(s, i);
+				if (skipTo != null) {
+					i = skipTo;
+					continue;
+				}
+			}
+			if (s[i] !== "<") {
+				i++;
+				continue;
+			}
+			if (s[i + 1] === "/") {
+				const nameStart = i + 2;
+				let nameEnd = nameStart;
+				while (nameEnd < s.length && NAME_CHAR.test(s[nameEnd])) nameEnd++;
+				const closeName = s.slice(nameStart, nameEnd).toLowerCase();
+				let k = nameEnd;
+				while (k < s.length && WS.test(s[k])) k++;
+				const isCloseTag = s[k] === ">";
+
+				if (isCloseTag && closeName === name) {
+					depth--;
+					if (depth === 0) {
+						return { bodyEnd: i, afterClose: k + 1, unclosed: false };
 					}
-					return match;
+					i = k + 1;
+					continue;
 				}
-				stack.push(tag);
-				return match;
-			},
-		);
+			}
+			const opener = XmlParser.#matchOpener(s, i);
+			if (opener && opener.name === name && !opener.selfClose) {
+				depth++;
+				sameNameNested = true;
+				i = opener.end;
+				continue;
+			}
+			i++;
+		}
+		// Unclosed: try tail recovery, but only if the body never
+		// nested a same-name opener. Same-name nesting is the model
+		// deliberately using opaque body for examples (`<set>` writing
+		// docs about `<set>`); we trust the body content as authored.
+		// No nesting means a plain botched `</set>` — recovery is safe.
+		// If the body's tail is a clean sequence of one or more
+		// well-formed tool calls (zero leftover text), end the body
+		// at the start of that tail and let the outer tokenizer parse
+		// those calls as proper siblings. Closes the silent-swallow
+		// gap when a model botches `</set>` after SEARCH/REPLACE and
+		// emits trailing `<sh>` / `<update>`.
+		if (sameNameNested) {
+			return { bodyEnd: s.length, afterClose: s.length, unclosed: true };
+		}
+		const recovery = XmlParser.#findTailRecovery(s, fromPos);
+		if (recovery) {
+			return {
+				bodyEnd: recovery.tailStart,
+				afterClose: recovery.tailStart,
+				unclosed: true,
+				recoveredTailCount: recovery.commandCount,
+			};
+		}
+		return { bodyEnd: s.length, afterClose: s.length, unclosed: true };
+	}
+
+	// Scan body content for the leftmost position whose suffix tokenizes
+	// cleanly into ≥1 commands with no leftover non-whitespace text.
+	// Returns { tailStart, commandCount } or null. Only considers opener
+	// positions; treats the suffix as outer-level so backtick fences and
+	// tag recognition match the parent tokenizer's behavior.
+	static #findTailRecovery(s, fromPos) {
+		let best = null;
+		let i = fromPos;
+		while (i < s.length) {
+			if (s[i] === "<" && XmlParser.#matchOpener(s, i)) {
+				const suffix = s.slice(i);
+				const result = XmlParser.#tokenize(suffix, []);
+				if (result.commands.length > 0 && result.unparsed === "") {
+					best = { tailStart: i, commandCount: result.commands.length };
+					break;
+				}
+			}
+			i++;
+		}
+		return best;
 	}
 
-	/**
-	 * Normalize native tool call formats to rummy XML.
-	 * Models sometimes emit their training-format tool calls instead of
-	 * our XML tags. The intent is unambiguous — translate silently.
-	 */
+	// Translate native training-format tool calls into rummy XML silently.
 	static #normalizeToolCalls(content) {
-		// Gemma: ```tool_code\n<xml>...\n``` — strip code fences around valid XML
+		// Gemma code-fenced XML.
 		let result = content.replace(
 			/```(?:tool_code|tool_command|xml)\n([\s\S]*?)```/g,
 			(_, inner) => inner.trim(),
 		);
 
-		// Qwen/gemma: <|tool_call>call:NAME{key:"value"}<tool_call|>
-		// NAME may be namespaced with any of /, :, or . separators
-		// (e.g. `rummy.nvim/get`, `rummy:get`) — extract the trailing word
-		// sequence as the tool name. Value forms observed in the wild:
-		//   key="v" / key:"v" / key:v (unquoted) / key:<|"|>v<|"|> (gemma chat-quotes)
+		// Qwen/gemma <|tool_call>call:NAME{...}<tool_call|>; NAME may be namespaced.
 		result = result.replace(
 			/<\|tool_call>call:([\w.:/-]+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
 			(match, qualifiedName, params) => {
@@ -469,28 +550,16 @@ export default class XmlParser {
 			},
 		);
 
-		// Catch-all: any remaining <|tool_call> tokens are malformed native
-		// attempts (no {} block, missing close, wrong shape entirely). Replace
-		// each with an <error> so the model gets feedback on its next turn and
-		// learns to switch to XML. Lazy-match up to the next native close, the
-		// next XML close tag, or end of input — preserves any trailing valid XML.
-		// Error body must NOT contain literal <get>/<set>/etc. — those would
-		// re-enter the parser as phantom tool calls. Describe the format in
-		// prose instead and point at the tool docs above.
+		// Catch-all malformed <|tool_call> → <error> in prose (no literal tags or they'd re-parse).
 		result = result.replace(
 			/<\|tool_call>[\s\S]*?(?:<\|?tool_call\|?>|<\/\w+>|$)/g,
 			() =>
 				"<error>Native tool call format not supported. Use the XML commands listed above (e.g. a get tag with a path attribute, or a set tag with path and body).</error>",
 		);
 
-		// Strip any orphan chat-format quote tokens left after replacement.
 		result = result.replace(/<\|"\|>/g, '"');
 
-		// Gemma sometimes leaks OpenAI-harmony channel markers around its
-		// real XML output: `<|channel>thought\n<channel|>…<set path=…/>`.
-		// These aren't tool calls (handled above), they're role/channel
-		// tokens. Strip any remaining `<|name>` / `<name|>` pseudo-tags
-		// before the XML parser sees them.
+		// Strip OpenAI-harmony role/channel pseudo-tags (gemma leaks these).
 		result = result.replace(/<\|[\w:/-]+>/g, "");
 		result = result.replace(/<[\w:/-]+\|>/g, "");