@possumtech/rummy 0.3.1 → 0.5.0

package/src/agent/XmlParser.js

@@ -14,8 +14,6 @@ export const ALL_TOOLS = new Set([
 	"update",
 	"unknown",
 	"think",
-	"thought",
-	"mcp",
 ]);
 
 /**
@@ -145,6 +143,8 @@ export default class XmlParser {
 	 * @param {string} content - Raw model response text
 	 * @returns {{ commands: Array, warnings: string[], unparsed: string }}
 	 */
+	static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS) || 99;
+
 	static parse(content) {
 		if (!content) return { commands: [], warnings: [], unparsed: "" };
 
@@ -154,33 +154,67 @@ export default class XmlParser {
 		const commands = [];
 		const warnings = [];
 		const textChunks = [];
+
+		// Pre-flight: balance unclosed attribute quotes that would otherwise
+		// cause htmlparser2 to consume the rest of input as a single attribute
+		// value, silently dropping every subsequent tool call.
+		const balanced = XmlParser.#balanceAttrQuotes(normalized, warnings);
 		let current = null;
 		let ended = false;
+		let capped = false;
 
 		const parser = new Parser(
 			{
 				onopentag(name, attrs) {
-					if (!ALL_TOOLS.has(name)) {
-						if (current) {
-							current.rawBody += `<${name}>`;
+					if (capped) return;
+
+					if (current) {
+						// Empty-body case: current tool opened but got no text
+						// content before a new tag. The model likely meant current
+						// to self-close but typed it in paired form, or emitted a
+						// mismatched close tag that htmlparser2 silently dropped.
+						// Close current, open new.
+						const hasBody = current.rawBody.trim() !== "";
+						const hasNestedOpens = (current.nested || []).length > 0;
+						if (!hasBody && !hasNestedOpens && ALL_TOOLS.has(name)) {
+							warnings.push(
+								`Unclosed <${current.name}> before <${name}> — recovered`,
+							);
+							commands.push(
+								resolveCommand(current.name, current.attrs, current.rawBody),
+							);
+							current = null;
+						} else {
+							// Nested tag inside a body with content — treat as body
+							// text. Tool bodies are opaque: the model writing a plan
+							// with <get/> in it, SEARCH/REPLACE in <set>, or XML
+							// examples in <known> all need to survive intact. Track
+							// nested opens on a stack so matching closes pop off and
+							// orphan closes (typos) still trigger recovery.
+							const attrStr = Object.entries(attrs)
+								.map(([k, v]) => (v === "" ? k : `${k}="${v}"`))
+								.join(" ");
+							current.rawBody += attrStr
+								? `<${name} ${attrStr}>`
+								: `<${name}>`;
+							current.nested ||= [];
+							current.nested.push(name);
+							return;
 						}
-						return;
 					}
 
-					// Known tool opened while another is still open — close the old one.
-					if (current) {
-						warnings.push(
-							`Unclosed <${current.name}> before <${name}> — recovered`,
-						);
-						commands.push(
-							resolveCommand(current.name, current.attrs, current.rawBody),
-						);
+					if (!ALL_TOOLS.has(name)) return;
+
+					if (commands.length >= XmlParser.MAX_COMMANDS) {
+						capped = true;
+						return;
 					}
 
-					current = { name, attrs, rawBody: "" };
+					current = { name, attrs, rawBody: "", nested: [] };
 				},
 
 				ontext(text) {
+					if (capped) return;
 					if (current) {
 						current.rawBody += text;
 					} else {
@@ -189,28 +223,53 @@ export default class XmlParser {
 				},
 
 				onclosetag(name, isImplied) {
-					if (current && name === current.name) {
-						if (ended) {
-							warnings.push(`Unclosed <${name}> tag — content captured anyway`);
+					if (capped) return;
+
+					if (current) {
+						// Matching nested close — pop stack, keep as text.
+						const nested = current.nested;
+						if (
+							nested.length > 0 &&
+							nested[nested.length - 1] === name
+						) {
+							nested.pop();
+							current.rawBody += `</${name}>`;
+							return;
+						}
+
+						// Matching close for outer tool — finalize.
+						if (name === current.name && nested.length === 0) {
+							if (ended) {
+								warnings.push(
+									`Unclosed <${name}> tag — content captured anyway`,
+								);
+							}
+							commands.push(
+								resolveCommand(current.name, current.attrs, current.rawBody),
+							);
+							current = null;
+							return;
+						}
+
+						// Orphan close for a known tool (likely typo) — recover.
+						if (ALL_TOOLS.has(name)) {
+							warnings.push(
+								`Mismatched </${name}> closing <${current.name}> — recovered`,
+							);
+							commands.push(
+								resolveCommand(current.name, current.attrs, current.rawBody),
+							);
+							current = null;
+							return;
 						}
-						commands.push(
-							resolveCommand(current.name, current.attrs, current.rawBody),
-						);
-						current = null;
-					} else if (current && ALL_TOOLS.has(name)) {
-						// Mismatched close tag for a known tool — close current tag,
-						// don't swallow subsequent commands as body text.
-						warnings.push(
-							`Mismatched </${name}> closing <${current.name}> — recovered`,
-						);
-						commands.push(
-							resolveCommand(current.name, current.attrs, current.rawBody),
-						);
-						current = null;
-					} else if (current) {
+
+						// Unknown orphan close — text.
 						current.rawBody += `</${name}>`;
-					} else if (isImplied && ALL_TOOLS.has(name)) {
-						// Self-closing tag that htmlparser2 auto-closed
+						return;
+					}
+
+					if (isImplied && ALL_TOOLS.has(name)) {
+						// Self-closing tag that htmlparser2 auto-closed at top level
 					}
 				},
 
@@ -225,12 +284,12 @@ export default class XmlParser {
 			},
 		);
 
-		parser.write(normalized);
+		parser.write(balanced);
 		ended = true;
 		parser.end();
 
 		// Flush any unclosed tool tag
-		if (current) {
+		if (current && !capped) {
 			warnings.push(`Unclosed <${current.name}> tag — content captured anyway`);
 			commands.push(
 				resolveCommand(current.name, current.attrs, current.rawBody),
@@ -238,10 +297,46 @@ export default class XmlParser {
 			current = null;
 		}
 
+		if (capped) {
+			warnings.push(
+				`Tool call limit (${XmlParser.MAX_COMMANDS}) reached — remaining commands dropped`,
+			);
+		}
+
 		const unparsed = textChunks.join("").trim();
 		return { commands, warnings, unparsed };
 	}
 
+	/**
+	 * Repair a specific malformed-tag pattern: an attribute value opened with
+	 * `="` that never closes before the next tag. Without repair, htmlparser2
+	 * consumes the rest of input as one giant attribute value and silently
+	 * drops every subsequent tool call.
+	 *
+	 * Pattern matched:  <TAG ... ATTR="text-with-no-quote</NEXT>
+	 * Repair:           <TAG ... ATTR="text-with-no-quote"></NEXT>
+	 *
+	 * Conservative — only triggers when the value contains no quote, no `>`,
+	 * and is followed by another tag opening or close. Well-formed input is
+	 * untouched.
+	 */
+	static #balanceAttrQuotes(content, warnings) {
+		let fixes = 0;
+		const repaired = content.replace(
+			/(<\w+\s[^<>]*?\w+=")([^"<>]*?)(<\/?\w+)/g,
+			(_, opening, value, nextTag) => {
+				fixes++;
+				return `${opening}${value}">${nextTag}`;
+			},
+		);
+		if (fixes > 0) {
+			warnings.push(
+				`Repaired ${fixes} malformed attribute(s) — close all attribute values with a quote.`,
+			);
+		}
+		return repaired;
+	}
+
 	/**
 	 * Normalize native tool call formats to rummy XML.
 	 * Models sometimes emit their training-format tool calls instead of
@@ -255,12 +350,30 @@ export default class XmlParser {
 		);
 
 		// Qwen/gemma: <|tool_call>call:NAME{key:"value"}<tool_call|>
+		// NAME may be namespaced with any of /, :, or . separators
+		// (e.g. `rummy.nvim/get`, `rummy:get`) — extract the trailing word
+		// sequence as the tool name. Value forms observed in the wild:
+		//   key="v" / key:"v" / key:v (unquoted) / key:<|"|>v<|"|> (gemma chat-quotes)
 		result = result.replace(
-			/<\|tool_call>call:(\w+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
-			(_, name, params) => {
-				if (!ALL_TOOLS.has(name)) return _;
-				const valueMatch = params.match(/["']([^"']+)["']/);
-				const body = valueMatch?.[1] || "";
+			/<\|tool_call>call:([\w.:/-]+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
+			(match, qualifiedName, params) => {
+				const name = qualifiedName.match(/\w+$/)?.[0] ?? qualifiedName;
+				if (!ALL_TOOLS.has(name)) {
+					return `<error>Unknown tool '${qualifiedName}' in <|tool_call> format. Use XML tool commands listed above.</error>`;
+				}
+				const valueMatch = params.match(
+					/[=:]\s*(?:<\|"\|>([^<]*?)<\|"\|>|"([^"]*)"|'([^']*)'|([^,}]+))/,
+				);
+				const body = (
+					valueMatch?.[1] ??
+					valueMatch?.[2] ??
+					valueMatch?.[3] ??
+					valueMatch?.[4] ??
+					""
+				).trim();
+				if (!body) {
+					return `<error>Could not extract argument from <|tool_call> ${match}. Use XML format like <${name}>value</${name}>.</error>`;
+				}
 				return `<${name}>${body}</${name}>`;
 			},
 		);
@@ -298,6 +411,23 @@ export default class XmlParser {
 			},
 		);
 
+		// Catch-all: any remaining <|tool_call> tokens are malformed native
+		// attempts (no {} block, missing close, wrong shape entirely). Replace
+		// each with an <error> so the model gets feedback on its next turn and
+		// learns to switch to XML. Lazy-match up to the next native close, the
+		// next XML close tag, or end of input — preserves any trailing valid XML.
+		// Error body must NOT contain literal <get>/<set>/etc. — those would
+		// re-enter the parser as phantom tool calls. Describe the format in
+		// prose instead and point at the tool docs above.
+		result = result.replace(
+			/<\|tool_call>[\s\S]*?(?:<\|?tool_call\|?>|<\/\w+>|$)/g,
+			() =>
+				"<error>Native tool call format not supported. Use the XML tool commands listed above (e.g. a get tag with a path attribute, or a set tag with path and body).</error>",
+		);
+
+		// Strip any orphan chat-format quote tokens left after replacement.
+		result = result.replace(/<\|"\|>/g, '"');
+
 		return result;
 	}
 }

package/src/agent/known_queries.sql

@@ -1,5 +1,5 @@
 -- PREP: get_known_entries
-SELECT path, scheme, status, fidelity, body, turn, hash, attributes
+SELECT path, scheme, status, fidelity, body, turn, hash, attributes, tokens
 FROM known_entries
 WHERE run_id = :run_id
 ORDER BY path;

package/src/agent/known_store.sql

@@ -1,13 +1,12 @@
 -- PREP: upsert_known_entry
 INSERT INTO known_entries (
 	run_id, loop_id, turn, path, body, status, fidelity, hash
-	, attributes, tokens, tokens_full, updated_at
+	, attributes, tokens, updated_at
 )
 VALUES (
 	:run_id, :loop_id, :turn, :path, :body, :status, :fidelity, :hash
 	, COALESCE(:attributes, '{}')
 	, countTokens(:body)
-	, countTokens(:body)
 	, COALESCE(:updated_at, CURRENT_TIMESTAMP)
 )
 ON CONFLICT (run_id, path) DO UPDATE SET
@@ -19,13 +18,12 @@ ON CONFLICT (run_id, path) DO UPDATE SET
 	, loop_id = excluded.loop_id
 	, turn = excluded.turn
 	, tokens = countTokens(excluded.body)
-	, tokens_full = countTokens(excluded.body)
 	, write_count = known_entries.write_count + 1
 	, updated_at = COALESCE(excluded.updated_at, CURRENT_TIMESTAMP);
 
 -- PREP: recount_tokens
 UPDATE known_entries
-SET tokens = :tokens, tokens_full = :tokens
+SET tokens = :tokens
 WHERE run_id = :run_id AND path = :path;
 
 -- PREP: get_stale_tokens
@@ -55,54 +53,30 @@ WHERE run_id = :run_id AND path = :path;
 UPDATE known_entries
 SET
 	fidelity = :fidelity
-	, tokens = CASE
-		WHEN :fidelity = 'archive'
-			THEN 0
-		WHEN :fidelity = 'index'
-			THEN 0
-		WHEN :fidelity = 'summary'
-			THEN COALESCE(
-				countTokens(json_extract(attributes, '$.summary')),
-				countTokens(substr(body, 1, 80))
-			)
-		ELSE tokens_full
-	END
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
 
 -- PREP: promote_path
 UPDATE known_entries
 SET
-	fidelity = 'full'
+	fidelity = 'promoted'
+	, status = 200
 	, turn = :turn
-	, tokens = tokens_full
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
 -- PREP: demote_path
 UPDATE known_entries
 SET
-	fidelity = 'archive'
-	, tokens = 0
+	fidelity = 'archived'
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
 -- PREP: set_fidelity
+-- Tokens unchanged — always reflects full body cost.
 UPDATE known_entries
 SET
 	fidelity = :fidelity
-	, tokens = CASE
-		WHEN :fidelity = 'archive'
-			THEN 0
-		WHEN :fidelity = 'index'
-			THEN 0
-		WHEN :fidelity = 'summary'
-			THEN COALESCE(
-				countTokens(json_extract(attributes, '$.summary')),
-				countTokens(substr(body, 1, 80))
-			)
-		ELSE countTokens(body)
-	END
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
@@ -137,9 +111,9 @@ WHERE run_id = :run_id AND path = :path;
 -- PREP: promote_by_pattern
 UPDATE known_entries
 SET
-	fidelity = 'full'
+	fidelity = 'promoted'
+	, status = 200
 	, turn = :turn
-	, tokens = tokens_full
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
 	run_id = :run_id
@@ -149,8 +123,7 @@ WHERE
 -- PREP: demote_by_pattern
 UPDATE known_entries
 SET
-	fidelity = 'archive'
-	, tokens = 0
+	fidelity = 'archived'
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
 	run_id = :run_id
@@ -158,7 +131,7 @@ WHERE
 	AND (:body IS NULL OR hedsearch(:body, body));
 
 -- PREP: get_entries_by_pattern
-SELECT path, body, scheme, status, fidelity, tokens_full, attributes
+SELECT path, body, scheme, status, fidelity, tokens, attributes
 FROM known_entries
 WHERE
 	run_id = :run_id
@@ -182,7 +155,6 @@ UPDATE known_entries
 SET
 	body = :new_body
 	, tokens = countTokens(:new_body)
-	, tokens_full = countTokens(:new_body)
 	, write_count = write_count + 1
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
@@ -191,49 +163,34 @@ WHERE
 	AND (:body IS NULL OR hedsearch(:body, body));
 
 -- PREP: restore_summarized_prompts
--- Restore prompt entries demoted to summary by a recovery phase that was
+-- Restore prompt entries demoted by a recovery phase that was
 -- interrupted (e.g. server crash). Safe to call unconditionally at loop
 -- start: if the full prompt would overflow, Prompt Demotion handles it.
 UPDATE known_entries
 SET
-	fidelity = 'full'
-	, tokens = tokens_full
-	, updated_at = CURRENT_TIMESTAMP
-WHERE run_id = :run_id AND scheme = 'prompt' AND fidelity = 'summary';
-
--- PREP: demote_previous_loop_logging
--- Demote full logging entries from all other loops to summary.
--- Fires at loop start so <previous> entries are already compact.
-UPDATE known_entries
-SET
-	fidelity = 'summary'
-	, tokens = COALESCE(
-		countTokens(json_extract(attributes, '$.summary'))
-		, countTokens(substr(body, 1, 80))
-	)
+	fidelity = 'promoted'
 	, updated_at = CURRENT_TIMESTAMP
-WHERE
-	run_id = :run_id
-	AND (loop_id IS NULL OR loop_id != :loop_id)
-	AND fidelity = 'full'
-	AND scheme IN (SELECT name FROM schemes WHERE category = 'logging');
-
--- PREP: demote_turn_data_entries
--- Demote full data entries from a turn to summary with 413 status.
--- Fires when end-of-turn materialization exceeds the context ceiling.
+WHERE run_id = :run_id AND scheme = 'prompt' AND fidelity = 'demoted';
+
+-- PREP: demote_turn_entries
+-- Demote all promoted entries from a turn for budget claw-back.
+-- Action schemes (set/rm/mv/cp) at status 200 keep their status — those
+-- represent committed side effects (files written/removed) that can't be
+-- clawed back; only the body in context is demoted, not the truth of what
+-- happened. Everything else flips to 413 since promotion was reversed.
+-- Tokens unchanged — always reports full cost regardless of fidelity.
 UPDATE known_entries
 SET
-	fidelity = 'summary'
-	, status = 413
-	, tokens = COALESCE(
-		countTokens(json_extract(attributes, '$.summary'))
-		, countTokens(substr(body, 1, 80))
-	)
+	fidelity = 'demoted'
+	, status = CASE
+		WHEN scheme IN ('set', 'rm', 'mv', 'cp') AND status = 200 THEN 200
+		ELSE 413
+	END
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
 	run_id = :run_id
 	AND turn = :turn
-	AND fidelity = 'full'
+	AND fidelity = 'promoted'
 	AND status < 400
-	AND scheme IN (SELECT name FROM schemes WHERE category = 'data')
-RETURNING path;
+RETURNING path, tokens;
+

package/src/agent/runs.sql

@@ -81,11 +81,11 @@ RETURNING next_turn - 1 AS turn;
 -- PREP: fork_known_entries
 INSERT INTO known_entries (
 	run_id, loop_id, turn, path, body, status, fidelity
-	, hash, attributes, tokens, tokens_full, refs, write_count
+	, hash, attributes, tokens, refs, write_count
 )
 SELECT
 	:new_run_id, NULL, turn, path, body, status, fidelity
-	, hash, attributes, tokens, tokens_full, refs, write_count
+	, hash, attributes, tokens, refs, write_count
 FROM known_entries
 WHERE run_id = :parent_run_id;
 

package/src/hooks/Hooks.js

@@ -56,6 +56,7 @@ export default function createHooks(debug = false) {
 		turn: {
 			started: createEvent("turn.started"),
 			response: createEvent("turn.response"),
+			proposal: createEvent("turn.proposal"),
 			proposing: createEvent("turn.proposing"),
 			completed: createEvent("turn.completed"),
 		},

package/src/hooks/PluginContext.js

@@ -69,10 +69,16 @@ export default class PluginContext {
 		this.#hooks.tools.ensureTool(this.#name);
 	}
 
+	// Mark this plugin's tool as hidden from model-facing tool lists.
+	// Handler still dispatches if the model emits the tag.
+	markHidden() {
+		this.#hooks.tools.markHidden(this.#name);
+	}
+
 	/**
 	 * Register a named callback for this plugin.
 	 * "handler" registers the tool handler.
-	 * "full"/"summary" register fidelity projections.
+	 * "promoted"/"demoted" register fidelity projections.
 	 * "docs" sets tool documentation.
 	 * Everything else resolves to a hook event.
 	 */
@@ -82,7 +88,7 @@ export default class PluginContext {
 			this.#hooks.tools.onHandle(this.#name, callback, priority);
 			return;
 		}
-		if (event === "full" || event === "summary") {
+		if (event === "promoted" || event === "demoted") {
 			this.#hooks.tools.onView(this.#name, callback, event);
 			return;
 		}

package/src/hooks/RummyContext.js

@@ -103,9 +103,12 @@ export default class RummyContext {
 
 	async set({ path, body, status = 200, fidelity, attributes } = {}) {
 		if (!path) {
-			const slugify = (await import("../sql/functions/slugify.js")).default;
-			const base = slugify(body || "");
-			path = `known://${base || Date.now()}`;
+			path = await this.entries.slugPath(
+				this.runId,
+				"known",
+				body || "",
+				attributes?.summary,
+			);
 		}
 		await this.entries.upsert(
 			this.runId,

package/src/hooks/ToolRegistry.js

@@ -1,19 +1,20 @@
 // Tool display order: gather → reason → act → communicate.
 // Position in the list implies priority to the model.
 const TOOL_ORDER = [
+	"think",
+	"unknown",
+	"known",
 	"get",
 	"set",
-	"known",
-	"unknown",
 	"env",
 	"sh",
 	"rm",
 	"cp",
 	"mv",
-	"search",
-	"summarize",
-	"update",
 	"ask_user",
+	"update",
+	"summarize",
+	"search",
 ];
 
 function sortByPriority(names) {
@@ -31,12 +32,21 @@ export default class ToolRegistry {
 	#tools = new Map();
 	#handlers = new Map();
 	#views = new Map();
+	#hidden = new Set();
 
 	ensureTool(scheme) {
 		if (this.#tools.has(scheme)) return;
 		this.#tools.set(scheme, Object.freeze({}));
 	}
 
+	// Mark a tool as hidden — handler still dispatches if the model emits the
+	// tag, but the tool is excluded from all model-facing tool lists. Used for
+	// legacy/internal schemes (e.g. <known>, <unknown>) we want to retire
+	// without deleting.
+	markHidden(scheme) {
+		this.#hidden.add(scheme);
+	}
+
 	get(name) {
 		return this.#tools.get(name);
 	}
@@ -52,7 +62,7 @@ export default class ToolRegistry {
 		list.sort((a, b) => a.priority - b.priority);
 	}
 
-	onView(scheme, fn, fidelity = "full") {
+	onView(scheme, fn, fidelity = "promoted") {
 		if (!this.#views.has(scheme)) this.#views.set(scheme, new Map());
 		this.#views.get(scheme).set(fidelity, fn);
 	}
@@ -66,32 +76,11 @@ export default class ToolRegistry {
 			);
 		}
 
-		const attrs =
-			typeof entry.attributes === "string"
-				? JSON.parse(entry.attributes)
-				: entry.attributes;
-		const summary = typeof attrs?.summary === "string" ? attrs.summary : null;
-
-		const fidelity = entry.fidelity || "full";
+		const fidelity = entry.fidelity || "promoted";
 		const fn = fidelityMap.get(fidelity);
-		if (!fn) {
-			// No view for this fidelity — fall back on model-authored summary
-			return summary || "";
-		}
-
-		const body = await fn(entry);
-
-		// Prepend summary keywords above plugin output at summary fidelity
-		if (fidelity === "summary" && summary && body) {
-			return `${summary}\n${body}`;
-		}
+		if (!fn) return "";
 
-		// Fall back to summary attribute when plugin returns empty
-		if (fidelity === "summary" && summary && !body) {
-			return summary;
-		}
-
-		return body;
+		return fn(entry);
 	}
 
 	hasView(scheme) {
@@ -112,15 +101,23 @@ export default class ToolRegistry {
 		return sortByPriority([...this.#tools.keys()]);
 	}
 
+	// Names advertised to the model — registered tools minus hidden ones.
+	// Use this anywhere a tool list is shown to the model.
+	get advertisedNames() {
+		return sortByPriority(
+			[...this.#tools.keys()].filter((n) => !this.#hidden.has(n)),
+		);
+	}
+
 	/**
 	 * Compute the active tool set for a loop.
-	 * All exclusions — mode, flags — handled here. One mechanism.
+	 * All exclusions — mode, flags, hidden — handled here. One mechanism.
 	 */
 	resolveForLoop(
 		mode,
 		{ noInteraction = false, noWeb = false, noProposals = false } = {},
 	) {
-		const excluded = new Set();
+		const excluded = new Set(this.#hidden);
 		if (mode === "ask") excluded.add("sh");
 		if (noInteraction) excluded.add("ask_user");
 		if (noWeb) excluded.add("search");