@possumtech/rummy 0.4.0 → 2.0.0

package/src/agent/XmlParser.js

@@ -1,18 +1,15 @@
 import { Parser } from "htmlparser2";
 import { parseEditContent } from "../plugins/hedberg/edits.js";
-import { normalizeAttrs, parseJsonEdit } from "../plugins/hedberg/normalize.js";
+import { parseJsonEdit } from "../plugins/hedberg/normalize.js";
 import { parseSed } from "../plugins/hedberg/sed.js";
 
 const STORE_TOOLS = new Set(["get", "rm", "set", "mv", "cp", "search"]);
 export const ALL_TOOLS = new Set([
 	...STORE_TOOLS,
-	"known",
 	"sh",
 	"env",
 	"ask_user",
-	"summarize",
 	"update",
-	"unknown",
 	"think",
 ]);
 
@@ -20,8 +17,7 @@ export const ALL_TOOLS = new Set([
  * Resolve the competing attr-vs-body philosophies per tool.
  * If the canonical attribute is missing, the body fills it. Silent.
  */
-function resolveCommand(name, attrs, rawBody) {
-	const a = normalizeAttrs(attrs);
+function resolveCommand(name, a, rawBody) {
 	const trimmed = rawBody.trim();
 
 	if (name === "set") {
@@ -88,47 +84,48 @@ function resolveCommand(name, attrs, rawBody) {
 				preview: a.preview,
 			};
 		}
-		// Plain write or fidelity change
+		// Plain write or visibility change
 		const body = trimmed || a.body || "";
 		return { name, ...a, body };
 	}
 
-	if (name === "summarize" || name === "update" || name === "unknown") {
+	if (name === "update") {
 		const body = trimmed || a.body || "";
-		return { name, body };
-	}
-
-	if (name === "known") {
-		const body = trimmed || a.body || "";
-		const path = a.path || null;
-		return { name, ...a, path, body };
+		const status = a.status ? Number(a.status) : 102;
+		return { name, ...a, body, status };
 	}
 
 	if (name === "get" || name === "rm") {
-		const path = a.path || trimmed || null;
-		return { name, path, body: a.body, preview: a.preview };
+		// Spread `a` so `line`, `limit`, `visibility`, and future attrs
+		// reach the handler. Earlier narrow extraction silently dropped
+		// `line=/limit=` and stranded the partial-read path advertised
+		// in getDoc.
+		return { name, ...a, path: a.path || trimmed || null };
 	}
 
 	if (name === "search") {
 		const path = a.path || trimmed || null;
 		const results = a.results ? Number(a.results) : null;
-		return { name, path, results };
+		return { name, ...a, path, results };
 	}
 
 	if (name === "mv" || name === "cp") {
-		const to = a.to || trimmed || null;
-		return { name, path: a.path, to };
+		// Spread `a` so `visibility` reaches the handler. mvDoc
+		// advertises `<mv path="known://..." visibility="summarized"/>`
+		// for batch visibility changes and was silently stripping that
+		// attr before.
+		return { name, ...a, path: a.path, to: a.to || trimmed || null };
 	}
 
 	if (name === "sh" || name === "env") {
 		const command = a.command || trimmed || null;
-		return { name, command };
+		return { name, ...a, command };
 	}
 
 	if (name === "ask_user") {
 		const question = a.question || null;
 		const options = a.options || trimmed || null;
-		return { name, question, options };
+		return { name, ...a, question, options };
 	}
 
 	return { name, ...a, body: trimmed || a.body };
@@ -143,7 +140,7 @@ export default class XmlParser {
 	 * @param {string} content - Raw model response text
 	 * @returns {{ commands: Array, warnings: string[], unparsed: string }}
 	 */
-	static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS) || 99;
+	static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS);
 
 	static parse(content) {
 		if (!content) return { commands: [], warnings: [], unparsed: "" };
@@ -154,6 +151,25 @@ export default class XmlParser {
 		const commands = [];
 		const warnings = [];
 		const textChunks = [];
+
+		// Pre-flight: neutralize tool tags inside markdown code spans.
+		// Models quote instructions containing `<get/>` etc. — the parser
+		// would treat them as real tool calls. Replace the angle brackets
+		// inside backtick spans so htmlparser2 ignores them.
+		const codeNeutralized = XmlParser.#neutralizeCodeSpans(normalized);
+
+		// Pre-flight: fix mismatched close tags that htmlparser2 silently
+		// drops (making our onclosetag recovery code unreachable). Must run
+		// before balanceAttrQuotes since the mismatch scan needs clean tags.
+		const mismatchFixed = XmlParser.#correctMismatchedCloses(
+			codeNeutralized,
+			warnings,
+		);
+
+		// Pre-flight: balance unclosed attribute quotes that would otherwise
+		// cause htmlparser2 to consume the rest of input as a single attribute
+		// value, silently dropping every subsequent tool call.
+		const balanced = XmlParser.#balanceAttrQuotes(mismatchFixed, warnings);
 		let current = null;
 		let ended = false;
 		let capped = false;
@@ -162,35 +178,48 @@ export default class XmlParser {
 			{
 				onopentag(name, attrs) {
 					if (capped) return;
-					if (!ALL_TOOLS.has(name)) {
-						if (current) {
+
+					if (current) {
+						// Empty-body case: current tool opened but got no text
+						// content before a new tag. The model likely meant current
+						// to self-close but typed it in paired form, or emitted a
+						// mismatched close tag that htmlparser2 silently dropped.
+						// Close current, open new.
+						const hasBody = current.rawBody.trim() !== "";
+						const hasNestedOpens = (current.nested || []).length > 0;
+						if (!hasBody && !hasNestedOpens && ALL_TOOLS.has(name)) {
+							warnings.push(
+								`Unclosed <${current.name}> before <${name}> — recovered`,
+							);
+							commands.push(
+								resolveCommand(current.name, current.attrs, current.rawBody),
+							);
+							current = null;
+						} else {
+							// Nested tag inside a body with content — treat as body
+							// text. Tool bodies are opaque: the model writing a plan
+							// with <get/> in it, SEARCH/REPLACE in <set>, or XML
+							// examples in <known> all need to survive intact. Track
+							// nested opens on a stack so matching closes pop off and
+							// orphan closes (typos) still trigger recovery.
 							const attrStr = Object.entries(attrs)
-								.map(([k, v]) => v === "" ? k : `${k}="${v}"`)
+								.map(([k, v]) => (v === "" ? k : `${k}="${v}"`))
 								.join(" ");
-							current.rawBody += attrStr
-								? `<${name} ${attrStr}>`
-								: `<${name}>`;
+							current.rawBody += attrStr ? `<${name} ${attrStr}>` : `<${name}>`;
+							current.nested ||= [];
+							current.nested.push(name);
+							return;
 						}
-						return;
 					}
 
-					// Known tool opened while another is still open — close the old one.
-					if (current) {
-						warnings.push(
-							`Unclosed <${current.name}> before <${name}> — recovered`,
-						);
-						commands.push(
-							resolveCommand(current.name, current.attrs, current.rawBody),
-						);
-					}
+					if (!ALL_TOOLS.has(name)) return;
 
 					if (commands.length >= XmlParser.MAX_COMMANDS) {
 						capped = true;
-						current = null;
 						return;
 					}
 
-					current = { name, attrs, rawBody: "" };
+					current = { name, attrs, rawBody: "", nested: [] };
 				},
 
 				ontext(text) {
@@ -204,28 +233,49 @@ export default class XmlParser {
 
 				onclosetag(name, isImplied) {
 					if (capped) return;
-					if (current && name === current.name) {
-						if (ended) {
-							warnings.push(`Unclosed <${name}> tag — content captured anyway`);
+
+					if (current) {
+						// Matching nested close — pop stack, keep as text.
+						const nested = current.nested;
+						if (nested.length > 0 && nested[nested.length - 1] === name) {
+							nested.pop();
+							current.rawBody += `</${name}>`;
+							return;
 						}
-						commands.push(
-							resolveCommand(current.name, current.attrs, current.rawBody),
-						);
-						current = null;
-					} else if (current && ALL_TOOLS.has(name)) {
-						// Mismatched close tag for a known tool — close current tag,
-						// don't swallow subsequent commands as body text.
-						warnings.push(
-							`Mismatched </${name}> closing <${current.name}> — recovered`,
-						);
-						commands.push(
-							resolveCommand(current.name, current.attrs, current.rawBody),
-						);
-						current = null;
-					} else if (current) {
+
+						// Matching close for outer tool — finalize.
+						if (name === current.name && nested.length === 0) {
+							if (ended) {
+								warnings.push(
+									`Unclosed <${name}> tag — content captured anyway`,
+								);
+							}
+							commands.push(
+								resolveCommand(current.name, current.attrs, current.rawBody),
+							);
+							current = null;
+							return;
+						}
+
+						// Orphan close for a known tool (likely typo) — recover.
+						if (ALL_TOOLS.has(name)) {
+							warnings.push(
+								`Mismatched </${name}> closing <${current.name}> — recovered`,
+							);
+							commands.push(
+								resolveCommand(current.name, current.attrs, current.rawBody),
+							);
+							current = null;
+							return;
+						}
+
+						// Unknown orphan close — text.
 						current.rawBody += `</${name}>`;
-					} else if (isImplied && ALL_TOOLS.has(name)) {
-						// Self-closing tag that htmlparser2 auto-closed
+						return;
+					}
+
+					if (isImplied && ALL_TOOLS.has(name)) {
+						// Self-closing tag that htmlparser2 auto-closed at top level
 					}
 				},
 
@@ -240,7 +290,7 @@ export default class XmlParser {
 			},
 		);
 
-		parser.write(normalized);
+		parser.write(balanced);
 		ended = true;
 		parser.end();
 
@@ -263,6 +313,88 @@ export default class XmlParser {
 		return { commands, warnings, unparsed };
 	}
 
+	/**
+	 * Repair a specific malformed-tag pattern: an attribute value opened with
+	 * `="` that never closes before the next tag. Without repair, htmlparser2
+	 * consumes the rest of input as one giant attribute value and silently
+	 * drops every subsequent tool call.
+	 *
+	 * Pattern matched:  <TAG ... ATTR="text-with-no-quote</NEXT>
+	 * Repair:           <TAG ... ATTR="text-with-no-quote"></NEXT>
+	 *
+	 * Conservative — only triggers when the value contains no quote, no `>`,
+	 * and is followed by another tag opening or close. Well-formed input is
+	 * untouched.
+	 */
+	static #balanceAttrQuotes(content, warnings) {
+		let fixes = 0;
+		const repaired = content.replace(
+			/(<\w+\s[^<>]*?\w+=")([^"<>]*?)(<\/?\w+)/g,
+			(_, opening, value, nextTag) => {
+				fixes++;
+				return `${opening}${value}">${nextTag}`;
+			},
+		);
+		if (fixes > 0) {
+			warnings.push(
+				`Repaired ${fixes} malformed attribute(s) — close all attribute values with a quote.`,
+			);
+		}
+		return repaired;
+	}
+
+	/**
+	 * Correct mismatched close tags before htmlparser2 sees them.
+	 *
+	 * htmlparser2 silently drops close tags that don't match the currently
+	 * open element (e.g. `<set>body</known>` — `</known>` vanishes). This
+	 * makes the explicit mismatch recovery in onclosetag unreachable and
+	 * causes all subsequent sibling commands to be absorbed as body text.
+	 *
+	 * Conservative: only corrects when the mismatch is at the outermost
+	 * tool depth (stack.length === 1). Nested mismatches inside body text
+	 * are left for htmlparser2 + body opacity to handle normally.
+	 */
+	/**
+	 * Neutralize XML tags inside markdown code spans so the parser
+	 * doesn't treat quoted tool names as real commands.
+	 * `<get/>` → `&lt;get/&gt;`  (htmlparser2 ignores entities)
+	 */
+	static #neutralizeCodeSpans(content) {
+		return content.replace(/`([^`]*)`/g, (match, inner) => {
+			if (!/<\/?[\w]/.test(inner)) return match;
+			return `\`${inner.replace(/</g, "&lt;").replace(/>/g, "&gt;")}\``;
+		});
+	}
+
+	static #correctMismatchedCloses(content, warnings) {
+		const stack = [];
+		return content.replace(
+			/<(\/?)(\w+)([^>]*?)(\/?)>/g,
+			(match, slash, tag, _attrs, selfClose) => {
+				if (!ALL_TOOLS.has(tag)) return match;
+				if (selfClose === "/") return match;
+				if (slash === "/") {
+					if (stack.length === 0) return match;
+					if (stack[stack.length - 1] === tag) {
+						stack.pop();
+						return match;
+					}
+					if (stack.length === 1) {
+						const top = stack.pop();
+						warnings.push(
+							`Mismatched </${tag}> closing <${top}> — corrected to </${top}>`,
+						);
+						return `</${top}>`;
+					}
+					return match;
+				}
+				stack.push(tag);
+				return match;
+			},
+		);
+	}
+
 	/**
 	 * Normalize native tool call formats to rummy XML.
 	 * Models sometimes emit their training-format tool calls instead of
@@ -276,12 +408,30 @@ export default class XmlParser {
 		);
 
 		// Qwen/gemma: <|tool_call>call:NAME{key:"value"}<tool_call|>
+		// NAME may be namespaced with any of /, :, or . separators
+		// (e.g. `rummy.nvim/get`, `rummy:get`) — extract the trailing word
+		// sequence as the tool name. Value forms observed in the wild:
+		//   key="v" / key:"v" / key:v (unquoted) / key:<|"|>v<|"|> (gemma chat-quotes)
 		result = result.replace(
-			/<\|tool_call>call:(\w+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
-			(_, name, params) => {
-				if (!ALL_TOOLS.has(name)) return _;
-				const valueMatch = params.match(/["']([^"']+)["']/);
-				const body = valueMatch?.[1] || "";
+			/<\|tool_call>call:([\w.:/-]+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
+			(match, qualifiedName, params) => {
+				const name = qualifiedName.match(/\w+$/)?.[0] ?? qualifiedName;
+				if (!ALL_TOOLS.has(name)) {
+					return `<error>Unknown command '${qualifiedName}' in <|tool_call> format. Use XML commands listed above.</error>`;
+				}
+				const valueMatch = params.match(
+					/[=:]\s*(?:<\|"\|>([^<]*?)<\|"\|>|"([^"]*)"|'([^']*)'|([^,}]+))/,
+				);
+				const body = (
+					valueMatch?.[1] ??
+					valueMatch?.[2] ??
+					valueMatch?.[3] ??
+					valueMatch?.[4] ??
+					""
+				).trim();
+				if (!body) {
+					return `<error>Could not extract argument from <|tool_call> ${match}. Use XML format like <${name}>value</${name}>.</error>`;
+				}
 				return `<${name}>${body}</${name}>`;
 			},
 		);
@@ -319,6 +469,31 @@ export default class XmlParser {
 			},
 		);
 
+		// Catch-all: any remaining <|tool_call> tokens are malformed native
+		// attempts (no {} block, missing close, wrong shape entirely). Replace
+		// each with an <error> so the model gets feedback on its next turn and
+		// learns to switch to XML. Lazy-match up to the next native close, the
+		// next XML close tag, or end of input — preserves any trailing valid XML.
+		// Error body must NOT contain literal <get>/<set>/etc. — those would
+		// re-enter the parser as phantom tool calls. Describe the format in
+		// prose instead and point at the tool docs above.
+		result = result.replace(
+			/<\|tool_call>[\s\S]*?(?:<\|?tool_call\|?>|<\/\w+>|$)/g,
+			() =>
+				"<error>Native tool call format not supported. Use the XML commands listed above (e.g. a get tag with a path attribute, or a set tag with path and body).</error>",
+		);
+
+		// Strip any orphan chat-format quote tokens left after replacement.
+		result = result.replace(/<\|"\|>/g, '"');
+
+		// Gemma sometimes leaks OpenAI-harmony channel markers around its
+		// real XML output: `<|channel>thought\n<channel|>…<set path=…/>`.
+		// These aren't tool calls (handled above), they're role/channel
+		// tokens. Strip any remaining `<|name>` / `<name|>` pseudo-tags
+		// before the XML parser sees them.
+		result = result.replace(/<\|[\w:/-]+>/g, "");
+		result = result.replace(/<[\w:/-]+\|>/g, "");
+
 		return result;
 	}
 }

package/src/agent/budget.js

@@ -0,0 +1,56 @@
+import { countTokens } from "./tokens.js";
+
+const CEILING_RATIO = Number(process.env.RUMMY_BUDGET_CEILING);
+if (!CEILING_RATIO) throw new Error("RUMMY_BUDGET_CEILING must be set");
+
+export function ceiling(contextSize) {
+	return Math.floor(contextSize * CEILING_RATIO);
+}
+
+/**
+ * Sum assembled-message token counts.
+ * Used by the budget enforce gate, which has the real messages.
+ */
+export function measureMessages(messages) {
+	return messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+}
+
+/**
+ * Sum projected row body token counts — what's actually in the packet
+ * for each entry at its current visibility. Used by prompt.js while
+ * generating the <prompt> tag (before assembly completes).
+ */
+export function measureRows(rows) {
+	return rows.reduce((sum, r) => sum + countTokens(r.body), 0);
+}
+
+/**
+ * Single source of truth for budget numbers. Every caller — prompt.js
+ * generating the <prompt> tag, budget.js enforcing the ceiling,
+ * AgentLoop emitting telemetry — passes in its own measured totalTokens
+ * and reads the same object back. No fallbacks: callers produce the
+ * measurement they have.
+ *
+ * Returns:
+ *   ceiling     — floor(contextSize × CEILING_RATIO), the hard wall
+ *   totalTokens — echoed back (the full packet size the caller measured)
+ *   tokenUsage  — same as totalTokens. Kept under this name for the
+ *                 `<prompt tokenUsage="N">` attribute on the wire. Must
+ *                 agree with totalTokens so the model's math is honest.
+ *   tokensFree  — ceiling − totalTokens (floor 0)
+ *   overflow    — max(0, totalTokens − ceiling)
+ *   ok          — overflow === 0
+ */
+export function computeBudget({ contextSize, totalTokens }) {
+	const cap = ceiling(contextSize);
+	const tokensFree = Math.max(0, cap - totalTokens);
+	const overflow = Math.max(0, totalTokens - cap);
+	return {
+		ceiling: cap,
+		totalTokens,
+		tokenUsage: totalTokens,
+		tokensFree,
+		overflow,
+		ok: overflow === 0,
+	};
+}

package/src/agent/errors.js

@@ -0,0 +1,22 @@
+/**
+ * Typed errors for the agent/Entries layer. Callers catch by type,
+ * not by regex.
+ */
+
+/**
+ * Thrown when a writer tier isn't permitted to write to a scheme.
+ * See SPEC writer_tiers: schemes declare writable_by = subset of
+ * {system, plugin, client, model}. A write from an excluded tier
+ * rejects with this error.
+ */
+export class PermissionError extends Error {
+	constructor(scheme, writer, allowed) {
+		super(
+			`403: writer "${writer}" not permitted for scheme "${scheme ?? "file"}" (allowed: ${allowed.join(", ")})`,
+		);
+		this.name = "PermissionError";
+		this.scheme = scheme ?? "file";
+		this.writer = writer;
+		this.allowed = [...allowed];
+	}
+}

package/src/agent/httpStatus.js

@@ -0,0 +1,39 @@
+/**
+ * Map the entry-layer (state, outcome) tuple to an HTTP status number for
+ * model-facing tag rendering.
+ *
+ * Model-facing tags still carry `status="NNN"` because the model's
+ * vocabulary (instructions + tooldocs + training) is HTTP-shaped. The DB
+ * stores categorical state + textual outcome (see SPEC entries); this helper
+ * is the one-way translation for rendering.
+ *
+ * Outcome strings prefixed with a 3-digit HTTP code (e.g.
+ * `"overflow:413:..."` or `"permission:403:..."`) extract the code
+ * verbatim. Otherwise state maps to a canonical HTTP:
+ *
+ *   resolved   → 200
+ *   proposed   → 202
+ *   streaming  → 102
+ *   cancelled  → 499
+ *   failed     → 500 (unless outcome carries a code)
+ */
+export function stateToStatus(state, outcome = null) {
+	if (outcome) {
+		const match = /(\d{3})/.exec(outcome);
+		if (match) return Number(match[1]);
+	}
+	switch (state) {
+		case "resolved":
+			return 200;
+		case "proposed":
+			return 202;
+		case "streaming":
+			return 102;
+		case "cancelled":
+			return 499;
+		case "failed":
+			return 500;
+		default:
+			throw new Error(`stateToStatus: unknown state "${state}"`);
+	}
+}

package/src/agent/known_checks.sql

@@ -17,27 +17,31 @@ SELECT path, body, attributes, turn
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND status = 202;
+	AND state = 'proposed';
 
 -- PREP: has_rejections
+-- Any failed entry in this loop counts as a rejection. Callers use
+-- this to mark the turn as having errors. Specific failure categories
+-- live in run_views.outcome (permission:, overflow:, validation:, ...).
 SELECT COUNT(*) AS count
 FROM known_entries
 WHERE
 	run_id = :run_id
 	AND loop_id = :loop_id
-	AND status = 403;
+	AND state = 'failed';
 
 -- PREP: has_accepted_actions
 SELECT COUNT(*) AS count
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND status = 200
+	AND state = 'resolved'
 	AND scheme IN ('set', 'sh', 'rm', 'mv', 'cp');
 
 -- PREP: get_file_entries
-SELECT path, status, fidelity, hash, updated_at
+SELECT path, state, outcome, visibility, hash, updated_at
 FROM known_entries
 WHERE
 	run_id = :run_id
 	AND scheme IS NULL;
+

package/src/agent/known_queries.sql

@@ -1,11 +1,13 @@
 -- PREP: get_known_entries
-SELECT path, scheme, status, fidelity, body, turn, hash, attributes, tokens
+SELECT
+	path, scheme, state, outcome, visibility, body, turn, hash
+	, attributes, countTokens(body) AS tokens, scope
 FROM known_entries
 WHERE run_id = :run_id
 ORDER BY path;
 
 -- PREP: get_results
-SELECT tool, target, status, path, body
+SELECT tool, state, outcome, path, body, turn, attributes
 FROM v_run_log
 WHERE run_id = :run_id;
 
@@ -18,7 +20,7 @@ WHERE
 ORDER BY id;
 
 -- PREP: get_turn_audit
-SELECT path, scheme, status, fidelity, turn, body, attributes
+SELECT path, scheme, state, outcome, visibility, turn, body, attributes
 FROM known_entries
 WHERE
 	run_id = :run_id
@@ -53,24 +55,18 @@ ORDER BY id DESC
 LIMIT 1;
 
 -- PREP: get_latest_summary
+-- Updates live in the unified log namespace at log://turn_N/update/<slug>,
+-- not at a dedicated "update" scheme. Match path shape instead of scheme.
 SELECT body
 FROM known_entries
 WHERE
 	run_id = :run_id
 	AND loop_id = :loop_id
-	AND scheme = 'summarize'
+	AND path LIKE 'log://turn_%/update/%'
 ORDER BY id DESC
 LIMIT 1;
 
--- PREP: get_history
-SELECT ke.path, ke.status, ke.body, ke.attributes, ke.turn
-FROM known_entries AS ke
-JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
-WHERE
-	ke.run_id = :run_id
-	AND ke.scheme IS NOT NULL
-	AND s.category NOT IN ('knowledge', 'file', 'audit')
-ORDER BY ke.id;
+-- get_history retired — use get_results (v_run_log) for both run/state and getRun.
 
 -- PREP: get_content
 SELECT path, body, turn