@possumtech/rummy 0.5.0 → 2.0.0

package/src/agent/XmlParser.js

@@ -1,18 +1,15 @@
 import { Parser } from "htmlparser2";
 import { parseEditContent } from "../plugins/hedberg/edits.js";
-import { normalizeAttrs, parseJsonEdit } from "../plugins/hedberg/normalize.js";
+import { parseJsonEdit } from "../plugins/hedberg/normalize.js";
 import { parseSed } from "../plugins/hedberg/sed.js";
 
 const STORE_TOOLS = new Set(["get", "rm", "set", "mv", "cp", "search"]);
 export const ALL_TOOLS = new Set([
 	...STORE_TOOLS,
-	"known",
 	"sh",
 	"env",
 	"ask_user",
-	"summarize",
 	"update",
-	"unknown",
 	"think",
 ]);
 
@@ -20,8 +17,7 @@ export const ALL_TOOLS = new Set([
  * Resolve the competing attr-vs-body philosophies per tool.
  * If the canonical attribute is missing, the body fills it. Silent.
  */
-function resolveCommand(name, attrs, rawBody) {
-	const a = normalizeAttrs(attrs);
+function resolveCommand(name, a, rawBody) {
 	const trimmed = rawBody.trim();
 
 	if (name === "set") {
@@ -88,47 +84,48 @@ function resolveCommand(name, attrs, rawBody) {
 				preview: a.preview,
 			};
 		}
-		// Plain write or fidelity change
+		// Plain write or visibility change
 		const body = trimmed || a.body || "";
 		return { name, ...a, body };
 	}
 
-	if (name === "summarize" || name === "update" || name === "unknown") {
+	if (name === "update") {
 		const body = trimmed || a.body || "";
-		return { name, body };
-	}
-
-	if (name === "known") {
-		const body = trimmed || a.body || "";
-		const path = a.path || null;
-		return { name, ...a, path, body };
+		const status = a.status ? Number(a.status) : 102;
+		return { name, ...a, body, status };
 	}
 
 	if (name === "get" || name === "rm") {
-		const path = a.path || trimmed || null;
-		return { name, path, body: a.body, preview: a.preview };
+		// Spread `a` so `line`, `limit`, `visibility`, and future attrs
+		// reach the handler. Earlier narrow extraction silently dropped
+		// `line=/limit=` and stranded the partial-read path advertised
+		// in getDoc.
+		return { name, ...a, path: a.path || trimmed || null };
 	}
 
 	if (name === "search") {
 		const path = a.path || trimmed || null;
 		const results = a.results ? Number(a.results) : null;
-		return { name, path, results };
+		return { name, ...a, path, results };
 	}
 
 	if (name === "mv" || name === "cp") {
-		const to = a.to || trimmed || null;
-		return { name, path: a.path, to };
+		// Spread `a` so `visibility` reaches the handler. mvDoc
+		// advertises `<mv path="known://..." visibility="summarized"/>`
+		// for batch visibility changes and was silently stripping that
+		// attr before.
+		return { name, ...a, path: a.path, to: a.to || trimmed || null };
 	}
 
 	if (name === "sh" || name === "env") {
 		const command = a.command || trimmed || null;
-		return { name, command };
+		return { name, ...a, command };
 	}
 
 	if (name === "ask_user") {
 		const question = a.question || null;
 		const options = a.options || trimmed || null;
-		return { name, question, options };
+		return { name, ...a, question, options };
 	}
 
 	return { name, ...a, body: trimmed || a.body };
@@ -143,7 +140,7 @@ export default class XmlParser {
 	 * @param {string} content - Raw model response text
 	 * @returns {{ commands: Array, warnings: string[], unparsed: string }}
 	 */
-	static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS) || 99;
+	static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS);
 
 	static parse(content) {
 		if (!content) return { commands: [], warnings: [], unparsed: "" };
@@ -155,10 +152,24 @@ export default class XmlParser {
 		const warnings = [];
 		const textChunks = [];
 
+		// Pre-flight: neutralize tool tags inside markdown code spans.
+		// Models quote instructions containing `<get/>` etc. — the parser
+		// would treat them as real tool calls. Replace the angle brackets
+		// inside backtick spans so htmlparser2 ignores them.
+		const codeNeutralized = XmlParser.#neutralizeCodeSpans(normalized);
+
+		// Pre-flight: fix mismatched close tags that htmlparser2 silently
+		// drops (making our onclosetag recovery code unreachable). Must run
+		// before balanceAttrQuotes since the mismatch scan needs clean tags.
+		const mismatchFixed = XmlParser.#correctMismatchedCloses(
+			codeNeutralized,
+			warnings,
+		);
+
 		// Pre-flight: balance unclosed attribute quotes that would otherwise
 		// cause htmlparser2 to consume the rest of input as a single attribute
 		// value, silently dropping every subsequent tool call.
-		const balanced = XmlParser.#balanceAttrQuotes(normalized, warnings);
+		const balanced = XmlParser.#balanceAttrQuotes(mismatchFixed, warnings);
 		let current = null;
 		let ended = false;
 		let capped = false;
@@ -194,9 +205,7 @@ export default class XmlParser {
 							const attrStr = Object.entries(attrs)
 								.map(([k, v]) => (v === "" ? k : `${k}="${v}"`))
 								.join(" ");
-							current.rawBody += attrStr
-								? `<${name} ${attrStr}>`
-								: `<${name}>`;
+							current.rawBody += attrStr ? `<${name} ${attrStr}>` : `<${name}>`;
 							current.nested ||= [];
 							current.nested.push(name);
 							return;
@@ -228,10 +237,7 @@ export default class XmlParser {
 					if (current) {
 						// Matching nested close — pop stack, keep as text.
 						const nested = current.nested;
-						if (
-							nested.length > 0 &&
-							nested[nested.length - 1] === name
-						) {
+						if (nested.length > 0 && nested[nested.length - 1] === name) {
 							nested.pop();
 							current.rawBody += `</${name}>`;
 							return;
@@ -337,6 +343,58 @@ export default class XmlParser {
 		return repaired;
 	}
 
+	/**
+	 * Correct mismatched close tags before htmlparser2 sees them.
+	 *
+	 * htmlparser2 silently drops close tags that don't match the currently
+	 * open element (e.g. `<set>body</known>` — `</known>` vanishes). This
+	 * makes the explicit mismatch recovery in onclosetag unreachable and
+	 * causes all subsequent sibling commands to be absorbed as body text.
+	 *
+	 * Conservative: only corrects when the mismatch is at the outermost
+	 * tool depth (stack.length === 1). Nested mismatches inside body text
+	 * are left for htmlparser2 + body opacity to handle normally.
+	 */
+	/**
+	 * Neutralize XML tags inside markdown code spans so the parser
+	 * doesn't treat quoted tool names as real commands.
+	 * `<get/>` → `&lt;get/&gt;`  (htmlparser2 ignores entities)
+	 */
+	static #neutralizeCodeSpans(content) {
+		return content.replace(/`([^`]*)`/g, (match, inner) => {
+			if (!/<\/?[\w]/.test(inner)) return match;
+			return `\`${inner.replace(/</g, "&lt;").replace(/>/g, "&gt;")}\``;
+		});
+	}
+
+	static #correctMismatchedCloses(content, warnings) {
+		const stack = [];
+		return content.replace(
+			/<(\/?)(\w+)([^>]*?)(\/?)>/g,
+			(match, slash, tag, _attrs, selfClose) => {
+				if (!ALL_TOOLS.has(tag)) return match;
+				if (selfClose === "/") return match;
+				if (slash === "/") {
+					if (stack.length === 0) return match;
+					if (stack[stack.length - 1] === tag) {
+						stack.pop();
+						return match;
+					}
+					if (stack.length === 1) {
+						const top = stack.pop();
+						warnings.push(
+							`Mismatched </${tag}> closing <${top}> — corrected to </${top}>`,
+						);
+						return `</${top}>`;
+					}
+					return match;
+				}
+				stack.push(tag);
+				return match;
+			},
+		);
+	}
+
 	/**
 	 * Normalize native tool call formats to rummy XML.
 	 * Models sometimes emit their training-format tool calls instead of
@@ -359,7 +417,7 @@ export default class XmlParser {
 			(match, qualifiedName, params) => {
 				const name = qualifiedName.match(/\w+$/)?.[0] ?? qualifiedName;
 				if (!ALL_TOOLS.has(name)) {
-					return `<error>Unknown tool '${qualifiedName}' in <|tool_call> format. Use XML tool commands listed above.</error>`;
+					return `<error>Unknown command '${qualifiedName}' in <|tool_call> format. Use XML commands listed above.</error>`;
 				}
 				const valueMatch = params.match(
 					/[=:]\s*(?:<\|"\|>([^<]*?)<\|"\|>|"([^"]*)"|'([^']*)'|([^,}]+))/,
@@ -422,12 +480,20 @@ export default class XmlParser {
 		result = result.replace(
 			/<\|tool_call>[\s\S]*?(?:<\|?tool_call\|?>|<\/\w+>|$)/g,
 			() =>
-				"<error>Native tool call format not supported. Use the XML tool commands listed above (e.g. a get tag with a path attribute, or a set tag with path and body).</error>",
+				"<error>Native tool call format not supported. Use the XML commands listed above (e.g. a get tag with a path attribute, or a set tag with path and body).</error>",
 		);
 
 		// Strip any orphan chat-format quote tokens left after replacement.
 		result = result.replace(/<\|"\|>/g, '"');
 
+		// Gemma sometimes leaks OpenAI-harmony channel markers around its
+		// real XML output: `<|channel>thought\n<channel|>…<set path=…/>`.
+		// These aren't tool calls (handled above), they're role/channel
+		// tokens. Strip any remaining `<|name>` / `<name|>` pseudo-tags
+		// before the XML parser sees them.
+		result = result.replace(/<\|[\w:/-]+>/g, "");
+		result = result.replace(/<[\w:/-]+\|>/g, "");
+
 		return result;
 	}
 }

package/src/agent/budget.js

@@ -0,0 +1,56 @@
+import { countTokens } from "./tokens.js";
+
+const CEILING_RATIO = Number(process.env.RUMMY_BUDGET_CEILING);
+if (!CEILING_RATIO) throw new Error("RUMMY_BUDGET_CEILING must be set");
+
+export function ceiling(contextSize) {
+	return Math.floor(contextSize * CEILING_RATIO);
+}
+
+/**
+ * Sum assembled-message token counts.
+ * Used by the budget enforce gate, which has the real messages.
+ */
+export function measureMessages(messages) {
+	return messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+}
+
+/**
+ * Sum projected row body token counts — what's actually in the packet
+ * for each entry at its current visibility. Used by prompt.js while
+ * generating the <prompt> tag (before assembly completes).
+ */
+export function measureRows(rows) {
+	return rows.reduce((sum, r) => sum + countTokens(r.body), 0);
+}
+
+/**
+ * Single source of truth for budget numbers. Every caller — prompt.js
+ * generating the <prompt> tag, budget.js enforcing the ceiling,
+ * AgentLoop emitting telemetry — passes in its own measured totalTokens
+ * and reads the same object back. No fallbacks: callers produce the
+ * measurement they have.
+ *
+ * Returns:
+ *   ceiling     — floor(contextSize × CEILING_RATIO), the hard wall
+ *   totalTokens — echoed back (the full packet size the caller measured)
+ *   tokenUsage  — same as totalTokens. Kept under this name for the
+ *                 `<prompt tokenUsage="N">` attribute on the wire. Must
+ *                 agree with totalTokens so the model's math is honest.
+ *   tokensFree  — ceiling − totalTokens (floor 0)
+ *   overflow    — max(0, totalTokens − ceiling)
+ *   ok          — overflow === 0
+ */
+export function computeBudget({ contextSize, totalTokens }) {
+	const cap = ceiling(contextSize);
+	const tokensFree = Math.max(0, cap - totalTokens);
+	const overflow = Math.max(0, totalTokens - cap);
+	return {
+		ceiling: cap,
+		totalTokens,
+		tokenUsage: totalTokens,
+		tokensFree,
+		overflow,
+		ok: overflow === 0,
+	};
+}

package/src/agent/errors.js

@@ -0,0 +1,22 @@
+/**
+ * Typed errors for the agent/Entries layer. Callers catch by type,
+ * not by regex.
+ */
+
+/**
+ * Thrown when a writer tier isn't permitted to write to a scheme.
+ * See SPEC writer_tiers: schemes declare writable_by = subset of
+ * {system, plugin, client, model}. A write from an excluded tier
+ * rejects with this error.
+ */
+export class PermissionError extends Error {
+	constructor(scheme, writer, allowed) {
+		super(
+			`403: writer "${writer}" not permitted for scheme "${scheme ?? "file"}" (allowed: ${allowed.join(", ")})`,
+		);
+		this.name = "PermissionError";
+		this.scheme = scheme ?? "file";
+		this.writer = writer;
+		this.allowed = [...allowed];
+	}
+}

package/src/agent/httpStatus.js

@@ -0,0 +1,39 @@
+/**
+ * Map the entry-layer (state, outcome) tuple to an HTTP status number for
+ * model-facing tag rendering.
+ *
+ * Model-facing tags still carry `status="NNN"` because the model's
+ * vocabulary (instructions + tooldocs + training) is HTTP-shaped. The DB
+ * stores categorical state + textual outcome (see SPEC entries); this helper
+ * is the one-way translation for rendering.
+ *
+ * Outcome strings prefixed with a 3-digit HTTP code (e.g.
+ * `"overflow:413:..."` or `"permission:403:..."`) extract the code
+ * verbatim. Otherwise state maps to a canonical HTTP:
+ *
+ *   resolved   → 200
+ *   proposed   → 202
+ *   streaming  → 102
+ *   cancelled  → 499
+ *   failed     → 500 (unless outcome carries a code)
+ */
+export function stateToStatus(state, outcome = null) {
+	if (outcome) {
+		const match = /(\d{3})/.exec(outcome);
+		if (match) return Number(match[1]);
+	}
+	switch (state) {
+		case "resolved":
+			return 200;
+		case "proposed":
+			return 202;
+		case "streaming":
+			return 102;
+		case "cancelled":
+			return 499;
+		case "failed":
+			return 500;
+		default:
+			throw new Error(`stateToStatus: unknown state "${state}"`);
+	}
+}

package/src/agent/known_checks.sql

@@ -17,27 +17,31 @@ SELECT path, body, attributes, turn
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND status = 202;
+	AND state = 'proposed';
 
 -- PREP: has_rejections
+-- Any failed entry in this loop counts as a rejection. Callers use
+-- this to mark the turn as having errors. Specific failure categories
+-- live in run_views.outcome (permission:, overflow:, validation:, ...).
 SELECT COUNT(*) AS count
 FROM known_entries
 WHERE
 	run_id = :run_id
 	AND loop_id = :loop_id
-	AND status = 403;
+	AND state = 'failed';
 
 -- PREP: has_accepted_actions
 SELECT COUNT(*) AS count
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND status = 200
+	AND state = 'resolved'
 	AND scheme IN ('set', 'sh', 'rm', 'mv', 'cp');
 
 -- PREP: get_file_entries
-SELECT path, status, fidelity, hash, updated_at
+SELECT path, state, outcome, visibility, hash, updated_at
 FROM known_entries
 WHERE
 	run_id = :run_id
 	AND scheme IS NULL;
+

package/src/agent/known_queries.sql

@@ -1,11 +1,13 @@
 -- PREP: get_known_entries
-SELECT path, scheme, status, fidelity, body, turn, hash, attributes, tokens
+SELECT
+	path, scheme, state, outcome, visibility, body, turn, hash
+	, attributes, countTokens(body) AS tokens, scope
 FROM known_entries
 WHERE run_id = :run_id
 ORDER BY path;
 
 -- PREP: get_results
-SELECT tool, target, status, path, body
+SELECT tool, state, outcome, path, body, turn, attributes
 FROM v_run_log
 WHERE run_id = :run_id;
 
@@ -18,7 +20,7 @@ WHERE
 ORDER BY id;
 
 -- PREP: get_turn_audit
-SELECT path, scheme, status, fidelity, turn, body, attributes
+SELECT path, scheme, state, outcome, visibility, turn, body, attributes
 FROM known_entries
 WHERE
 	run_id = :run_id
@@ -53,24 +55,18 @@ ORDER BY id DESC
 LIMIT 1;
 
 -- PREP: get_latest_summary
+-- Updates live in the unified log namespace at log://turn_N/update/<slug>,
+-- not at a dedicated "update" scheme. Match path shape instead of scheme.
 SELECT body
 FROM known_entries
 WHERE
 	run_id = :run_id
 	AND loop_id = :loop_id
-	AND scheme = 'summarize'
+	AND path LIKE 'log://turn_%/update/%'
 ORDER BY id DESC
 LIMIT 1;
 
--- PREP: get_history
-SELECT ke.path, ke.status, ke.body, ke.attributes, ke.turn
-FROM known_entries AS ke
-JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
-WHERE
-	ke.run_id = :run_id
-	AND ke.scheme IS NOT NULL
-	AND s.category NOT IN ('knowledge', 'file', 'audit')
-ORDER BY ke.id;
+-- get_history retired — use get_results (v_run_log) for both run/state and getRun.
 
 -- PREP: get_content
 SELECT path, body, turn