@possumtech/rummy 0.2.7 → 0.3.0

package/src/agent/XmlParser.js

@@ -1,17 +1,9 @@
 import { Parser } from "htmlparser2";
 import { parseEditContent } from "../plugins/hedberg/edits.js";
-import { normalizeAttrs } from "../plugins/hedberg/normalize.js";
+import { normalizeAttrs, parseJsonEdit } from "../plugins/hedberg/normalize.js";
 import { parseSed } from "../plugins/hedberg/sed.js";
 
-const STORE_TOOLS = new Set([
-	"get",
-	"store",
-	"rm",
-	"set",
-	"mv",
-	"cp",
-	"search",
-]);
+const STORE_TOOLS = new Set(["get", "rm", "set", "mv", "cp", "search"]);
 const ALL_TOOLS = new Set([
 	...STORE_TOOLS,
 	"known",
@@ -51,31 +43,10 @@ function resolveCommand(name, attrs, rawBody) {
 				};
 			}
 		}
-		// JSON-style { search, replace } — accept valid JSON and =style variants
-		if (trimmed.startsWith("{") && /search/.test(trimmed)) {
-			let search = null;
-			let replace = null;
-			try {
-				const json = JSON.parse(trimmed);
-				search = json.search;
-				replace = json.replace ?? "";
-			} catch {
-				// Try = style: { search="old", replace="new" }
-				const searchMatch = trimmed.match(/search\s*=\s*"([^"]*)"/);
-				const replaceMatch = trimmed.match(/replace\s*=\s*"([^"]*)"/);
-				if (searchMatch) {
-					search = searchMatch[1];
-					replace = replaceMatch?.[1] ?? "";
-				}
-			}
-			if (search != null) {
-				return {
-					name,
-					path: a.path,
-					search,
-					replace,
-				};
-			}
+		// JSON-style { search, replace }
+		const jsonEdit = parseJsonEdit(trimmed);
+		if (jsonEdit) {
+			return { name, path: a.path, ...jsonEdit };
 		}
 		// Sed syntax: s/search/replace/flags — supports chained commands
 		if (trimmed.startsWith("s/")) {
@@ -116,9 +87,9 @@ function resolveCommand(name, attrs, rawBody) {
 				preview: a.preview,
 			};
 		}
-		// Plain write → create/overwrite
+		// Plain write or fidelity change
 		const body = trimmed || a.body || "";
-		return { name, path: a.path, body, preview: a.preview };
+		return { name, ...a, body };
 	}
 
 	if (name === "summarize" || name === "update" || name === "unknown") {
@@ -132,7 +103,7 @@ function resolveCommand(name, attrs, rawBody) {
 		return { name, path, body };
 	}
 
-	if (name === "get" || name === "store" || name === "rm") {
+	if (name === "get" || name === "rm") {
 		const path = a.path || trimmed || null;
 		return { name, path, body: a.body, preview: a.preview };
 	}
@@ -174,6 +145,9 @@ export default class XmlParser {
 	static parse(content) {
 		if (!content) return { commands: [], warnings: [], unparsed: "" };
 
+		// Normalize native tool call formats to rummy XML
+		const normalized = XmlParser.#normalizeToolCalls(content);
+
 		const commands = [];
 		const warnings = [];
 		const textChunks = [];
@@ -190,6 +164,16 @@ export default class XmlParser {
 						return;
 					}
 
+					// Known tool opened while another is still open — close the old one.
+					if (current) {
+						warnings.push(
+							`Unclosed <${current.name}> before <${name}> — recovered`,
+						);
+						commands.push(
+							resolveCommand(current.name, current.attrs, current.rawBody),
+						);
+					}
+
 					current = { name, attrs, rawBody: "" };
 				},
 
@@ -210,6 +194,16 @@ export default class XmlParser {
 							resolveCommand(current.name, current.attrs, current.rawBody),
 						);
 						current = null;
+					} else if (current && ALL_TOOLS.has(name)) {
+						// Mismatched close tag for a known tool — close current tag,
+						// don't swallow subsequent commands as body text.
+						warnings.push(
+							`Mismatched </${name}> closing <${current.name}> — recovered`,
+						);
+						commands.push(
+							resolveCommand(current.name, current.attrs, current.rawBody),
+						);
+						current = null;
 					} else if (current) {
 						current.rawBody += `</${name}>`;
 					} else if (isImplied && ALL_TOOLS.has(name)) {
@@ -228,7 +222,7 @@ export default class XmlParser {
 			},
 		);
 
-		parser.write(content);
+		parser.write(normalized);
 		ended = true;
 		parser.end();
 
@@ -244,4 +238,63 @@ export default class XmlParser {
 		const unparsed = textChunks.join("").trim();
 		return { commands, warnings, unparsed };
 	}
+
+	/**
+	 * Normalize native tool call formats to rummy XML.
+	 * Models sometimes emit their training-format tool calls instead of
+	 * our XML tags. The intent is unambiguous — translate silently.
+	 */
+	static #normalizeToolCalls(content) {
+		// Gemma: ```tool_code\n<xml>...\n``` — strip code fences around valid XML
+		let result = content.replace(
+			/```(?:tool_code|tool_command|xml)\n([\s\S]*?)```/g,
+			(_, inner) => inner.trim(),
+		);
+
+		// Qwen/gemma: <|tool_call>call:NAME{key:"value"}<tool_call|>
+		result = result.replace(
+			/<\|tool_call>call:(\w+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
+			(_, name, params) => {
+				if (!ALL_TOOLS.has(name)) return _;
+				const valueMatch = params.match(/["']([^"']+)["']/);
+				const body = valueMatch?.[1] || "";
+				return `<${name}>${body}</${name}>`;
+			},
+		);
+
+		// OpenAI function_call JSON: {"name":"search","arguments":{"query":"..."}}
+		result = result.replace(
+			/\{"name"\s*:\s*"(\w+)"\s*,\s*"arguments"\s*:\s*\{([^}]*)\}\}/g,
+			(_, name, args) => {
+				if (!ALL_TOOLS.has(name)) return _;
+				const pairs = [...args.matchAll(/"(\w+)"\s*:\s*"([^"]*)"/g)];
+				const body = pairs[0]?.[2] || "";
+				return `<${name}>${body}</${name}>`;
+			},
+		);
+
+		// Anthropic: <tool_use><name>search</name><input>{"query":"..."}</input></tool_use>
+		result = result.replace(
+			/<tool_use>\s*<name>(\w+)<\/name>\s*<input>\{([^}]*)\}<\/input>\s*<\/tool_use>/g,
+			(_, name, args) => {
+				if (!ALL_TOOLS.has(name)) return _;
+				const pairs = [...args.matchAll(/"(\w+)"\s*:\s*"([^"]*)"/g)];
+				const body = pairs[0]?.[2] || "";
+				return `<${name}>${body}</${name}>`;
+			},
+		);
+
+		// Mistral: [TOOL_CALLS] [{"name":"search","arguments":{"query":"..."}}]
+		result = result.replace(
+			/\[TOOL_CALLS\]\s*\[\{"name"\s*:\s*"(\w+)"\s*,\s*"arguments"\s*:\s*\{([^}]*)\}\}\]/g,
+			(_, name, args) => {
+				if (!ALL_TOOLS.has(name)) return _;
+				const pairs = [...args.matchAll(/"(\w+)"\s*:\s*"([^"]*)"/g)];
+				const body = pairs[0]?.[2] || "";
+				return `<${name}>${body}</${name}>`;
+			},
+		);
+
+		return result;
+	}
 }

package/src/agent/known_checks.sql

@@ -17,25 +17,26 @@ SELECT path, body, attributes, turn
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND state = 'proposed';
+	AND status = 202;
 
 -- PREP: has_rejections
 SELECT COUNT(*) AS count
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND state = 'rejected';
+	AND loop_id = :loop_id
+	AND status = 403;
 
 -- PREP: has_accepted_actions
 SELECT COUNT(*) AS count
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND state = 'pass'
+	AND status = 200
 	AND scheme IN ('set', 'sh', 'rm', 'mv', 'cp');
 
 -- PREP: get_file_entries
-SELECT path, state, hash, updated_at
+SELECT path, status, fidelity, hash, updated_at
 FROM known_entries
 WHERE
 	run_id = :run_id

package/src/agent/known_queries.sql

@@ -1,5 +1,5 @@
 -- PREP: get_known_entries
-SELECT path, scheme, state, body, turn, hash, attributes
+SELECT path, scheme, status, fidelity, body, turn, hash, attributes
 FROM known_entries
 WHERE run_id = :run_id
 ORDER BY path;
@@ -18,7 +18,7 @@ WHERE
 ORDER BY id;
 
 -- PREP: get_turn_audit
-SELECT path, scheme, state, turn, body, attributes
+SELECT path, scheme, status, fidelity, turn, body, attributes
 FROM known_entries
 WHERE
 	run_id = :run_id
@@ -57,12 +57,13 @@ SELECT body
 FROM known_entries
 WHERE
 	run_id = :run_id
+	AND loop_id = :loop_id
 	AND scheme = 'summarize'
 ORDER BY id DESC
 LIMIT 1;
 
 -- PREP: get_history
-SELECT ke.path, ke.state AS status, ke.body, ke.attributes, ke.turn
+SELECT ke.path, ke.status, ke.body, ke.attributes, ke.turn
 FROM known_entries AS ke
 JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
 WHERE

package/src/agent/known_store.sql

@@ -1,19 +1,22 @@
 -- PREP: upsert_known_entry
 INSERT INTO known_entries (
-	run_id, turn, path, body, state, hash, attributes
-	, tokens, tokens_full, updated_at
+	run_id, loop_id, turn, path, body, status, fidelity, hash
+	, attributes, tokens, tokens_full, updated_at
 )
 VALUES (
-	:run_id, :turn, :path, :body, :state, :hash, COALESCE(:attributes, '{}')
+	:run_id, :loop_id, :turn, :path, :body, :status, :fidelity, :hash
+	, COALESCE(:attributes, '{}')
 	, countTokens(:body)
 	, countTokens(:body)
 	, COALESCE(:updated_at, CURRENT_TIMESTAMP)
 )
 ON CONFLICT (run_id, path) DO UPDATE SET
 	body = excluded.body
-	, state = excluded.state
+	, status = excluded.status
+	, fidelity = excluded.fidelity
 	, hash = COALESCE(excluded.hash, known_entries.hash)
 	, attributes = COALESCE(excluded.attributes, known_entries.attributes)
+	, loop_id = excluded.loop_id
 	, turn = excluded.turn
 	, tokens = countTokens(excluded.body)
 	, tokens_full = countTokens(excluded.body)
@@ -43,17 +46,25 @@ WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
 -- PREP: resolve_known_entry
 UPDATE known_entries
 SET
-	state = :state
+	status = :status
 	, body = :body
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
--- PREP: set_file_state
+-- PREP: set_file_fidelity
 UPDATE known_entries
 SET
-	state = :state
+	fidelity = :fidelity
 	, tokens = CASE
-		WHEN :state = 'summary' THEN countTokens(body)
+		WHEN :fidelity = 'archive'
+			THEN 0
+		WHEN :fidelity = 'index'
+			THEN 0
+		WHEN :fidelity = 'summary'
+			THEN COALESCE(
+				countTokens(json_extract(attributes, '$.summary')),
+				countTokens(substr(body, 1, 80))
+			)
 		ELSE tokens_full
 	END
 	, updated_at = CURRENT_TIMESTAMP
@@ -62,7 +73,7 @@ WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
 -- PREP: promote_path
 UPDATE known_entries
 SET
-	state = 'full'
+	fidelity = 'full'
 	, turn = :turn
 	, tokens = tokens_full
 	, updated_at = CURRENT_TIMESTAMP
@@ -71,23 +82,42 @@ WHERE run_id = :run_id AND path = :path;
 -- PREP: demote_path
 UPDATE known_entries
 SET
-	state = 'stored'
+	fidelity = 'archive'
 	, tokens = 0
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
+-- PREP: set_fidelity
+UPDATE known_entries
+SET
+	fidelity = :fidelity
+	, tokens = CASE
+		WHEN :fidelity = 'archive'
+			THEN 0
+		WHEN :fidelity = 'index'
+			THEN 0
+		WHEN :fidelity = 'summary'
+			THEN COALESCE(
+				countTokens(json_extract(attributes, '$.summary')),
+				countTokens(substr(body, 1, 80))
+			)
+		ELSE countTokens(body)
+	END
+	, updated_at = CURRENT_TIMESTAMP
+WHERE run_id = :run_id AND path = :path;
+
 -- PREP: get_entry_body
 SELECT body
 FROM known_entries
 WHERE run_id = :run_id AND path = :path;
 
 -- PREP: get_entry_state
-SELECT state, scheme, turn
+SELECT status, fidelity, scheme, turn
 FROM known_entries
 WHERE run_id = :run_id AND path = :path;
 
 -- PREP: get_file_states_by_pattern
-SELECT path, state, turn
+SELECT path, status, fidelity, turn
 FROM known_entries
 WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL
 ORDER BY path;
@@ -107,7 +137,7 @@ WHERE run_id = :run_id AND path = :path;
 -- PREP: promote_by_pattern
 UPDATE known_entries
 SET
-	state = 'full'
+	fidelity = 'full'
 	, turn = :turn
 	, tokens = tokens_full
 	, updated_at = CURRENT_TIMESTAMP
@@ -119,7 +149,7 @@ WHERE
 -- PREP: demote_by_pattern
 UPDATE known_entries
 SET
-	state = 'stored'
+	fidelity = 'archive'
 	, tokens = 0
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
@@ -128,7 +158,7 @@ WHERE
 	AND (:body IS NULL OR hedsearch(:body, body));
 
 -- PREP: get_entries_by_pattern
-SELECT path, body, scheme, state, tokens_full, attributes
+SELECT path, body, scheme, status, fidelity, tokens_full, attributes
 FROM known_entries
 WHERE
 	run_id = :run_id

package/src/agent/loops.sql

@@ -0,0 +1,63 @@
+-- PREP: enqueue_loop
+INSERT INTO loops (run_id, sequence, mode, model, prompt, config)
+VALUES (:run_id, :sequence, :mode, :model, :prompt, :config)
+RETURNING id, sequence;
+
+-- PREP: next_loop
+UPDATE runs
+SET next_loop = next_loop + 1
+WHERE id = :run_id
+RETURNING next_loop - 1 AS sequence;
+
+-- PREP: claim_next_loop
+UPDATE loops
+SET status = 102
+WHERE
+	id = (
+		SELECT
+			id
+		FROM loops
+		WHERE run_id = :run_id AND status = 100
+		ORDER BY id
+		LIMIT 1
+	)
+RETURNING id, run_id, sequence, mode, model, prompt, config;
+
+-- PREP: complete_loop
+UPDATE loops
+SET status = :status, result = :result
+WHERE id = :id;
+
+-- PREP: abort_active_loop
+UPDATE loops
+SET status = 499
+WHERE run_id = :run_id AND status = 102;
+
+-- PREP: get_pending_loops
+SELECT id, sequence, mode, model, prompt, status, created_at
+FROM loops
+WHERE run_id = :run_id AND status IN (100, 102)
+ORDER BY id;
+
+-- PREP: reset_active_loops
+UPDATE loops
+SET status = 100
+WHERE status = 102;
+
+-- PREP: get_current_loop
+SELECT id, sequence, mode, model, prompt, status
+FROM loops
+WHERE run_id = :run_id AND status = 102
+LIMIT 1;
+
+-- PREP: get_loop_by_id
+SELECT id, run_id, sequence, mode, model, prompt, status, config
+FROM loops
+WHERE id = :id;
+
+-- PREP: get_latest_completed_loop
+SELECT id, sequence, mode, status
+FROM loops
+WHERE run_id = :run_id AND status IN (200, 500)
+ORDER BY id DESC
+LIMIT 1;

package/src/agent/runs.sql

@@ -22,14 +22,14 @@ RETURNING id;
 -- PREP: get_run_by_alias
 SELECT
 	id, project_id, parent_run_id, model, status, alias
-	, temperature, persona, context_limit, next_turn, created_at
+	, temperature, persona, context_limit, next_turn, next_loop, created_at
 FROM runs
 WHERE alias = :alias;
 
 -- PREP: get_run_by_id
 SELECT
 	id, project_id, parent_run_id, model, status, alias
-	, temperature, persona, context_limit, next_turn, created_at
+	, temperature, persona, context_limit, next_turn, next_loop, created_at
 FROM runs
 WHERE id = :id;
 
@@ -80,11 +80,11 @@ RETURNING next_turn - 1 AS turn;
 
 -- PREP: fork_known_entries
 INSERT INTO known_entries (
-	run_id, turn, path, body, state
+	run_id, loop_id, turn, path, body, status, fidelity
 	, hash, attributes, tokens, tokens_full, refs, write_count
 )
 SELECT
-	:new_run_id, turn, path, body, state
+	:new_run_id, NULL, turn, path, body, status, fidelity
 	, hash, attributes, tokens, tokens_full, refs, write_count
 FROM known_entries
 WHERE run_id = :parent_run_id;
@@ -94,7 +94,7 @@ SELECT r.id
 FROM runs AS r
 WHERE
 	r.project_id = :project_id
-	AND r.status IN ('queued', 'running', 'proposed');
+	AND r.status IN (100, 102, 202);
 
 -- PREP: get_latest_run
 SELECT r.id
@@ -110,5 +110,5 @@ WHERE r.project_id = :project_id;
 
 -- PREP: abort_stuck_runs
 UPDATE runs
-SET status = 'aborted'
-WHERE status IN ('running', 'queued');
+SET status = 499
+WHERE status IN (100, 102);

package/src/agent/schemes.sql

@@ -1,3 +1,6 @@
 -- PREP: upsert_scheme
-INSERT OR REPLACE INTO schemes (name, fidelity, model_visible, valid_states, category)
-VALUES (:name, :fidelity, :model_visible, :valid_states, :category);
+INSERT OR REPLACE INTO schemes (name, model_visible, category)
+VALUES (:name, :model_visible, :category);
+
+-- PREP: get_all_schemes
+SELECT name, model_visible, category FROM schemes;

package/src/agent/tokens.js

@@ -1,28 +1,13 @@
 /**
- * Token counting with tiktoken (o200k_base) and simple fallback.
- * o200k_base is the tokenizer for GPT-4o and newer OpenAI models.
- * Better multilingual and code handling than cl100k_base.
- * Exact counts vary by model tokenizer — these are for budgeting, not billing.
+ * Token estimation. Conservative character-based approximation.
+ * RUMMY_TOKEN_DIVISOR controls characters per token.
+ * No external dependencies. The budget contract is exact.
+ * contextSize is the ceiling. countTokens is the measurement.
  */
 
-let encoder = null;
-
-try {
-	const tiktoken = await import("tiktoken");
-	encoder = tiktoken.get_encoding("o200k_base");
-} catch {
-	// tiktoken unavailable — use character-based estimate
-}
+const DIVISOR = Number(process.env.RUMMY_TOKEN_DIVISOR);
 
 export function countTokens(text) {
 	if (!text) return 0;
-	if (encoder) {
-		try {
-			const tokens = encoder.encode(text);
-			return tokens.length;
-		} catch {
-			// Fallback on encoding error
-		}
-	}
-	return Math.ceil(text.length / 4);
+	return Math.ceil(text.length / DIVISOR);
 }

package/src/agent/turns.sql

@@ -1,12 +1,14 @@
 -- PREP: create_turn
-INSERT INTO turns (run_id, sequence)
-VALUES (:run_id, :sequence)
+INSERT INTO turns (run_id, loop_id, sequence)
+VALUES (:run_id, :loop_id, :sequence)
 RETURNING id, sequence;
 
 -- PREP: update_turn_stats
 UPDATE turns
 SET
-	prompt_tokens = :prompt_tokens
+	context_tokens = :context_tokens
+	, reasoning_content = :reasoning_content
+	, prompt_tokens = :prompt_tokens
 	, cached_tokens = :cached_tokens
 	, completion_tokens = :completion_tokens
 	, reasoning_tokens = :reasoning_tokens
@@ -25,8 +27,15 @@ SELECT
 FROM turns
 WHERE run_id = :run_id;
 
+-- PREP: get_last_context_tokens
+SELECT context_tokens
+FROM turns
+WHERE run_id = :run_id AND context_tokens > 0
+ORDER BY sequence DESC
+LIMIT 1;
+
 -- PREP: get_run_log
-SELECT ke.path, ke.state AS status, ke.body, ke.attributes
+SELECT ke.path, ke.status, ke.body, ke.attributes
 FROM known_entries AS ke
 JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
 WHERE

package/src/hooks/Hooks.js

@@ -39,6 +39,7 @@ export default function createHooks(debug = false) {
 			},
 		},
 		run: {
+			created: createEvent("run.created"),
 			started: createEvent("run.started"),
 			progress: createEvent("run.progress"),
 			state: createEvent("run.state"),
@@ -47,10 +48,15 @@ export default function createHooks(debug = false) {
 				completed: createEvent("run.step.completed"),
 			},
 		},
+		loop: {
+			started: createEvent("loop.started"),
+			completed: createEvent("loop.completed"),
+		},
 		turn: {
 			started: createEvent("turn.started"),
 			response: createEvent("turn.response"),
 			proposing: createEvent("turn.proposing"),
+			completed: createEvent("turn.completed"),
 		},
 		assembly: {
 			system: createFilter("assembly.system"),
@@ -67,6 +73,10 @@ export default function createHooks(debug = false) {
 			started: createEvent("act.started"),
 			completed: createEvent("act.completed"),
 		},
+		panic: {
+			started: createEvent("panic.started"),
+			completed: createEvent("panic.completed"),
+		},
 		llm: {
 			request: {
 				started: createEvent("llm.request.started"),
@@ -80,9 +90,17 @@ export default function createHooks(debug = false) {
 			tools: createFilter("prompt.tools"),
 		},
 		entry: {
+			recording: createFilter("entry.recording"),
 			created: createEvent("entry.created"),
 			changed: createEvent("entry.changed"),
 		},
+		tool: {
+			before: createEvent("tool.before"),
+			after: createEvent("tool.after"),
+		},
+		context: {
+			materialized: createEvent("context.materialized"),
+		},
 		action: {},
 		ui: {
 			render: createEvent("ui.render"),

package/src/hooks/PluginContext.js

@@ -48,22 +48,27 @@ export default class PluginContext {
 		return this.#schemes;
 	}
 
-	registerScheme({
-		name,
-		fidelity = "full",
-		modelVisible = 1,
-		validStates = ["full", "proposed", "pass", "rejected", "error"],
-		category = "result",
-	} = {}) {
+	registerScheme({ name, modelVisible = 1, category = "logging" } = {}) {
+		if (!PluginContext.CATEGORIES.has(category)) {
+			throw new Error(
+				`Invalid category "${category}". Must be one of: ${[...PluginContext.CATEGORIES].join(", ")}`,
+			);
+		}
 		this.#schemes.push({
 			name: name || this.#name,
-			fidelity,
 			model_visible: modelVisible,
-			valid_states: JSON.stringify(validStates),
 			category,
 		});
 	}
 
+	static CATEGORIES = Object.freeze(
+		new Set(["data", "logging", "unknown", "prompt"]),
+	);
+
+	ensureTool() {
+		this.#hooks.tools.ensureTool(this.#name);
+	}
+
 	/**
 	 * Register a named callback for this plugin.
 	 * "handler" registers the tool handler.
@@ -78,7 +83,6 @@ export default class PluginContext {
 			return;
 		}
 		if (event === "full" || event === "summary") {
-			this.#hooks.tools.ensureTool(this.#name);
 			this.#hooks.tools.onView(this.#name, callback, event);
 			return;
 		}