@possumtech/rummy 0.2.8 → 0.3.1

package/src/agent/XmlParser.js

@@ -1,18 +1,10 @@
 import { Parser } from "htmlparser2";
 import { parseEditContent } from "../plugins/hedberg/edits.js";
-import { normalizeAttrs } from "../plugins/hedberg/normalize.js";
+import { normalizeAttrs, parseJsonEdit } from "../plugins/hedberg/normalize.js";
 import { parseSed } from "../plugins/hedberg/sed.js";
 
-const STORE_TOOLS = new Set([
-	"get",
-	"store",
-	"rm",
-	"set",
-	"mv",
-	"cp",
-	"search",
-]);
-const ALL_TOOLS = new Set([
+const STORE_TOOLS = new Set(["get", "rm", "set", "mv", "cp", "search"]);
+export const ALL_TOOLS = new Set([
 	...STORE_TOOLS,
 	"known",
 	"sh",
@@ -21,6 +13,9 @@ const ALL_TOOLS = new Set([
 	"summarize",
 	"update",
 	"unknown",
+	"think",
+	"thought",
+	"mcp",
 ]);
 
 /**
@@ -51,31 +46,10 @@ function resolveCommand(name, attrs, rawBody) {
 				};
 			}
 		}
-		// JSON-style { search, replace } — accept valid JSON and =style variants
-		if (trimmed.startsWith("{") && /search/.test(trimmed)) {
-			let search = null;
-			let replace = null;
-			try {
-				const json = JSON.parse(trimmed);
-				search = json.search;
-				replace = json.replace ?? "";
-			} catch {
-				// Try = style: { search="old", replace="new" }
-				const searchMatch = trimmed.match(/search\s*=\s*"([^"]*)"/);
-				const replaceMatch = trimmed.match(/replace\s*=\s*"([^"]*)"/);
-				if (searchMatch) {
-					search = searchMatch[1];
-					replace = replaceMatch?.[1] ?? "";
-				}
-			}
-			if (search != null) {
-				return {
-					name,
-					path: a.path,
-					search,
-					replace,
-				};
-			}
+		// JSON-style { search, replace }
+		const jsonEdit = parseJsonEdit(trimmed);
+		if (jsonEdit) {
+			return { name, path: a.path, ...jsonEdit };
 		}
 		// Sed syntax: s/search/replace/flags — supports chained commands
 		if (trimmed.startsWith("s/")) {
@@ -116,9 +90,9 @@ function resolveCommand(name, attrs, rawBody) {
 				preview: a.preview,
 			};
 		}
-		// Plain write → create/overwrite
+		// Plain write or fidelity change
 		const body = trimmed || a.body || "";
-		return { name, path: a.path, body, preview: a.preview };
+		return { name, ...a, body };
 	}
 
 	if (name === "summarize" || name === "update" || name === "unknown") {
@@ -129,10 +103,10 @@ function resolveCommand(name, attrs, rawBody) {
 	if (name === "known") {
 		const body = trimmed || a.body || "";
 		const path = a.path || null;
-		return { name, path, body };
+		return { name, ...a, path, body };
 	}
 
-	if (name === "get" || name === "store" || name === "rm") {
+	if (name === "get" || name === "rm") {
 		const path = a.path || trimmed || null;
 		return { name, path, body: a.body, preview: a.preview };
 	}
@@ -174,6 +148,9 @@ export default class XmlParser {
 	static parse(content) {
 		if (!content) return { commands: [], warnings: [], unparsed: "" };
 
+		// Normalize native tool call formats to rummy XML
+		const normalized = XmlParser.#normalizeToolCalls(content);
+
 		const commands = [];
 		const warnings = [];
 		const textChunks = [];
@@ -248,7 +225,7 @@ export default class XmlParser {
 			},
 		);
 
-		parser.write(content);
+		parser.write(normalized);
 		ended = true;
 		parser.end();
 
@@ -264,4 +241,63 @@ export default class XmlParser {
 		const unparsed = textChunks.join("").trim();
 		return { commands, warnings, unparsed };
 	}
+
+	/**
+	 * Normalize native tool call formats to rummy XML.
+	 * Models sometimes emit their training-format tool calls instead of
+	 * our XML tags. The intent is unambiguous — translate silently.
+	 */
+	static #normalizeToolCalls(content) {
+		// Gemma: ```tool_code\n<xml>...\n``` — strip code fences around valid XML
+		let result = content.replace(
+			/```(?:tool_code|tool_command|xml)\n([\s\S]*?)```/g,
+			(_, inner) => inner.trim(),
+		);
+
+		// Qwen/gemma: <|tool_call>call:NAME{key:"value"}<tool_call|>
+		result = result.replace(
+			/<\|tool_call>call:(\w+)\{([^}]*)\}<(?:tool_call\||\|tool_call)>/g,
+			(_, name, params) => {
+				if (!ALL_TOOLS.has(name)) return _;
+				const valueMatch = params.match(/["']([^"']+)["']/);
+				const body = valueMatch?.[1] || "";
+				return `<${name}>${body}</${name}>`;
+			},
+		);
+
+		// OpenAI function_call JSON: {"name":"search","arguments":{"query":"..."}}
+		result = result.replace(
+			/\{"name"\s*:\s*"(\w+)"\s*,\s*"arguments"\s*:\s*\{([^}]*)\}\}/g,
+			(_, name, args) => {
+				if (!ALL_TOOLS.has(name)) return _;
+				const pairs = [...args.matchAll(/"(\w+)"\s*:\s*"([^"]*)"/g)];
+				const body = pairs[0]?.[2] || "";
+				return `<${name}>${body}</${name}>`;
+			},
+		);
+
+		// Anthropic: <tool_use><name>search</name><input>{"query":"..."}</input></tool_use>
+		result = result.replace(
+			/<tool_use>\s*<name>(\w+)<\/name>\s*<input>\{([^}]*)\}<\/input>\s*<\/tool_use>/g,
+			(_, name, args) => {
+				if (!ALL_TOOLS.has(name)) return _;
+				const pairs = [...args.matchAll(/"(\w+)"\s*:\s*"([^"]*)"/g)];
+				const body = pairs[0]?.[2] || "";
+				return `<${name}>${body}</${name}>`;
+			},
+		);
+
+		// Mistral: [TOOL_CALLS] [{"name":"search","arguments":{"query":"..."}}]
+		result = result.replace(
+			/\[TOOL_CALLS\]\s*\[\{"name"\s*:\s*"(\w+)"\s*,\s*"arguments"\s*:\s*\{([^}]*)\}\}\]/g,
+			(_, name, args) => {
+				if (!ALL_TOOLS.has(name)) return _;
+				const pairs = [...args.matchAll(/"(\w+)"\s*:\s*"([^"]*)"/g)];
+				const body = pairs[0]?.[2] || "";
+				return `<${name}>${body}</${name}>`;
+			},
+		);
+
+		return result;
+	}
 }

package/src/agent/known_store.sql

@@ -56,7 +56,15 @@ UPDATE known_entries
 SET
 	fidelity = :fidelity
 	, tokens = CASE
-		WHEN :fidelity = 'summary' THEN countTokens(body)
+		WHEN :fidelity = 'archive'
+			THEN 0
+		WHEN :fidelity = 'index'
+			THEN 0
+		WHEN :fidelity = 'summary'
+			THEN COALESCE(
+				countTokens(json_extract(attributes, '$.summary')),
+				countTokens(substr(body, 1, 80))
+			)
 		ELSE tokens_full
 	END
 	, updated_at = CURRENT_TIMESTAMP
@@ -74,7 +82,7 @@ WHERE run_id = :run_id AND path = :path;
 -- PREP: demote_path
 UPDATE known_entries
 SET
-	fidelity = 'stored'
+	fidelity = 'archive'
 	, tokens = 0
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
@@ -84,7 +92,15 @@ UPDATE known_entries
 SET
 	fidelity = :fidelity
 	, tokens = CASE
-		WHEN :fidelity = 'stored' THEN 0
+		WHEN :fidelity = 'archive'
+			THEN 0
+		WHEN :fidelity = 'index'
+			THEN 0
+		WHEN :fidelity = 'summary'
+			THEN COALESCE(
+				countTokens(json_extract(attributes, '$.summary')),
+				countTokens(substr(body, 1, 80))
+			)
 		ELSE countTokens(body)
 	END
 	, updated_at = CURRENT_TIMESTAMP
@@ -133,7 +149,7 @@ WHERE
 -- PREP: demote_by_pattern
 UPDATE known_entries
 SET
-	fidelity = 'stored'
+	fidelity = 'archive'
 	, tokens = 0
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
@@ -173,3 +189,51 @@ WHERE
 	run_id = :run_id
 	AND hedmatch(:path, path)
 	AND (:body IS NULL OR hedsearch(:body, body));
+
+-- PREP: restore_summarized_prompts
+-- Restore prompt entries demoted to summary by a recovery phase that was
+-- interrupted (e.g. server crash). Safe to call unconditionally at loop
+-- start: if the full prompt would overflow, Prompt Demotion handles it.
+UPDATE known_entries
+SET
+	fidelity = 'full'
+	, tokens = tokens_full
+	, updated_at = CURRENT_TIMESTAMP
+WHERE run_id = :run_id AND scheme = 'prompt' AND fidelity = 'summary';
+
+-- PREP: demote_previous_loop_logging
+-- Demote full logging entries from all other loops to summary.
+-- Fires at loop start so <previous> entries are already compact.
+UPDATE known_entries
+SET
+	fidelity = 'summary'
+	, tokens = COALESCE(
+		countTokens(json_extract(attributes, '$.summary'))
+		, countTokens(substr(body, 1, 80))
+	)
+	, updated_at = CURRENT_TIMESTAMP
+WHERE
+	run_id = :run_id
+	AND (loop_id IS NULL OR loop_id != :loop_id)
+	AND fidelity = 'full'
+	AND scheme IN (SELECT name FROM schemes WHERE category = 'logging');
+
+-- PREP: demote_turn_data_entries
+-- Demote full data entries from a turn to summary with 413 status.
+-- Fires when end-of-turn materialization exceeds the context ceiling.
+UPDATE known_entries
+SET
+	fidelity = 'summary'
+	, status = 413
+	, tokens = COALESCE(
+		countTokens(json_extract(attributes, '$.summary'))
+		, countTokens(substr(body, 1, 80))
+	)
+	, updated_at = CURRENT_TIMESTAMP
+WHERE
+	run_id = :run_id
+	AND turn = :turn
+	AND fidelity = 'full'
+	AND status < 400
+	AND scheme IN (SELECT name FROM schemes WHERE category = 'data')
+RETURNING path;

package/src/agent/schemes.sql

@@ -1,3 +1,6 @@
 -- PREP: upsert_scheme
 INSERT OR REPLACE INTO schemes (name, model_visible, category)
 VALUES (:name, :model_visible, :category);
+
+-- PREP: get_all_schemes
+SELECT name, model_visible, category FROM schemes;

package/src/agent/tokens.js

@@ -1,28 +1,14 @@
 /**
- * Token counting with tiktoken (o200k_base) and simple fallback.
- * o200k_base is the tokenizer for GPT-4o and newer OpenAI models.
- * Better multilingual and code handling than cl100k_base.
- * Exact counts vary by model tokenizer — these are for budgeting, not billing.
+ * Token estimation. Conservative character-based approximation.
+ * RUMMY_TOKEN_DIVISOR controls characters per token.
+ * No external dependencies. The budget contract is exact.
+ * contextSize is the ceiling. countTokens is the measurement.
  */
 
-let encoder = null;
-
-try {
-	const tiktoken = await import("tiktoken");
-	encoder = tiktoken.get_encoding("o200k_base");
-} catch {
-	// tiktoken unavailable — use character-based estimate
-}
+const DIVISOR = Number(process.env.RUMMY_TOKEN_DIVISOR);
+if (!DIVISOR) throw new Error("RUMMY_TOKEN_DIVISOR must be a non-zero number");
 
 export function countTokens(text) {
 	if (!text) return 0;
-	if (encoder) {
-		try {
-			const tokens = encoder.encode(text);
-			return tokens.length;
-		} catch {
-			// Fallback on encoding error
-		}
-	}
-	return Math.ceil(text.length / 4);
+	return Math.ceil(text.length / DIVISOR);
 }

package/src/agent/turns.sql

@@ -6,7 +6,9 @@ RETURNING id, sequence;
 -- PREP: update_turn_stats
 UPDATE turns
 SET
-	prompt_tokens = :prompt_tokens
+	context_tokens = :context_tokens
+	, reasoning_content = :reasoning_content
+	, prompt_tokens = :prompt_tokens
 	, cached_tokens = :cached_tokens
 	, completion_tokens = :completion_tokens
 	, reasoning_tokens = :reasoning_tokens
@@ -25,6 +27,18 @@ SELECT
 FROM turns
 WHERE run_id = :run_id;
 
+-- PREP: get_turn_context_tokens
+SELECT context_tokens
+FROM turns
+WHERE run_id = :run_id AND sequence = :sequence;
+
+-- PREP: get_last_context_tokens
+SELECT context_tokens
+FROM turns
+WHERE run_id = :run_id AND context_tokens > 0
+ORDER BY sequence DESC
+LIMIT 1;
+
 -- PREP: get_run_log
 SELECT ke.path, ke.status, ke.body, ke.attributes
 FROM known_entries AS ke

package/src/hooks/HookRegistry.js

@@ -63,6 +63,13 @@ export default class HookRegistry {
 		this.#events.get(tag).sort((a, b) => a.priority - b.priority);
 	}
 
+	removeEvent(tag, callback) {
+		const hooks = this.#events.get(tag);
+		if (!hooks) return;
+		const idx = hooks.findIndex((h) => h.callback === callback);
+		if (idx !== -1) hooks.splice(idx, 1);
+	}
+
 	async emitEvent(tag, ...args) {
 		const hooks = this.#events.get(tag) || [];
 		for (const h of hooks) {

package/src/hooks/Hooks.js

@@ -11,6 +11,7 @@ export default function createHooks(debug = false) {
 
 	const createEvent = (tag) => ({
 		on: (callback, priority) => registry.addEvent(tag, callback, priority),
+		off: (callback) => registry.removeEvent(tag, callback),
 		emit: (...args) => registry.emitEvent(tag, ...args),
 	});
 
@@ -39,6 +40,7 @@ export default function createHooks(debug = false) {
 			},
 		},
 		run: {
+			created: createEvent("run.created"),
 			started: createEvent("run.started"),
 			progress: createEvent("run.progress"),
 			state: createEvent("run.state"),
@@ -47,10 +49,15 @@ export default function createHooks(debug = false) {
 				completed: createEvent("run.step.completed"),
 			},
 		},
+		loop: {
+			started: createEvent("loop.started"),
+			completed: createEvent("loop.completed"),
+		},
 		turn: {
 			started: createEvent("turn.started"),
 			response: createEvent("turn.response"),
 			proposing: createEvent("turn.proposing"),
+			completed: createEvent("turn.completed"),
 		},
 		assembly: {
 			system: createFilter("assembly.system"),
@@ -80,9 +87,17 @@ export default function createHooks(debug = false) {
 			tools: createFilter("prompt.tools"),
 		},
 		entry: {
+			recording: createFilter("entry.recording"),
 			created: createEvent("entry.created"),
 			changed: createEvent("entry.changed"),
 		},
+		tool: {
+			before: createEvent("tool.before"),
+			after: createEvent("tool.after"),
+		},
+		context: {
+			materialized: createEvent("context.materialized"),
+		},
 		action: {},
 		ui: {
 			render: createEvent("ui.render"),

package/src/hooks/PluginContext.js

@@ -48,7 +48,12 @@ export default class PluginContext {
 		return this.#schemes;
 	}
 
-	registerScheme({ name, modelVisible = 1, category = "result" } = {}) {
+	registerScheme({ name, modelVisible = 1, category = "logging" } = {}) {
+		if (!PluginContext.CATEGORIES.has(category)) {
+			throw new Error(
+				`Invalid category "${category}". Must be one of: ${[...PluginContext.CATEGORIES].join(", ")}`,
+			);
+		}
 		this.#schemes.push({
 			name: name || this.#name,
 			model_visible: modelVisible,
@@ -56,6 +61,14 @@ export default class PluginContext {
 		});
 	}
 
+	static CATEGORIES = Object.freeze(
+		new Set(["data", "logging", "unknown", "prompt"]),
+	);
+
+	ensureTool() {
+		this.#hooks.tools.ensureTool(this.#name);
+	}
+
 	/**
 	 * Register a named callback for this plugin.
 	 * "handler" registers the tool handler.

package/src/hooks/RummyContext.js

@@ -55,8 +55,20 @@ export default class RummyContext {
 		return this.#context.loopId || null;
 	}
 
-	get noContext() {
-		return this.#context.noContext === true;
+	get noRepo() {
+		return this.#context.noRepo === true;
+	}
+
+	get noInteraction() {
+		return this.#context.noInteraction === true;
+	}
+
+	get noWeb() {
+		return this.#context.noWeb === true;
+	}
+
+	get toolSet() {
+		return this.#context.toolSet || null;
 	}
 
 	get contextSize() {
@@ -89,7 +101,7 @@ export default class RummyContext {
 
 	// --- Tool methods (same operations the model uses) ---
 
-	async set({ path, body, status = 200, attributes } = {}) {
+	async set({ path, body, status = 200, fidelity, attributes } = {}) {
 		if (!path) {
 			const slugify = (await import("../sql/functions/slugify.js")).default;
 			const base = slugify(body || "");
@@ -101,7 +113,7 @@ export default class RummyContext {
 			path,
 			body || "",
 			status,
-			{ attributes, loopId: this.loopId },
+			{ fidelity, attributes, loopId: this.loopId },
 		);
 		return path;
 	}

package/src/hooks/ToolRegistry.js

@@ -1,3 +1,32 @@
+// Tool display order: gather → reason → act → communicate.
+// Position in the list implies priority to the model.
+const TOOL_ORDER = [
+	"get",
+	"set",
+	"known",
+	"unknown",
+	"env",
+	"sh",
+	"rm",
+	"cp",
+	"mv",
+	"search",
+	"summarize",
+	"update",
+	"ask_user",
+];
+
+function sortByPriority(names) {
+	return names.toSorted((a, b) => {
+		const ia = TOOL_ORDER.indexOf(a);
+		const ib = TOOL_ORDER.indexOf(b);
+		if (ia === -1 && ib === -1) return a.localeCompare(b);
+		if (ia === -1) return 1;
+		if (ib === -1) return 1;
+		return ia - ib;
+	});
+}
+
 export default class ToolRegistry {
 	#tools = new Map();
 	#handlers = new Map();
@@ -5,12 +34,9 @@ export default class ToolRegistry {
 
 	ensureTool(scheme) {
 		if (this.#tools.has(scheme)) return;
-		this.#tools.set(scheme, Object.freeze({ modes: new Set(["ask", "act"]) }));
+		this.#tools.set(scheme, Object.freeze({}));
 	}
 
-	// Exception: old register() removed. Plugins use core.on("handler")/core.on("full").
-	// The only remaining caller pathway is ensureTool + onHandle + onView.
-
 	get(name) {
 		return this.#tools.get(name);
 	}
@@ -39,10 +65,33 @@ export default class ToolRegistry {
 					`Every tool must define how its entries appear in the model view.`,
 			);
 		}
+
+		const attrs =
+			typeof entry.attributes === "string"
+				? JSON.parse(entry.attributes)
+				: entry.attributes;
+		const summary = typeof attrs?.summary === "string" ? attrs.summary : null;
+
 		const fidelity = entry.fidelity || "full";
 		const fn = fidelityMap.get(fidelity);
-		if (!fn) return "";
-		return await fn(entry);
+		if (!fn) {
+			// No view for this fidelity — fall back on model-authored summary
+			return summary || "";
+		}
+
+		const body = await fn(entry);
+
+		// Prepend summary keywords above plugin output at summary fidelity
+		if (fidelity === "summary" && summary && body) {
+			return `${summary}\n${body}`;
+		}
+
+		// Fall back to summary attribute when plugin returns empty
+		if (fidelity === "summary" && summary && !body) {
+			return summary;
+		}
+
+		return body;
 	}
 
 	hasView(scheme) {
@@ -59,22 +108,31 @@ export default class ToolRegistry {
 		}
 	}
 
-	get actTools() {
-		return new Set(
-			[...this.#tools.entries()]
-				.filter(([, def]) => def.category === "act")
-				.map(([name]) => name),
-		);
-	}
-
 	get names() {
-		return [...this.#tools.keys()];
+		return sortByPriority([...this.#tools.keys()]);
 	}
 
-	namesForMode(mode) {
-		return [...this.#tools.entries()]
-			.filter(([, def]) => def.modes.has(mode))
-			.map(([name]) => name);
+	/**
+	 * Compute the active tool set for a loop.
+	 * All exclusions — mode, flags — handled here. One mechanism.
+	 */
+	resolveForLoop(
+		mode,
+		{ noInteraction = false, noWeb = false, noProposals = false } = {},
+	) {
+		const excluded = new Set();
+		if (mode === "ask") excluded.add("sh");
+		if (noInteraction) excluded.add("ask_user");
+		if (noWeb) excluded.add("search");
+		if (noProposals) {
+			excluded.add("ask_user");
+			excluded.add("env");
+			excluded.add("sh");
+		}
+		const names = sortByPriority(
+			[...this.#tools.keys()].filter((n) => !excluded.has(n)),
+		);
+		return new Set(names);
 	}
 
 	entries() {

package/src/llm/LlmProvider.js

@@ -90,18 +90,37 @@ export default class LlmProvider {
 	}
 
 	async getContextSize(model) {
+		// DB is the authority — check models table first
+		if (this.#db) {
+			const row = await this.#db.get_model_by_alias.get({ alias: model });
+			if (row?.context_length) return row.context_length;
+		}
+
+		// Fall back to API query
 		const resolvedModel = await this.resolve(model);
+		let size;
 		if (resolvedModel.startsWith("ollama/")) {
 			const localModel = resolvedModel.replace("ollama/", "");
-			return this.#getOllama().getContextSize(localModel);
-		}
-		if (resolvedModel.startsWith("openai/")) {
-			return this.#getOpenAi().getContextSize(resolvedModel);
-		}
-		if (resolvedModel.startsWith("x.ai/")) {
+			size = await this.#getOllama().getContextSize(localModel);
+		} else if (resolvedModel.startsWith("openai/")) {
+			size = await this.#getOpenAi().getContextSize(resolvedModel);
+		} else if (resolvedModel.startsWith("x.ai/")) {
 			const localModel = resolvedModel.replace("x.ai/", "");
-			return this.#getXai().getContextSize(localModel);
+			size = await this.#getXai().getContextSize(localModel);
+		} else {
+			size = await this.#getOpenRouter().getContextSize(resolvedModel);
+		}
+
+		// Cache back to DB for next time
+		if (this.#db && size) {
+			await this.#db.update_model_context_length
+				.run({
+					alias: model,
+					context_length: size,
+				})
+				.catch(() => {});
 		}
-		return this.#getOpenRouter().getContextSize(resolvedModel);
+
+		return size;
 	}
 }

package/src/llm/OpenAiClient.js

@@ -49,6 +49,12 @@ export default class OpenAiClient {
 			);
 			msg.reasoning_content =
 				parts.length > 0 ? [...new Set(parts)].join("\n") : null;
+
+			if (process.env.RUMMY_DEBUG === "true" && msg.reasoning_content) {
+				console.warn(
+					`[RUMMY] Reasoning (${msg.reasoning_content.length} chars): ${msg.reasoning_content.slice(0, 120)}`,
+				);
+			}
 		}
 
 		return data;
@@ -59,6 +65,20 @@ export default class OpenAiClient {
 		const headers = { "Content-Type": "application/json" };
 		if (this.#apiKey) headers.Authorization = `Bearer ${this.#apiKey}`;
 
+		// Try /props first — llama.cpp exposes runtime n_ctx here
+		try {
+			const propsResponse = await fetch(`${this.#baseUrl}/props`, {
+				headers,
+				signal: AbortSignal.timeout(timeout),
+			});
+			if (propsResponse.ok) {
+				const props = await propsResponse.json();
+				const runtimeCtx = props?.default_generation_settings?.n_ctx;
+				if (runtimeCtx) return runtimeCtx;
+			}
+		} catch {}
+
+		// Fall back to /v1/models for training context
 		const response = await fetch(`${this.#baseUrl}/v1/models`, {
 			headers,
 			signal: AbortSignal.timeout(timeout),