@possumtech/rummy 0.2.1

package/src/agent/TurnExecutor.js

@@ -0,0 +1,457 @@
+import RummyContext from "../hooks/RummyContext.js";
+import ContextAssembler from "./ContextAssembler.js";
+import KnownStore from "./KnownStore.js";
+import msg from "./messages.js";
+import ResponseHealer from "./ResponseHealer.js";
+import { countTokens } from "./tokens.js";
+import XmlParser from "./XmlParser.js";
+
+export default class TurnExecutor {
+	#db;
+	#llmProvider;
+	#hooks;
+	#knownStore;
+
+	constructor(db, llmProvider, hooks, knownStore) {
+		this.#db = db;
+		this.#llmProvider = llmProvider;
+		this.#hooks = hooks;
+		this.#knownStore = knownStore;
+	}
+
+	async execute({
+		mode,
+		project,
+		projectId,
+		currentRunId,
+		currentAlias,
+		requestedModel,
+		loopPrompt,
+		noContext,
+		contextSize,
+		options,
+		signal,
+	}) {
+		const turn = await this.#knownStore.nextTurn(currentRunId);
+
+		const turnRow = await this.#db.create_turn.get({
+			run_id: currentRunId,
+			sequence: turn,
+		});
+
+		const unresolved = await this.#knownStore.getUnresolved(currentRunId);
+		if (unresolved.length > 0) {
+			throw new Error(
+				msg("error.unresolved_proposed", { count: unresolved.length }),
+			);
+		}
+
+		// Build RummyContext before turn.started so plugins can write entries
+		const rummy = new RummyContext(
+			{
+				tag: "turn",
+				attrs: {},
+				content: null,
+				children: [
+					{ tag: "system", attrs: {}, content: null, children: [] },
+					{ tag: "context", attrs: {}, content: null, children: [] },
+					{ tag: "user", attrs: {}, content: null, children: [] },
+					{ tag: "assistant", attrs: {}, content: null, children: [] },
+				],
+			},
+			{
+				hooks: this.#hooks,
+				db: this.#db,
+				store: this.#knownStore,
+				project,
+				type: mode,
+				sequence: turn,
+				runId: currentRunId,
+				turnId: turnRow.id,
+				noContext,
+				contextSize,
+				systemPrompt: null,
+				loopPrompt,
+			},
+		);
+		// Plugins write prompt/progress/instructions entries
+		await this.#hooks.turn.started.emit({
+			rummy,
+			mode,
+			prompt: loopPrompt,
+			isContinuation: options?.isContinuation,
+		});
+
+		await this.#hooks.processTurn(rummy);
+
+		// Project instructions://system through the instructions tool's projection
+		const instrEntry = await this.#knownStore.getEntriesByPattern(
+			currentRunId,
+			"instructions://system",
+			null,
+		);
+		const instrAttrs = instrEntry[0]
+			? await this.#knownStore.getAttributes(
+					currentRunId,
+					"instructions://system",
+				)
+			: null;
+		const systemPrompt = await this.#hooks.tools.view("instructions", {
+			path: "instructions://system",
+			scheme: "instructions",
+			body: instrEntry[0]?.body || "",
+			attributes: instrAttrs,
+			fidelity: "full",
+			category: "system",
+		});
+
+		// Materialize turn_context: VIEW rows projected through tools
+		await this.#db.clear_turn_context.run({ run_id: currentRunId, turn });
+		const viewRows = await this.#db.get_model_context.all({
+			run_id: currentRunId,
+		});
+		for (const row of viewRows) {
+			const scheme = row.scheme || "file";
+			const projectedBody = await this.#hooks.tools.view(scheme, {
+				path: row.path,
+				scheme,
+				body: row.body,
+				attributes: row.attributes ? JSON.parse(row.attributes) : null,
+				fidelity: row.fidelity,
+				category: row.category,
+			});
+
+			await this.#db.insert_turn_context.run({
+				run_id: currentRunId,
+				turn,
+				ordinal: row.ordinal,
+				path: row.path,
+				fidelity: row.fidelity,
+				state: row.state,
+				body: projectedBody ?? "",
+				tokens: countTokens(projectedBody ?? ""),
+				attributes: row.attributes,
+				category: row.category,
+				source_turn: row.turn,
+			});
+		}
+
+		await this.#hooks.run.progress.emit({
+			projectId,
+			run: currentAlias,
+			turn,
+			status: "thinking",
+		});
+
+		// Assemble messages from projected system prompt + materialized turn_context
+		const rows = await this.#db.get_turn_context.all({
+			run_id: currentRunId,
+			turn,
+		});
+		const messages = await ContextAssembler.assembleFromTurnContext(
+			rows,
+			{
+				type: mode,
+				systemPrompt,
+				contextSize,
+			},
+			this.#hooks,
+		);
+
+		const filteredMessages = await this.#hooks.llm.messages.filter(messages, {
+			model: requestedModel,
+			projectId,
+			runId: currentRunId,
+		});
+
+		// Store assembled messages as audit
+		// Call LLM
+		await this.#hooks.llm.request.started.emit({ model: requestedModel, turn });
+		const rawResult = await this.#llmProvider.completion(
+			filteredMessages,
+			requestedModel,
+			{ temperature: options?.temperature, signal },
+		);
+		const result = await this.#hooks.llm.response.filter(rawResult, {
+			model: requestedModel,
+			projectId,
+			runId: currentRunId,
+		});
+		await this.#hooks.llm.request.completed.emit({
+			model: requestedModel,
+			turn,
+			usage: result.usage,
+		});
+		const responseMessage = result.choices?.[0]?.message;
+		const content = responseMessage?.content || "";
+
+		await this.#hooks.run.progress.emit({
+			projectId,
+			run: currentAlias,
+			turn,
+			status: "processing",
+		});
+
+		// Parse and emit — plugins handle audit storage
+		const { commands, unparsed } = XmlParser.parse(content);
+
+		const systemMsg = filteredMessages.find((m) => m.role === "system");
+		const userMsg = filteredMessages.find((m) => m.role === "user");
+		await this.#hooks.turn.response.emit({
+			rummy,
+			turn,
+			result,
+			responseMessage,
+			content,
+			commands,
+			unparsed,
+			systemMsg: systemMsg?.content,
+			userMsg: userMsg?.content,
+		});
+
+		// --- PHASE 1: RECORD ---
+		// Every command becomes an entry. No execution yet.
+
+		const recorded = [];
+		let summaryText = null;
+		let updateText = null;
+
+		for (const cmd of commands) {
+			const entry = await this.#record(currentRunId, turn, mode, cmd);
+			if (!entry) continue;
+
+			if (entry.scheme === "summarize") summaryText = entry.body;
+			else if (entry.scheme === "update") updateText = entry.body;
+			else recorded.push(entry);
+		}
+
+		// If model sent both, summary wins
+		if (summaryText && updateText) updateText = null;
+
+		// If model sent neither, heal from content
+		let statusHealed = false;
+		if (!summaryText && !updateText) {
+			const healed = ResponseHealer.healStatus(content, commands);
+			summaryText = healed.summaryText;
+			updateText = healed.updateText;
+			statusHealed = true;
+		}
+
+		// Record healed status
+		if (summaryText) {
+			const summaryPath = await this.#knownStore.slugPath(
+				currentRunId,
+				"summarize",
+				summaryText,
+			);
+			await this.#knownStore.upsert(
+				currentRunId,
+				turn,
+				summaryPath,
+				summaryText,
+				"summary",
+			);
+		} else if (updateText) {
+			const updatePath = await this.#knownStore.slugPath(
+				currentRunId,
+				"update",
+				updateText,
+			);
+			await this.#knownStore.upsert(
+				currentRunId,
+				turn,
+				updatePath,
+				updateText,
+				"info",
+			);
+		}
+
+		// --- PHASE 2: DISPATCH ---
+		// Handlers perform side effects: promote, demote, patch, propose.
+
+		let hasErrors = false;
+		for (const entry of recorded) {
+			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
+			await this.#hooks.entry.created.emit(entry);
+		}
+
+		// Materialize proposals (e.g. file plugin applies accumulated revisions)
+		await this.#hooks.turn.proposing.emit({ rummy, recorded });
+
+		// Check if any dispatched entries ended in error state
+		for (const entry of recorded) {
+			const row = await this.#db.get_entry_state.get({
+				run_id: currentRunId,
+				path: entry.resultPath || entry.path,
+			});
+			if (row?.state === "error") hasErrors = true;
+		}
+
+		// Errors override summarize — the model thinks it's done but it's not
+		if (hasErrors && summaryText) {
+			summaryText = null;
+			updateText = "Tool errors detected — retry or investigate.";
+		}
+
+		// --- Classify for return value ---
+
+		const actionCalls = recorded.filter((e) =>
+			["get", "store", "set", "rm", "mv", "cp", "sh", "env", "search"].includes(
+				e.scheme,
+			),
+		);
+		const writeCalls = recorded.filter(
+			(e) =>
+				e.scheme === "known" ||
+				(e.scheme === "set" && !e.attributes?.blocks && !e.attributes?.search),
+		);
+		const unknownCalls = recorded.filter((e) => e.scheme === "unknown");
+
+		const hasAct = actionCalls.some((c) =>
+			["set", "rm", "sh", "mv", "cp"].includes(c.scheme),
+		);
+		const hasReads = actionCalls.some((c) =>
+			["get", "env", "search"].includes(c.scheme),
+		);
+		const hasWrites = writeCalls.length > 0 || unknownCalls.length > 0;
+		const flags = { hasAct, hasReads, hasWrites };
+
+		const askUserEntry = recorded.find((e) => e.scheme === "ask_user");
+
+		return {
+			turn,
+			turnId: turnRow.id,
+			actionCalls,
+			writeCalls,
+			unknownCalls,
+			summaryText,
+			updateText,
+			statusHealed,
+			askUserCmd: askUserEntry || null,
+			flags,
+			model: result.model || requestedModel,
+			modelAlias: requestedModel,
+			temperature:
+				options?.temperature ??
+				Number.parseFloat(process.env.RUMMY_TEMPERATURE || "0.7"),
+			contextSize,
+			usage: result.usage,
+		};
+	}
+
+	/**
+	 * Record a parsed command as a known_entries row.
+	 * Returns the recorded entry descriptor, or null if rejected/skipped.
+	 */
+	async #record(runId, turn, mode, cmd) {
+		// Mode enforcement — reject prohibited commands in ask mode
+		if (mode === "ask") {
+			if (cmd.name === "sh") {
+				console.warn("[RUMMY] Rejected <sh> in ask mode");
+				return null;
+			}
+			if (cmd.name === "set" && cmd.path) {
+				const scheme = KnownStore.scheme(cmd.path);
+				if (scheme === null) {
+					console.warn(`[RUMMY] Rejected file set to ${cmd.path} in ask mode`);
+					return null;
+				}
+			}
+			if (cmd.name === "rm" && cmd.path) {
+				const scheme = KnownStore.scheme(cmd.path);
+				if (scheme === null) {
+					console.warn(`[RUMMY] Rejected file rm of ${cmd.path} in ask mode`);
+					return null;
+				}
+			}
+			if ((cmd.name === "mv" || cmd.name === "cp") && cmd.to) {
+				const destScheme = KnownStore.scheme(cmd.to);
+				if (destScheme === null) {
+					console.warn(
+						`[RUMMY] Rejected ${cmd.name} to file ${cmd.to} in ask mode`,
+					);
+					return null;
+				}
+			}
+		}
+
+		const scheme = cmd.name;
+
+		// Structural tags — record and return (no handler dispatch)
+		if (scheme === "summarize" || scheme === "update") {
+			return { scheme, body: cmd.body, resultPath: null, attributes: null };
+		}
+
+		// Unknown — deduplicated, sticky
+		if (scheme === "unknown") {
+			const existingValues = await this.#knownStore.getUnknownValues(runId);
+			if (existingValues.has(cmd.body)) return null;
+			const unknownPath = await this.#knownStore.slugPath(
+				runId,
+				"unknown",
+				cmd.body,
+			);
+			await this.#knownStore.upsert(runId, turn, unknownPath, cmd.body, "full");
+			return {
+				scheme,
+				path: unknownPath,
+				body: cmd.body,
+				resultPath: unknownPath,
+				attributes: null,
+			};
+		}
+
+		// Normalize path — encode spaces in scheme:// paths
+		const rawTarget = cmd.path || cmd.command || cmd.question || "";
+		const target = rawTarget.includes("://")
+			? rawTarget.replace(
+					/:\/\/(.*)$/,
+					(_, rest) => `://${encodeURIComponent(decodeURIComponent(rest))}`,
+				)
+			: rawTarget;
+		const resultPath = await this.#knownStore.dedup(runId, scheme, target);
+
+		// Pass parsed command fields through as attributes
+		const { name: _, ...attributes } = cmd;
+		if (cmd.path) attributes.path = target;
+
+		// known tool or naked write → known:// slug from body
+		if (scheme === "known" || (scheme === "set" && !cmd.path)) {
+			if (!cmd.body) return null;
+			const knownPath =
+				cmd.path || (await this.#knownStore.slugPath(runId, "known", cmd.body));
+			await this.#knownStore.upsert(runId, turn, knownPath, cmd.body, "full");
+			return {
+				scheme: "known",
+				path: knownPath,
+				body: cmd.body,
+				resultPath: knownPath,
+				attributes,
+			};
+		}
+
+		// Record the entry
+		const body = cmd.body || cmd.command || cmd.question || "";
+		const state = this.#initialState(scheme);
+		await this.#knownStore.upsert(runId, turn, resultPath, body, state, {
+			attributes,
+		});
+
+		return {
+			scheme,
+			path: resultPath,
+			body,
+			attributes,
+			state,
+			resultPath,
+		};
+	}
+
+	/**
+	 * Initial state for a recorded command entry.
+	 * All entries start at "full". Handlers change state during dispatch.
+	 */
+	#initialState(_scheme) {
+		return "full";
+	}
+}

package/src/agent/XmlParser.js

@@ -0,0 +1,247 @@
+import { Parser } from "htmlparser2";
+import { parseEditContent } from "../plugins/hedberg/edits.js";
+import { normalizeAttrs } from "../plugins/hedberg/normalize.js";
+import { parseSed } from "../plugins/hedberg/sed.js";
+
+const STORE_TOOLS = new Set([
+	"get",
+	"store",
+	"rm",
+	"set",
+	"mv",
+	"cp",
+	"search",
+]);
+const ALL_TOOLS = new Set([
+	...STORE_TOOLS,
+	"known",
+	"sh",
+	"env",
+	"ask_user",
+	"summarize",
+	"update",
+	"unknown",
+]);
+
+/**
+ * Resolve the competing attr-vs-body philosophies per tool.
+ * If the canonical attribute is missing, the body fills it. Silent.
+ */
+function resolveCommand(name, attrs, rawBody) {
+	const a = normalizeAttrs(attrs);
+	const trimmed = rawBody.trim();
+
+	if (name === "set") {
+		// Structured edit detection — merge conflict, udiff, Claude XML
+		const hasEdit =
+			/<{3,12} SEARCH/.test(trimmed) ||
+			/>{3,12} REPLACE/.test(trimmed) ||
+			(trimmed.includes("@@") &&
+				(trimmed.includes("\n-") || trimmed.includes("\n+"))) ||
+			trimmed.includes("<old_text>");
+		if (hasEdit) {
+			const blocks = parseEditContent(rawBody);
+			if (blocks.length > 0) {
+				return {
+					name,
+					path: a.path,
+					body: a.body,
+					preview: a.preview,
+					blocks,
+				};
+			}
+		}
+		// JSON-style { search, replace } — accept valid JSON and =style variants
+		if (trimmed.startsWith("{") && /search/.test(trimmed)) {
+			let search = null;
+			let replace = null;
+			try {
+				const json = JSON.parse(trimmed);
+				search = json.search;
+				replace = json.replace ?? "";
+			} catch {
+				// Try = style: { search="old", replace="new" }
+				const searchMatch = trimmed.match(/search\s*=\s*"([^"]*)"/);
+				const replaceMatch = trimmed.match(/replace\s*=\s*"([^"]*)"/);
+				if (searchMatch) {
+					search = searchMatch[1];
+					replace = replaceMatch?.[1] ?? "";
+				}
+			}
+			if (search != null) {
+				return {
+					name,
+					path: a.path,
+					search,
+					replace,
+				};
+			}
+		}
+		// Sed syntax: s/search/replace/flags — supports chained commands
+		if (trimmed.startsWith("s/")) {
+			const blocks = parseSed(trimmed);
+			if (blocks?.length === 1) {
+				return {
+					name,
+					path: a.path,
+					search: blocks[0].search,
+					replace: blocks[0].replace,
+					flags: blocks[0].flags,
+					sed: true,
+				};
+			}
+			if (blocks?.length > 1) {
+				return { name, path: a.path, blocks };
+			}
+		}
+		// search+replace attrs → attribute edit mode
+		if (a.search) {
+			const replace = a.replace ?? trimmed;
+			return {
+				name,
+				path: a.path,
+				body: a.body,
+				preview: a.preview,
+				search: a.search,
+				replace,
+			};
+		}
+		// Body attr + body content → search/replace (attr is search, body is replace)
+		if (trimmed && a.body) {
+			return {
+				name,
+				path: a.path,
+				search: a.body,
+				replace: trimmed,
+				preview: a.preview,
+			};
+		}
+		// Plain write → create/overwrite
+		const body = trimmed || a.body || "";
+		return { name, path: a.path, body, preview: a.preview };
+	}
+
+	if (name === "summarize" || name === "update" || name === "unknown") {
+		const body = trimmed || a.body || "";
+		return { name, body };
+	}
+
+	if (name === "known") {
+		const body = trimmed || a.body || "";
+		const path = a.path || null;
+		return { name, path, body };
+	}
+
+	if (name === "get" || name === "store" || name === "rm") {
+		const path = a.path || trimmed || null;
+		return { name, path, body: a.body, preview: a.preview };
+	}
+
+	if (name === "search") {
+		const path = a.path || trimmed || null;
+		const results = a.results ? Number(a.results) : null;
+		return { name, path, results };
+	}
+
+	if (name === "mv" || name === "cp") {
+		const to = a.to || trimmed || null;
+		return { name, path: a.path, to };
+	}
+
+	if (name === "sh" || name === "env") {
+		const command = a.command || trimmed || null;
+		return { name, command };
+	}
+
+	if (name === "ask_user") {
+		const question = a.question || null;
+		const options = a.options || trimmed || null;
+		return { name, question, options };
+	}
+
+	return { name, ...a, body: trimmed || a.body };
+}
+
+export default class XmlParser {
+	/**
+	 * Parse tool commands from model content using htmlparser2.
+	 * Handles malformed XML gracefully — unclosed tags, missing slashes, etc.
+	 * Every tool can appear as self-closing (attrs only) or with body content.
+	 * Competing attr-vs-body philosophies are resolved silently.
+	 * @param {string} content - Raw model response text
+	 * @returns {{ commands: Array, warnings: string[], unparsed: string }}
+	 */
+	static parse(content) {
+		if (!content) return { commands: [], warnings: [], unparsed: "" };
+
+		const commands = [];
+		const warnings = [];
+		const textChunks = [];
+		let current = null;
+		let ended = false;
+
+		const parser = new Parser(
+			{
+				onopentag(name, attrs) {
+					if (!ALL_TOOLS.has(name)) {
+						if (current) {
+							current.rawBody += `<${name}>`;
+						}
+						return;
+					}
+
+					current = { name, attrs, rawBody: "" };
+				},
+
+				ontext(text) {
+					if (current) {
+						current.rawBody += text;
+					} else {
+						textChunks.push(text);
+					}
+				},
+
+				onclosetag(name, isImplied) {
+					if (current && name === current.name) {
+						if (ended) {
+							warnings.push(`Unclosed <${name}> tag — content captured anyway`);
+						}
+						commands.push(
+							resolveCommand(current.name, current.attrs, current.rawBody),
+						);
+						current = null;
+					} else if (current) {
+						current.rawBody += `</${name}>`;
+					} else if (isImplied && ALL_TOOLS.has(name)) {
+						// Self-closing tag that htmlparser2 auto-closed
+					}
+				},
+
+				onerror(err) {
+					warnings.push(`Parse error: ${err.message}`);
+				},
+			},
+			{
+				recognizeSelfClosing: true,
+				lowerCaseTags: true,
+				lowerCaseAttributeNames: true,
+			},
+		);
+
+		parser.write(content);
+		ended = true;
+		parser.end();
+
+		// Flush any unclosed tool tag
+		if (current) {
+			warnings.push(`Unclosed <${current.name}> tag — content captured anyway`);
+			commands.push(
+				resolveCommand(current.name, current.attrs, current.rawBody),
+			);
+			current = null;
+		}
+
+		const unparsed = textChunks.join("").trim();
+		return { commands, warnings, unparsed };
+	}
+}