@possumtech/rummy 0.3.1 → 0.4.0

package/src/agent/TurnExecutor.js

@@ -6,6 +6,10 @@ import ResponseHealer from "./ResponseHealer.js";
 import { countTokens } from "./tokens.js";
 import XmlParser from "./XmlParser.js";
 
+const ACTION_SCHEMES = new Set(["get", "set", "rm", "mv", "cp", "sh", "env", "search"]);
+const MUTATION_SCHEMES = new Set(["set", "rm", "sh", "mv", "cp"]);
+const READ_SCHEMES = new Set(["get", "env", "search"]);
+
 export default class TurnExecutor {
 	#db;
 	#llmProvider;
@@ -117,13 +121,6 @@ export default class TurnExecutor {
 			sequence: turn,
 		});
 
-		const unresolved = await this.#knownStore.getUnresolved(currentRunId);
-		if (unresolved.length > 0) {
-			throw new Error(
-				msg("error.unresolved_proposed", { count: unresolved.length }),
-			);
-		}
-
 		// Build RummyContext before turn.started so plugins can write entries
 		const rummy = new RummyContext(
 			{
@@ -297,6 +294,10 @@ export default class TurnExecutor {
 			/\b(503|429|timeout|ECONNREFUSED|ECONNRESET|unavailable)\b/i.test(
 				e.message,
 			);
+		const isContextExceeded = (e) =>
+			/\b(context.*(size|length|limit)|token.*(limit|exceed)|too.*(long|large))\b/i.test(
+				e.message,
+			);
 
 		for (let llmAttempt = 0; ; llmAttempt++) {
 			try {
@@ -315,6 +316,18 @@ export default class TurnExecutor {
 					await new Promise((r) => setTimeout(r, delay));
 					continue;
 				}
+				if (isContextExceeded(err)) {
+					console.warn(
+						`[RUMMY] LLM context exceeded: ${err.message.slice(0, 120)}. Returning 413.`,
+					);
+					return {
+						turn,
+						turnId: turnRow.id,
+						status: 413,
+						assembledTokens,
+						contextSize,
+					};
+				}
 				throw err;
 			}
 		}
@@ -371,15 +384,7 @@ export default class TurnExecutor {
 		});
 
 		// --- PHASE 1: RECORD ---
-		// Split lifecycle signals from action commands.
-		// Lifecycle signals (summarize, update, unknown, known) are state
-		// declarations — always recorded, never 409'd by sequential dispatch.
-		const LIFECYCLE = new Set(["summarize", "update", "unknown", "known"]);
-
 		const recorded = [];
-		const lifecycle = [];
-		const actions = [];
-
 		for (const cmd of commands) {
 			const entry = await this.#record(
 				currentRunId,
@@ -388,33 +393,19 @@ export default class TurnExecutor {
 				mode,
 				cmd,
 			);
-			if (!entry) continue;
-			recorded.push(entry);
-
-			if (LIFECYCLE.has(entry.scheme)) {
-				lifecycle.push(entry);
-			} else {
-				actions.push(entry);
-			}
+			if (entry) recorded.push(entry);
 		}
 
 		// --- PHASE 2: DISPATCH ---
+		// Sequential queue. Each tool completes before the next starts.
+		// On failure: abort remaining. On proposal: notify client, await
+		// resolution, continue.
 		let hasErrors = false;
-		let hasProposed = false;
 		let abortAfter = null;
-		const dispatched = [...lifecycle];
-
-		// Lifecycle signals first — always dispatched, never aborted.
-		for (const entry of lifecycle) {
-			await this.#hooks.tool.before.emit({ entry, rummy });
-			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
-			await this.#hooks.tool.after.emit({ entry, rummy });
-			await this.#hooks.entry.created.emit(entry);
-		}
 
-		for (const entry of actions) {
+		for (const entry of recorded) {
 			if (abortAfter) {
-				const errorMsg = `Aborted — preceding <${abortAfter}> requires resolution.`;
+				const errorMsg = `Aborted — preceding <${abortAfter}> failed.`;
 				await this.#knownStore.upsert(
 					currentRunId,
 					turn,
@@ -431,35 +422,40 @@ export default class TurnExecutor {
 			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
 			await this.#hooks.tool.after.emit({ entry, rummy });
 			await this.#hooks.entry.created.emit(entry);
-			dispatched.push(entry);
 
-			const row = await this.#db.get_entry_state.get({
-				run_id: currentRunId,
-				path: entry.resultPath || entry.path,
-			});
-			if (row?.status === 202) {
-				hasProposed = true;
-				abortAfter = entry.scheme;
-			} else if (row?.status >= 400) {
-				hasErrors = true;
-				abortAfter = entry.scheme;
-			}
-		}
+			// Materialize proposals for this entry (set revisions → 202)
+			await this.#hooks.turn.proposing.emit({ rummy, recorded: [entry] });
 
-		// Materialize proposals only if we dispatched actions
-		if (!abortAfter || hasProposed) {
-			await this.#hooks.turn.proposing.emit({ rummy, recorded: dispatched });
-		}
+			// Check for any proposals created by this entry's dispatch
+			const proposed = await this.#knownStore.getUnresolved(currentRunId);
+			for (const p of proposed) {
+				await this.#hooks.turn.proposal.emit({
+					projectId,
+					run: currentAlias,
+					proposed: [p],
+				});
+				await this.#knownStore.waitForResolution(currentRunId, p.path);
+				const resolved = await this.#db.get_entry_state.get({
+					run_id: currentRunId,
+					path: p.path,
+				});
+				if (resolved?.status >= 400) {
+					hasErrors = true;
+					abortAfter = entry.scheme;
+				}
+			}
 
-		// Recheck after materialization (set handler may create proposals)
-		if (!hasProposed && !hasErrors) {
-			for (const entry of actions) {
+			// Also check the entry itself for direct failures
+			if (!hasErrors) {
+				const entryPath = entry.resultPath || entry.path;
 				const row = await this.#db.get_entry_state.get({
 					run_id: currentRunId,
-					path: entry.resultPath || entry.path,
+					path: entryPath,
 				});
-				if (row?.status === 202) hasProposed = true;
-				if (row?.status >= 400) hasErrors = true;
+				if (row?.status >= 400) {
+					hasErrors = true;
+					abortAfter = entry.scheme;
+				}
 			}
 		}
 
@@ -468,8 +464,7 @@ export default class TurnExecutor {
 		// budget recovery phase before continuing.
 		let budgetRecovery = null;
 		// Use actual prompt_tokens from this turn's LLM response as the ground-truth
-		// token count for post-turn budget checks — more accurate than the estimate.
-		const currentPromptTokens = result.usage?.prompt_tokens ?? 0;
+		// Post-dispatch budget check — demotion handled by budget plugin
 		if (contextSize) {
 			const postMat = await this.#materializeTurnContext({
 				runId: currentRunId,
@@ -481,92 +476,31 @@ export default class TurnExecutor {
 				contextSize,
 				demoted,
 			});
-			const postBudget = await this.#hooks.budget.enforce({
+			budgetRecovery = await this.#hooks.budget.postDispatch({
 				contextSize,
 				messages: postMat.messages,
 				rows: postMat.rows,
-				lastPromptTokens: currentPromptTokens,
+				runId: currentRunId,
+				loopId: currentLoopId,
+				turn,
+				db: this.#db,
+				store: this.#knownStore,
 			});
-			if (postBudget.status === 413) {
-				// Demote this turn's data entries.
-				const demotedEntries = await this.#db.demote_turn_data_entries.all({
-					run_id: currentRunId,
-					turn,
-				});
-				const paths = demotedEntries.map((r) => r.path).join(", ");
-
-				// Also summarize the prompt — forces the model to earn it back.
-				const promptRow = postMat.rows.find((r) => r.scheme === "prompt");
-				if (promptRow) {
-					await this.#knownStore.setFidelity(
-						currentRunId,
-						promptRow.path,
-						"summary",
-					);
-				}
-
-				// Re-materialize after both demotions for accurate token count.
-				const recoveryMat = await this.#materializeTurnContext({
-					runId: currentRunId,
-					loopId: currentLoopId,
-					turn,
-					systemPrompt,
-					mode,
-					toolSet: effectiveToolSet,
-					contextSize,
-					demoted,
-				});
-				const recoveryBudget = await this.#hooks.budget.enforce({
-					contextSize,
-					messages: recoveryMat.messages,
-					rows: recoveryMat.rows,
-					lastPromptTokens: currentPromptTokens,
-				});
-				const safeLevel = Math.floor(contextSize * 0.9);
-				const tokensToFree = Math.max(
-					0,
-					recoveryBudget.assembledTokens - safeLevel,
-				);
-
-				const promptLine =
-					tokensToFree > 0
-						? `Info: Prompt auto-summarized. Full prompt restores automatically when you free ${tokensToFree} tokens.`
-						: "Info: Prompt auto-summarized. It will restore automatically.";
-				const body = [
-					"Error 413: Context Size Exceeded",
-					"",
-					"Required: YOU MUST demote larger and/or less relevant items to optimize your context.",
-					`Info: ${paths} have been automatically summarized to avoid overflow.`,
-					promptLine,
-					"Info: YOU MAY use bulk patterns to demote and promote entries by pattern.",
-					"Info: Well-designed paths and summaries improve context management.",
-					'Example: <set path="known://people/*" fidelity="summary"/>',
-				].join("\n");
-
-				await this.#knownStore.upsert(
-					currentRunId,
-					turn,
-					`budget://${currentLoopId}/${turn}`,
-					body,
-					413,
-					{ loopId: currentLoopId },
-				);
-
-				budgetRecovery = {
-					target: safeLevel,
-					promptPath: promptRow?.path ?? null,
-				};
-			}
 		}
 
-		// Lifecycle signals are always available — never 409'd.
-		const summaryEntry = lifecycle.find((e) => e.scheme === "summarize");
-		const updateEntry = lifecycle.find((e) => e.scheme === "update");
+		const summaryEntry = recorded.findLast((e) => e.scheme === "summarize");
+		const updateEntry = recorded.findLast((e) => e.scheme === "update");
 		let summaryText = summaryEntry?.body || null;
 		let updateText = updateEntry?.body || null;
 
-		// If model sent both, update wins — if it can't decide, it's not done
-		if (summaryText && updateText) summaryText = null;
+		// If model sent both, last signal wins — respects the model's final intent
+		if (summaryText && updateText) {
+			const lastLifecycle = recorded.findLast(
+				(e) => e.scheme === "summarize" || e.scheme === "update",
+			);
+			if (lastLifecycle.scheme === "summarize") updateText = null;
+			else summaryText = null;
+		}
 
 		// If model says "done" but actions failed, override — the model's
 		// assertion that it's done is false if it failed to do what it tried.
@@ -598,11 +532,7 @@ export default class TurnExecutor {
 
 		// --- Classify for return value ---
 
-		const actionCalls = recorded.filter((e) =>
-			["get", "set", "rm", "mv", "cp", "sh", "env", "search"].includes(
-				e.scheme,
-			),
-		);
+		const actionCalls = recorded.filter((e) => ACTION_SCHEMES.has(e.scheme));
 		const writeCalls = recorded.filter(
 			(e) =>
 				e.scheme === "known" ||
@@ -610,12 +540,8 @@ export default class TurnExecutor {
 		);
 		const unknownCalls = recorded.filter((e) => e.scheme === "unknown");
 
-		const hasAct = actionCalls.some((c) =>
-			["set", "rm", "sh", "mv", "cp"].includes(c.scheme),
-		);
-		const hasReads = actionCalls.some((c) =>
-			["get", "env", "search"].includes(c.scheme),
-		);
+		const hasAct = actionCalls.some((c) => MUTATION_SCHEMES.has(c.scheme));
+		const hasReads = actionCalls.some((c) => READ_SCHEMES.has(c.scheme));
 		const hasWrites = writeCalls.length > 0 || unknownCalls.length > 0;
 		const flags = { hasAct, hasReads, hasWrites };
 
@@ -651,83 +577,7 @@ export default class TurnExecutor {
 	 * Returns the recorded entry descriptor, or null if rejected/skipped.
 	 */
 	async #record(runId, loopId, turn, mode, cmd) {
-		if (mode === "ask") {
-			if (cmd.name === "sh") {
-				console.warn("[RUMMY] Rejected <sh> in ask mode");
-				return null;
-			}
-			if (cmd.name === "set" && cmd.path && cmd.body) {
-				const scheme = KnownStore.scheme(cmd.path);
-				if (scheme === null) {
-					console.warn(
-						`[RUMMY] Rejected file edit to ${cmd.path} in ${mode} mode`,
-					);
-					return null;
-				}
-			}
-			if (cmd.name === "rm" && cmd.path) {
-				const scheme = KnownStore.scheme(cmd.path);
-				if (scheme === null) {
-					console.warn(`[RUMMY] Rejected file rm of ${cmd.path} in ask mode`);
-					return null;
-				}
-			}
-			if ((cmd.name === "mv" || cmd.name === "cp") && cmd.to) {
-				const destScheme = KnownStore.scheme(cmd.to);
-				if (destScheme === null) {
-					console.warn(
-						`[RUMMY] Rejected ${cmd.name} to file ${cmd.to} in ask mode`,
-					);
-					return null;
-				}
-			}
-		}
-
 		const scheme = cmd.name;
-
-		// Structural tags — recorded like any other entry
-		if (scheme === "summarize" || scheme === "update") {
-			const statusPath = await this.#knownStore.slugPath(
-				runId,
-				scheme,
-				cmd.body,
-			);
-			await this.#knownStore.upsert(runId, turn, statusPath, cmd.body, 200, {
-				loopId,
-			});
-			return {
-				scheme,
-				body: cmd.body,
-				path: statusPath,
-				resultPath: statusPath,
-				attributes: null,
-			};
-		}
-
-		// Unknown — deduplicated, sticky
-		if (scheme === "unknown") {
-			const existingValues = await this.#knownStore.getUnknownValues(runId);
-			if (existingValues.has(cmd.body)) {
-				console.warn(`[RUMMY] Unknown deduped: "${cmd.body.slice(0, 60)}"`);
-				return null;
-			}
-			const unknownPath = await this.#knownStore.slugPath(
-				runId,
-				"unknown",
-				cmd.body,
-			);
-			await this.#knownStore.upsert(runId, turn, unknownPath, cmd.body, 200, {
-				loopId,
-			});
-			return {
-				scheme,
-				path: unknownPath,
-				body: cmd.body,
-				resultPath: unknownPath,
-				attributes: null,
-			};
-		}
-
 		const rawTarget = cmd.path || cmd.command || cmd.question || "";
 		// Reject paths that are likely reasoning bleed — too long or contain non-printing chars
 		if (rawTarget.length > 512 || /\p{Cc}/u.test(rawTarget)) {
@@ -766,108 +616,15 @@ export default class TurnExecutor {
 		const { name: _, ...attributes } = cmd;
 		if (cmd.path) attributes.path = target;
 
-		// known tool or naked write → known:// slug from body
-		if (scheme === "known" || (scheme === "set" && !cmd.path)) {
-			if (!cmd.body) return null;
-
-			// Size gate: reject entries > 512 tokens — force atomic entries
-			const entryTokens = countTokens(cmd.body);
-			const MAX_ENTRY_TOKENS = 512;
-			if (scheme === "known" && entryTokens > MAX_ENTRY_TOKENS) {
-				const rejectPath = await this.#knownStore.slugPath(
-					runId,
-					scheme,
-					cmd.body,
-				);
-				await this.#knownStore.upsert(
-					runId,
-					turn,
-					rejectPath,
-					`Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
-					413,
-					{ loopId },
-				);
-				return {
-					scheme,
-					path: rejectPath,
-					body: "",
-					resultPath: rejectPath,
-					attributes,
-					status: 413,
-				};
-			}
-
-			let knownPath = cmd.path;
-			if (!knownPath) {
-				knownPath = await this.#knownStore.slugPath(
-					runId,
-					"known",
-					cmd.body,
-					cmd.summary,
-				);
-			}
-			// Dedup: if this exact path already exists, update rather than duplicate
-			const existing = await this.#knownStore.getEntriesByPattern(
-				runId,
-				knownPath,
-				null,
-			);
-			if (existing.length > 0) {
-				// Path exists — update body and turn, skip creating a new entry
-				await this.#knownStore.upsert(
-					runId,
-					turn,
-					existing[0].path,
-					cmd.body || existing[0].body,
-					200,
-					{
-						attributes,
-						loopId,
-					},
-				);
-				return {
-					scheme: "known",
-					path: existing[0].path,
-					body: cmd.body || existing[0].body,
-					resultPath: existing[0].path,
-					attributes,
-				};
-			}
-			await this.#knownStore.upsert(runId, turn, knownPath, cmd.body, 200, {
-				attributes,
-				loopId,
-			});
-			return {
-				scheme: "known",
-				path: knownPath,
-				body: cmd.body,
-				resultPath: knownPath,
-				attributes,
-			};
-		}
-
 		const body = cmd.body || cmd.command || cmd.question || "";
 
 		// Filter: plugins can validate/transform before recording
 		const filtered = await this.#hooks.entry.recording.filter(
 			{ scheme, path: resultPath, body, attributes, status: 200 },
-			{ runId, turn, loopId },
+			{ runId, turn, loopId, mode },
 		);
 		if (filtered.status >= 400) return filtered;
 
-		// Record the entry — 200 OK, handlers change status during dispatch
-		await this.#knownStore.upsert(
-			runId,
-			turn,
-			filtered.path,
-			filtered.body,
-			200,
-			{
-				attributes: filtered.attributes,
-				loopId,
-			},
-		);
-
 		return {
 			scheme: filtered.scheme,
 			path: filtered.path,

package/src/agent/XmlParser.js

@@ -14,8 +14,6 @@ export const ALL_TOOLS = new Set([
 	"update",
 	"unknown",
 	"think",
-	"thought",
-	"mcp",
 ]);
 
 /**
@@ -145,6 +143,8 @@ export default class XmlParser {
 	 * @param {string} content - Raw model response text
 	 * @returns {{ commands: Array, warnings: string[], unparsed: string }}
 	 */
+	static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS) || 99;
+
 	static parse(content) {
 		if (!content) return { commands: [], warnings: [], unparsed: "" };
 
@@ -156,13 +156,20 @@ export default class XmlParser {
 		const textChunks = [];
 		let current = null;
 		let ended = false;
+		let capped = false;
 
 		const parser = new Parser(
 			{
 				onopentag(name, attrs) {
+					if (capped) return;
 					if (!ALL_TOOLS.has(name)) {
 						if (current) {
-							current.rawBody += `<${name}>`;
+							const attrStr = Object.entries(attrs)
+								.map(([k, v]) => v === "" ? k : `${k}="${v}"`)
+								.join(" ");
+							current.rawBody += attrStr
+								? `<${name} ${attrStr}>`
+								: `<${name}>`;
 						}
 						return;
 					}
@@ -177,10 +184,17 @@ export default class XmlParser {
 						);
 					}
 
+					if (commands.length >= XmlParser.MAX_COMMANDS) {
+						capped = true;
+						current = null;
+						return;
+					}
+
 					current = { name, attrs, rawBody: "" };
 				},
 
 				ontext(text) {
+					if (capped) return;
 					if (current) {
 						current.rawBody += text;
 					} else {
@@ -189,6 +203,7 @@ export default class XmlParser {
 				},
 
 				onclosetag(name, isImplied) {
+					if (capped) return;
 					if (current && name === current.name) {
 						if (ended) {
 							warnings.push(`Unclosed <${name}> tag — content captured anyway`);
@@ -230,7 +245,7 @@ export default class XmlParser {
 		parser.end();
 
 		// Flush any unclosed tool tag
-		if (current) {
+		if (current && !capped) {
 			warnings.push(`Unclosed <${current.name}> tag — content captured anyway`);
 			commands.push(
 				resolveCommand(current.name, current.attrs, current.rawBody),
@@ -238,6 +253,12 @@ export default class XmlParser {
 			current = null;
 		}
 
+		if (capped) {
+			warnings.push(
+				`Tool call limit (${XmlParser.MAX_COMMANDS}) reached — remaining commands dropped`,
+			);
+		}
+
 		const unparsed = textChunks.join("").trim();
 		return { commands, warnings, unparsed };
 	}

package/src/agent/known_queries.sql

@@ -1,5 +1,5 @@
 -- PREP: get_known_entries
-SELECT path, scheme, status, fidelity, body, turn, hash, attributes
+SELECT path, scheme, status, fidelity, body, turn, hash, attributes, tokens
 FROM known_entries
 WHERE run_id = :run_id
 ORDER BY path;