@possumtech/rummy 0.3.0 → 0.3.1

package/src/agent/TurnExecutor.js

@@ -19,6 +19,68 @@ export default class TurnExecutor {
 		this.#knownStore = knownStore;
 	}
 
+	/**
+	 * Rebuild turn_context from v_model_context, then assemble messages.
+	 * Called at turn start and again after any fidelity demotion within the turn.
+	 */
+	async #materializeTurnContext({
+		runId,
+		loopId,
+		turn,
+		systemPrompt,
+		mode,
+		toolSet,
+		contextSize,
+		demoted,
+	}) {
+		await this.#db.clear_turn_context.run({ run_id: runId, turn });
+		const viewRows = await this.#db.get_model_context.all({ run_id: runId });
+		for (const row of viewRows) {
+			const scheme = row.scheme || "file";
+			const projectedBody = await this.#hooks.tools.view(scheme, {
+				path: row.path,
+				scheme,
+				body: row.body,
+				attributes: row.attributes ? JSON.parse(row.attributes) : null,
+				fidelity: row.fidelity,
+				category: row.category,
+			});
+			await this.#db.insert_turn_context.run({
+				run_id: runId,
+				loop_id: loopId,
+				turn,
+				ordinal: row.ordinal,
+				path: row.path,
+				fidelity: row.fidelity,
+				status: row.status,
+				body: projectedBody ?? "",
+				tokens: countTokens(projectedBody ?? ""),
+				attributes: row.attributes,
+				category: row.category,
+				source_turn: row.turn,
+			});
+		}
+		const rows = await this.#db.get_turn_context.all({ run_id: runId, turn });
+		const lastCtx = await this.#db.get_last_context_tokens.get({
+			run_id: runId,
+		});
+		const lastContextTokens = lastCtx?.context_tokens ?? 0;
+		const messages = await ContextAssembler.assembleFromTurnContext(
+			rows,
+			{
+				type: mode,
+				systemPrompt,
+				contextSize,
+				demoted,
+				toolSet,
+				lastContextTokens,
+				turn,
+			},
+			this.#hooks,
+		);
+		return { rows, messages, lastContextTokens };
+	}
+
 	async execute({
 		mode,
 		project,
@@ -28,12 +90,25 @@ export default class TurnExecutor {
 		currentLoopId,
 		requestedModel,
 		loopPrompt,
+		loopIteration,
 		noRepo,
 		toolSet,
+		inRecovery = false,
 		contextSize,
 		options,
 		signal,
 	}) {
+		const RECOVERY_EXCLUDED = new Set([
+			"sh",
+			"env",
+			"search",
+			"ask_user",
+			"set",
+		]);
+		const effectiveToolSet = inRecovery
+			? new Set([...toolSet].filter((t) => !RECOVERY_EXCLUDED.has(t)))
+			: toolSet;
+
 		const turn = await this.#knownStore.nextTurn(currentRunId);
 
 		const turnRow = await this.#db.create_turn.get({
@@ -73,7 +148,7 @@ export default class TurnExecutor {
 				loopId: currentLoopId,
 				turnId: turnRow.id,
 				noRepo,
-				toolSet,
+				toolSet: effectiveToolSet,
 				contextSize,
 				systemPrompt: null,
 				loopPrompt,
@@ -85,6 +160,7 @@ export default class TurnExecutor {
 			mode,
 			prompt: loopPrompt,
 			isContinuation: options?.isContinuation,
+			loopIteration,
 		});
 
 		await this.#hooks.processTurn(rummy);
@@ -111,43 +187,23 @@ export default class TurnExecutor {
 		});
 
 		// Materialize turn_context: VIEW rows projected through tools
-		await this.#db.clear_turn_context.run({ run_id: currentRunId, turn });
-		const viewRows = await this.#db.get_model_context.all({
-			run_id: currentRunId,
-		});
-		for (const row of viewRows) {
-			const scheme = row.scheme || "file";
-			const projectedBody = await this.#hooks.tools.view(scheme, {
-				path: row.path,
-				scheme,
-				body: row.body,
-				attributes: row.attributes ? JSON.parse(row.attributes) : null,
-				fidelity: row.fidelity,
-				category: row.category,
-			});
-
-			await this.#db.insert_turn_context.run({
-				run_id: currentRunId,
-				loop_id: currentLoopId,
+		const demoted = [];
+		let { rows, messages, lastContextTokens } =
+			await this.#materializeTurnContext({
+				runId: currentRunId,
+				loopId: currentLoopId,
 				turn,
-				ordinal: row.ordinal,
-				path: row.path,
-				fidelity: row.fidelity,
-				status: row.status,
-				body: projectedBody ?? "",
-				tokens: countTokens(projectedBody ?? ""),
-				attributes: row.attributes,
-				category: row.category,
-				source_turn: row.turn,
+				systemPrompt,
+				mode,
+				toolSet: effectiveToolSet,
+				contextSize,
+				demoted,
 			});
-		}
-
-		const demoted = [];
 
 		await this.#hooks.context.materialized.emit({
 			runId: currentRunId,
 			turn,
-			rowCount: viewRows.length,
+			rowCount: rows.length,
 		});
 
 		await this.#hooks.run.progress.emit({
@@ -157,50 +213,75 @@ export default class TurnExecutor {
 			status: "thinking",
 		});
 
-		let rows = await this.#db.get_turn_context.all({
-			run_id: currentRunId,
-			turn,
-		});
-		const lastCtx = await this.#db.get_last_context_tokens.get({
-			run_id: currentRunId,
-		});
-		const lastContextTokens = lastCtx?.context_tokens ?? 0;
-
-		let messages = await ContextAssembler.assembleFromTurnContext(
-			rows,
-			{
-				type: mode,
-				systemPrompt,
-				contextSize,
-				demoted,
-				toolSet,
-				lastContextTokens,
-			},
-			this.#hooks,
-		);
-
 		const budgetResult = await this.#hooks.budget.enforce({
 			contextSize,
 			messages,
 			rows,
+			lastPromptTokens: lastContextTokens,
 		});
 		messages = budgetResult.messages;
 		rows = budgetResult.rows;
-		const assembledTokens =
+		let assembledTokens =
 			budgetResult.assembledTokens ??
 			messages.reduce((sum, m) => sum + countTokens(m.content), 0);
 
-		// Budget overflow — return 413 to caller without calling LLM.
-		// Panic mode suppresses this — the model must run to free space.
-		if (budgetResult.status === 413 && mode !== "panic") {
-			return {
-				turn,
-				turnId: turnRow.id,
-				status: 413,
-				assembledTokens,
-				contextSize,
-				overflow: budgetResult.overflow,
-			};
+		if (budgetResult.status === 413) {
+			if (loopIteration === 1) {
+				// Prompt Demotion: first-turn overflow — demote incoming prompt to summary
+				const promptRow = rows.findLast(
+					(r) => r.category === "prompt" && r.scheme === "prompt",
+				);
+				if (promptRow) {
+					await this.#knownStore.setFidelity(
+						currentRunId,
+						promptRow.path,
+						"summary",
+					);
+				}
+				const reMat = await this.#materializeTurnContext({
+					runId: currentRunId,
+					loopId: currentLoopId,
+					turn,
+					systemPrompt,
+					mode,
+					toolSet: effectiveToolSet,
+					contextSize,
+					demoted,
+				});
+				rows = reMat.rows;
+				messages = reMat.messages;
+				const recheck = await this.#hooks.budget.enforce({
+					contextSize,
+					messages,
+					rows,
+					lastPromptTokens: reMat.lastContextTokens,
+				});
+				messages = recheck.messages;
+				rows = recheck.rows;
+				assembledTokens =
+					recheck.assembledTokens ??
+					messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+				if (recheck.status === 413) {
+					return {
+						turn,
+						turnId: turnRow.id,
+						status: 413,
+						assembledTokens,
+						contextSize,
+						overflow: recheck.overflow,
+					};
+				}
+			} else {
+				// Base context too large even without new prompt — genuine failure
+				return {
+					turn,
+					turnId: turnRow.id,
+					status: 413,
+					assembledTokens,
+					contextSize,
+					overflow: budgetResult.overflow,
+				};
+			}
 		}
 
 		const filteredMessages = await this.#hooks.llm.messages.filter(messages, {
@@ -318,104 +399,164 @@ export default class TurnExecutor {
 		}
 
 		// --- PHASE 2: DISPATCH ---
-		// Budget plugin activates the guard on the store for dispatch.
-		const guard = this.#hooks.budget.activate(
-			this.#knownStore,
-			contextSize,
-			assembledTokens,
-		);
-		const { BudgetExceeded } = this.#hooks.budget;
-
 		let hasErrors = false;
 		let hasProposed = false;
 		let abortAfter = null;
 		const dispatched = [...lifecycle];
 
-		try {
-			// Lifecycle signals first — always dispatched, never aborted.
-			for (const entry of lifecycle) {
-				await this.#hooks.tool.before.emit({ entry, rummy });
-				await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
-				await this.#hooks.tool.after.emit({ entry, rummy });
-				await this.#hooks.entry.created.emit(entry);
+		// Lifecycle signals first — always dispatched, never aborted.
+		for (const entry of lifecycle) {
+			await this.#hooks.tool.before.emit({ entry, rummy });
+			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
+			await this.#hooks.tool.after.emit({ entry, rummy });
+			await this.#hooks.entry.created.emit(entry);
+		}
+
+		for (const entry of actions) {
+			if (abortAfter) {
+				const errorMsg = `Aborted — preceding <${abortAfter}> requires resolution.`;
+				await this.#knownStore.upsert(
+					currentRunId,
+					turn,
+					entry.resultPath || entry.path,
+					errorMsg,
+					409,
+					{ attributes: { error: errorMsg }, loopId: currentLoopId },
+				);
+				hasErrors = true;
+				continue;
 			}
 
-			for (const entry of actions) {
-				if (abortAfter || guard.isTripped) {
-					await this.#knownStore.upsert(
-						currentRunId,
-						turn,
-						entry.resultPath || entry.path,
-						"",
-						guard.isTripped ? 413 : 409,
-						{
-							attributes: {
-								error: guard.isTripped
-									? `Budget exceeded by <${guard.tripSource}>.`
-									: `Aborted — preceding <${abortAfter}> requires resolution.`,
-							},
-							loopId: currentLoopId,
-						},
-					);
-					hasErrors = true;
-					continue;
-				}
+			await this.#hooks.tool.before.emit({ entry, rummy });
+			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
+			await this.#hooks.tool.after.emit({ entry, rummy });
+			await this.#hooks.entry.created.emit(entry);
+			dispatched.push(entry);
 
-				try {
-					await this.#hooks.tool.before.emit({ entry, rummy });
-					await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
-					await this.#hooks.tool.after.emit({ entry, rummy });
-					await this.#hooks.entry.created.emit(entry);
-					dispatched.push(entry);
-				} catch (err) {
-					if (err instanceof BudgetExceeded) {
-						guard.trip(entry.scheme);
-						await this.#knownStore.upsert(
-							currentRunId,
-							turn,
-							entry.resultPath || entry.path,
-							`Budget exceeded: ${err.requested} tokens requested, ${err.remaining} remaining.`,
-							413,
-							{ attributes: { error: err.message }, loopId: currentLoopId },
-						);
-						hasErrors = true;
-						abortAfter = entry.scheme;
-						continue;
-					}
-					throw err;
-				}
+			const row = await this.#db.get_entry_state.get({
+				run_id: currentRunId,
+				path: entry.resultPath || entry.path,
+			});
+			if (row?.status === 202) {
+				hasProposed = true;
+				abortAfter = entry.scheme;
+			} else if (row?.status >= 400) {
+				hasErrors = true;
+				abortAfter = entry.scheme;
+			}
+		}
+
+		// Materialize proposals only if we dispatched actions
+		if (!abortAfter || hasProposed) {
+			await this.#hooks.turn.proposing.emit({ rummy, recorded: dispatched });
+		}
 
+		// Recheck after materialization (set handler may create proposals)
+		if (!hasProposed && !hasErrors) {
+			for (const entry of actions) {
 				const row = await this.#db.get_entry_state.get({
 					run_id: currentRunId,
 					path: entry.resultPath || entry.path,
 				});
-				if (row?.status === 202) {
-					hasProposed = true;
-					abortAfter = entry.scheme;
-				} else if (row?.status >= 400) {
-					hasErrors = true;
-					abortAfter = entry.scheme;
-				}
+				if (row?.status === 202) hasProposed = true;
+				if (row?.status >= 400) hasErrors = true;
 			}
+		}
 
-			// Materialize proposals only if we dispatched actions
-			if (!abortAfter || hasProposed) {
-				await this.#hooks.turn.proposing.emit({ rummy, recorded: dispatched });
-			}
+		// Turn Demotion: if end-of-turn context exceeds ceiling, demote this
+		// turn's data entries and the incoming prompt to summary, then force a
+		// budget recovery phase before continuing.
+		let budgetRecovery = null;
+		// Use actual prompt_tokens from this turn's LLM response as the ground-truth
+		// token count for post-turn budget checks — more accurate than the estimate.
+		const currentPromptTokens = result.usage?.prompt_tokens ?? 0;
+		if (contextSize) {
+			const postMat = await this.#materializeTurnContext({
+				runId: currentRunId,
+				loopId: currentLoopId,
+				turn,
+				systemPrompt,
+				mode,
+				toolSet: effectiveToolSet,
+				contextSize,
+				demoted,
+			});
+			const postBudget = await this.#hooks.budget.enforce({
+				contextSize,
+				messages: postMat.messages,
+				rows: postMat.rows,
+				lastPromptTokens: currentPromptTokens,
+			});
+			if (postBudget.status === 413) {
+				// Demote this turn's data entries.
+				const demotedEntries = await this.#db.demote_turn_data_entries.all({
+					run_id: currentRunId,
+					turn,
+				});
+				const paths = demotedEntries.map((r) => r.path).join(", ");
 
-			// Recheck after materialization (set handler may create proposals)
-			if (!hasProposed && !hasErrors) {
-				for (const entry of actions) {
-					const row = await this.#db.get_entry_state.get({
-						run_id: currentRunId,
-						path: entry.resultPath || entry.path,
-					});
-					if (row?.status === 202) hasProposed = true;
-					if (row?.status >= 400) hasErrors = true;
+				// Also summarize the prompt — forces the model to earn it back.
+				const promptRow = postMat.rows.find((r) => r.scheme === "prompt");
+				if (promptRow) {
+					await this.#knownStore.setFidelity(
+						currentRunId,
+						promptRow.path,
+						"summary",
+					);
 				}
+
+				// Re-materialize after both demotions for accurate token count.
+				const recoveryMat = await this.#materializeTurnContext({
+					runId: currentRunId,
+					loopId: currentLoopId,
+					turn,
+					systemPrompt,
+					mode,
+					toolSet: effectiveToolSet,
+					contextSize,
+					demoted,
+				});
+				const recoveryBudget = await this.#hooks.budget.enforce({
+					contextSize,
+					messages: recoveryMat.messages,
+					rows: recoveryMat.rows,
+					lastPromptTokens: currentPromptTokens,
+				});
+				const safeLevel = Math.floor(contextSize * 0.9);
+				const tokensToFree = Math.max(
+					0,
+					recoveryBudget.assembledTokens - safeLevel,
+				);
+
+				const promptLine =
+					tokensToFree > 0
+						? `Info: Prompt auto-summarized. Full prompt restores automatically when you free ${tokensToFree} tokens.`
+						: "Info: Prompt auto-summarized. It will restore automatically.";
+				const body = [
+					"Error 413: Context Size Exceeded",
+					"",
+					"Required: YOU MUST demote larger and/or less relevant items to optimize your context.",
+					`Info: ${paths} have been automatically summarized to avoid overflow.`,
+					promptLine,
+					"Info: YOU MAY use bulk patterns to demote and promote entries by pattern.",
+					"Info: Well-designed paths and summaries improve context management.",
+					'Example: <set path="known://people/*" fidelity="summary"/>',
+				].join("\n");
+
+				await this.#knownStore.upsert(
+					currentRunId,
+					turn,
+					`budget://${currentLoopId}/${turn}`,
+					body,
+					413,
+					{ loopId: currentLoopId },
+				);
+
+				budgetRecovery = {
+					target: safeLevel,
+					promptPath: promptRow?.path ?? null,
+				};
 			}
-		} finally {
-			this.#hooks.budget.deactivate(this.#knownStore);
 		}
 
 		// Lifecycle signals are always available — never 409'd.
@@ -433,6 +574,15 @@ export default class TurnExecutor {
 			console.warn(
 				"[RUMMY] Overriding <summarize> — actions in this turn failed. Continuing.",
 			);
+			// Mark the recorded summarize entry as 409 so the model sees it was rejected
+			if (summaryEntry?.path) {
+				await this.#knownStore.resolve(
+					currentRunId,
+					summaryEntry.path,
+					409,
+					"Overridden — actions in this turn failed. Use <update/> until resolved.",
+				);
+			}
 			updateText = summaryText;
 			summaryText = null;
 		}
@@ -484,12 +634,11 @@ export default class TurnExecutor {
 			flags,
 			model: result.model || requestedModel,
 			modelAlias: requestedModel,
-			temperature:
-				options?.temperature ??
-				Number.parseFloat(process.env.RUMMY_TEMPERATURE || "0.7"),
+			temperature: options?.temperature,
 			contextSize,
 			assembledTokens,
 			usage: result.usage,
+			budgetRecovery,
 		};
 
 		await this.#hooks.turn.completed.emit(turnResult);
@@ -502,8 +651,7 @@ export default class TurnExecutor {
 	 * Returns the recorded entry descriptor, or null if rejected/skipped.
 	 */
 	async #record(runId, loopId, turn, mode, cmd) {
-		// Mode enforcement — reject prohibited commands in ask/panic mode
-		if (mode === "ask" || mode === "panic") {
+		if (mode === "ask") {
 			if (cmd.name === "sh") {
 				console.warn("[RUMMY] Rejected <sh> in ask mode");
 				return null;
@@ -581,6 +729,31 @@ export default class TurnExecutor {
 		}
 
 		const rawTarget = cmd.path || cmd.command || cmd.question || "";
+		// Reject paths that are likely reasoning bleed — too long or contain non-printing chars
+		if (rawTarget.length > 512 || /\p{Cc}/u.test(rawTarget)) {
+			const rejectPath = await this.#knownStore.dedup(
+				runId,
+				scheme,
+				`${scheme}://invalid`,
+				turn,
+			);
+			await this.#knownStore.upsert(
+				runId,
+				turn,
+				rejectPath,
+				`Invalid path: too long or contains non-printing characters`,
+				400,
+				{ loopId },
+			);
+			return {
+				scheme,
+				path: rejectPath,
+				body: "",
+				attributes: {},
+				status: 400,
+				resultPath: rejectPath,
+			};
+		}
 		const target = rawTarget;
 		const resultPath = await this.#knownStore.dedup(
 			runId,
@@ -648,6 +821,7 @@ export default class TurnExecutor {
 					cmd.body || existing[0].body,
 					200,
 					{
+						attributes,
 						loopId,
 					},
 				);
@@ -660,6 +834,7 @@ export default class TurnExecutor {
 				};
 			}
 			await this.#knownStore.upsert(runId, turn, knownPath, cmd.body, 200, {
+				attributes,
 				loopId,
 			});
 			return {
@@ -702,34 +877,4 @@ export default class TurnExecutor {
 			resultPath: filtered.path,
 		};
 	}
-
-	async #rematerialize(runId, loopId, turn) {
-		await this.#db.clear_turn_context.run({ run_id: runId, turn });
-		const viewRows = await this.#db.get_model_context.all({ run_id: runId });
-		for (const row of viewRows) {
-			const scheme = row.scheme || "file";
-			const projectedBody = await this.#hooks.tools.view(scheme, {
-				path: row.path,
-				scheme,
-				body: row.body,
-				attributes: row.attributes ? JSON.parse(row.attributes) : null,
-				fidelity: row.fidelity,
-				category: row.category,
-			});
-			await this.#db.insert_turn_context.run({
-				run_id: runId,
-				loop_id: loopId,
-				turn,
-				ordinal: row.ordinal,
-				path: row.path,
-				fidelity: row.fidelity,
-				status: row.status,
-				body: projectedBody ?? "",
-				tokens: countTokens(projectedBody ?? ""),
-				attributes: row.attributes,
-				category: row.category,
-				source_turn: row.turn,
-			});
-		}
-	}
 }

package/src/agent/XmlParser.js

@@ -4,7 +4,7 @@ import { normalizeAttrs, parseJsonEdit } from "../plugins/hedberg/normalize.js";
 import { parseSed } from "../plugins/hedberg/sed.js";
 
 const STORE_TOOLS = new Set(["get", "rm", "set", "mv", "cp", "search"]);
-const ALL_TOOLS = new Set([
+export const ALL_TOOLS = new Set([
 	...STORE_TOOLS,
 	"known",
 	"sh",
@@ -13,6 +13,9 @@ const ALL_TOOLS = new Set([
 	"summarize",
 	"update",
 	"unknown",
+	"think",
+	"thought",
+	"mcp",
 ]);
 
 /**
@@ -100,7 +103,7 @@ function resolveCommand(name, attrs, rawBody) {
 	if (name === "known") {
 		const body = trimmed || a.body || "";
 		const path = a.path || null;
-		return { name, path, body };
+		return { name, ...a, path, body };
 	}
 
 	if (name === "get" || name === "rm") {