@possumtech/rummy 0.3.1 → 0.5.0

package/src/agent/TurnExecutor.js

@@ -1,11 +1,22 @@
 import RummyContext from "../hooks/RummyContext.js";
 import ContextAssembler from "./ContextAssembler.js";
-import KnownStore from "./KnownStore.js";
-import msg from "./messages.js";
 import ResponseHealer from "./ResponseHealer.js";
 import { countTokens } from "./tokens.js";
 import XmlParser from "./XmlParser.js";
 
+const ACTION_SCHEMES = new Set([
+	"get",
+	"set",
+	"rm",
+	"mv",
+	"cp",
+	"sh",
+	"env",
+	"search",
+]);
+const MUTATION_SCHEMES = new Set(["set", "rm", "sh", "mv", "cp"]);
+const READ_SCHEMES = new Set(["get", "env", "search"]);
+
 export default class TurnExecutor {
 	#db;
 	#llmProvider;
@@ -54,7 +65,12 @@ export default class TurnExecutor {
 				fidelity: row.fidelity,
 				status: row.status,
 				body: projectedBody ?? "",
-				tokens: countTokens(projectedBody ?? ""),
+				// Full-body token count, not projected. This is the cost to
+				// promote the entry — the number the model needs to do Token
+				// Budget math. Projecting the demoted symbol-preview (145
+				// tokens for a 2108-token file) was misleading the model into
+				// promotes that blew the Token Budget by 10-30× per entry.
+				tokens: countTokens(row.body ?? ""),
 				attributes: row.attributes,
 				category: row.category,
 				source_turn: row.turn,
@@ -65,6 +81,35 @@ export default class TurnExecutor {
 			run_id: runId,
 		});
 		const lastContextTokens = lastCtx?.context_tokens ?? 0;
+
+		// Baseline materialization — assemble with model's promoted spending
+		// removed (promoted data, promoted logging). The resulting size is the
+		// fixed overhead the model can't reduce without further demotion.
+		const baselineRows = rows.filter(
+			(r) =>
+				!(
+					(r.category === "data" || r.category === "logging") &&
+					r.fidelity === "promoted"
+				),
+		);
+		const baselineMessages = await ContextAssembler.assembleFromTurnContext(
+			baselineRows,
+			{
+				type: mode,
+				systemPrompt,
+				contextSize,
+				demoted,
+				toolSet,
+				lastContextTokens,
+				turn,
+			},
+			this.#hooks,
+		);
+		const baselineTokens = baselineMessages.reduce(
+			(sum, m) => sum + countTokens(m.content),
+			0,
+		);
+
 		const messages = await ContextAssembler.assembleFromTurnContext(
 			rows,
 			{
@@ -75,6 +120,7 @@ export default class TurnExecutor {
 				toolSet,
 				lastContextTokens,
 				turn,
+				baselineTokens,
 			},
 			this.#hooks,
 		);
@@ -117,13 +163,6 @@ export default class TurnExecutor {
 			sequence: turn,
 		});
 
-		const unresolved = await this.#knownStore.getUnresolved(currentRunId);
-		if (unresolved.length > 0) {
-			throw new Error(
-				msg("error.unresolved_proposed", { count: unresolved.length }),
-			);
-		}
-
 		// Build RummyContext before turn.started so plugins can write entries
 		const rummy = new RummyContext(
 			{
@@ -182,7 +221,7 @@ export default class TurnExecutor {
 			scheme: "instructions",
 			body: instrEntry[0]?.body || "",
 			attributes: instrAttrs,
-			fidelity: "full",
+			fidelity: "promoted",
 			category: "system",
 		});
 
@@ -235,7 +274,7 @@ export default class TurnExecutor {
 					await this.#knownStore.setFidelity(
 						currentRunId,
 						promptRow.path,
-						"summary",
+						"demoted",
 					);
 				}
 				const reMat = await this.#materializeTurnContext({
@@ -284,10 +323,13 @@ export default class TurnExecutor {
 			}
 		}
 
+		const runRow = await this.#db.get_run_by_id.get({ id: currentRunId });
 		const filteredMessages = await this.#hooks.llm.messages.filter(messages, {
 			model: requestedModel,
 			projectId,
 			runId: currentRunId,
+			runAlias: runRow?.alias || `run_${currentRunId}`,
+			turn,
 		});
 
 		// Call LLM
@@ -297,6 +339,10 @@ export default class TurnExecutor {
 			/\b(503|429|timeout|ECONNREFUSED|ECONNRESET|unavailable)\b/i.test(
 				e.message,
 			);
+		const isContextExceeded = (e) =>
+			/\b(context.*(size|length|limit)|token.*(limit|exceed)|too.*(long|large))\b/i.test(
+				e.message,
+			);
 
 		for (let llmAttempt = 0; ; llmAttempt++) {
 			try {
@@ -315,6 +361,18 @@ export default class TurnExecutor {
 					await new Promise((r) => setTimeout(r, delay));
 					continue;
 				}
+				if (isContextExceeded(err)) {
+					console.warn(
+						`[RUMMY] LLM context exceeded: ${err.message.slice(0, 120)}. Returning 413.`,
+					);
+					return {
+						turn,
+						turnId: turnRow.id,
+						status: 413,
+						assembledTokens,
+						contextSize,
+					};
+				}
 				throw err;
 			}
 		}
@@ -371,15 +429,7 @@ export default class TurnExecutor {
 		});
 
 		// --- PHASE 1: RECORD ---
-		// Split lifecycle signals from action commands.
-		// Lifecycle signals (summarize, update, unknown, known) are state
-		// declarations — always recorded, never 409'd by sequential dispatch.
-		const LIFECYCLE = new Set(["summarize", "update", "unknown", "known"]);
-
 		const recorded = [];
-		const lifecycle = [];
-		const actions = [];
-
 		for (const cmd of commands) {
 			const entry = await this.#record(
 				currentRunId,
@@ -388,33 +438,19 @@ export default class TurnExecutor {
 				mode,
 				cmd,
 			);
-			if (!entry) continue;
-			recorded.push(entry);
-
-			if (LIFECYCLE.has(entry.scheme)) {
-				lifecycle.push(entry);
-			} else {
-				actions.push(entry);
-			}
+			if (entry) recorded.push(entry);
 		}
 
 		// --- PHASE 2: DISPATCH ---
+		// Sequential queue. Each tool completes before the next starts.
+		// On failure: abort remaining. On proposal: notify client, await
+		// resolution, continue.
 		let hasErrors = false;
-		let hasProposed = false;
 		let abortAfter = null;
-		const dispatched = [...lifecycle];
 
-		// Lifecycle signals first — always dispatched, never aborted.
-		for (const entry of lifecycle) {
-			await this.#hooks.tool.before.emit({ entry, rummy });
-			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
-			await this.#hooks.tool.after.emit({ entry, rummy });
-			await this.#hooks.entry.created.emit(entry);
-		}
-
-		for (const entry of actions) {
+		for (const entry of recorded) {
 			if (abortAfter) {
-				const errorMsg = `Aborted — preceding <${abortAfter}> requires resolution.`;
+				const errorMsg = `Aborted — preceding <${abortAfter}> failed.`;
 				await this.#knownStore.upsert(
 					currentRunId,
 					turn,
@@ -431,35 +467,40 @@ export default class TurnExecutor {
 			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
 			await this.#hooks.tool.after.emit({ entry, rummy });
 			await this.#hooks.entry.created.emit(entry);
-			dispatched.push(entry);
 
-			const row = await this.#db.get_entry_state.get({
-				run_id: currentRunId,
-				path: entry.resultPath || entry.path,
-			});
-			if (row?.status === 202) {
-				hasProposed = true;
-				abortAfter = entry.scheme;
-			} else if (row?.status >= 400) {
-				hasErrors = true;
-				abortAfter = entry.scheme;
-			}
-		}
+			// Materialize proposals for this entry (set revisions → 202)
+			await this.#hooks.turn.proposing.emit({ rummy, recorded: [entry] });
 
-		// Materialize proposals only if we dispatched actions
-		if (!abortAfter || hasProposed) {
-			await this.#hooks.turn.proposing.emit({ rummy, recorded: dispatched });
-		}
+			// Check for any proposals created by this entry's dispatch
+			const proposed = await this.#knownStore.getUnresolved(currentRunId);
+			for (const p of proposed) {
+				await this.#hooks.turn.proposal.emit({
+					projectId,
+					run: currentAlias,
+					proposed: [p],
+				});
+				await this.#knownStore.waitForResolution(currentRunId, p.path);
+				const resolved = await this.#db.get_entry_state.get({
+					run_id: currentRunId,
+					path: p.path,
+				});
+				if (resolved?.status >= 400) {
+					hasErrors = true;
+					abortAfter = entry.scheme;
+				}
+			}
 
-		// Recheck after materialization (set handler may create proposals)
-		if (!hasProposed && !hasErrors) {
-			for (const entry of actions) {
+			// Also check the entry itself for direct failures
+			if (!hasErrors) {
+				const entryPath = entry.resultPath || entry.path;
 				const row = await this.#db.get_entry_state.get({
 					run_id: currentRunId,
-					path: entry.resultPath || entry.path,
+					path: entryPath,
 				});
-				if (row?.status === 202) hasProposed = true;
-				if (row?.status >= 400) hasErrors = true;
+				if (row?.status >= 400) {
+					hasErrors = true;
+					abortAfter = entry.scheme;
+				}
 			}
 		}
 
@@ -468,8 +509,7 @@ export default class TurnExecutor {
 		// budget recovery phase before continuing.
 		let budgetRecovery = null;
 		// Use actual prompt_tokens from this turn's LLM response as the ground-truth
-		// token count for post-turn budget checks — more accurate than the estimate.
-		const currentPromptTokens = result.usage?.prompt_tokens ?? 0;
+		// Post-dispatch budget check — demotion handled by budget plugin
 		if (contextSize) {
 			const postMat = await this.#materializeTurnContext({
 				runId: currentRunId,
@@ -481,92 +521,31 @@ export default class TurnExecutor {
 				contextSize,
 				demoted,
 			});
-			const postBudget = await this.#hooks.budget.enforce({
+			budgetRecovery = await this.#hooks.budget.postDispatch({
 				contextSize,
 				messages: postMat.messages,
 				rows: postMat.rows,
-				lastPromptTokens: currentPromptTokens,
+				runId: currentRunId,
+				loopId: currentLoopId,
+				turn,
+				db: this.#db,
+				store: this.#knownStore,
 			});
-			if (postBudget.status === 413) {
-				// Demote this turn's data entries.
-				const demotedEntries = await this.#db.demote_turn_data_entries.all({
-					run_id: currentRunId,
-					turn,
-				});
-				const paths = demotedEntries.map((r) => r.path).join(", ");
-
-				// Also summarize the prompt — forces the model to earn it back.
-				const promptRow = postMat.rows.find((r) => r.scheme === "prompt");
-				if (promptRow) {
-					await this.#knownStore.setFidelity(
-						currentRunId,
-						promptRow.path,
-						"summary",
-					);
-				}
-
-				// Re-materialize after both demotions for accurate token count.
-				const recoveryMat = await this.#materializeTurnContext({
-					runId: currentRunId,
-					loopId: currentLoopId,
-					turn,
-					systemPrompt,
-					mode,
-					toolSet: effectiveToolSet,
-					contextSize,
-					demoted,
-				});
-				const recoveryBudget = await this.#hooks.budget.enforce({
-					contextSize,
-					messages: recoveryMat.messages,
-					rows: recoveryMat.rows,
-					lastPromptTokens: currentPromptTokens,
-				});
-				const safeLevel = Math.floor(contextSize * 0.9);
-				const tokensToFree = Math.max(
-					0,
-					recoveryBudget.assembledTokens - safeLevel,
-				);
-
-				const promptLine =
-					tokensToFree > 0
-						? `Info: Prompt auto-summarized. Full prompt restores automatically when you free ${tokensToFree} tokens.`
-						: "Info: Prompt auto-summarized. It will restore automatically.";
-				const body = [
-					"Error 413: Context Size Exceeded",
-					"",
-					"Required: YOU MUST demote larger and/or less relevant items to optimize your context.",
-					`Info: ${paths} have been automatically summarized to avoid overflow.`,
-					promptLine,
-					"Info: YOU MAY use bulk patterns to demote and promote entries by pattern.",
-					"Info: Well-designed paths and summaries improve context management.",
-					'Example: <set path="known://people/*" fidelity="summary"/>',
-				].join("\n");
-
-				await this.#knownStore.upsert(
-					currentRunId,
-					turn,
-					`budget://${currentLoopId}/${turn}`,
-					body,
-					413,
-					{ loopId: currentLoopId },
-				);
-
-				budgetRecovery = {
-					target: safeLevel,
-					promptPath: promptRow?.path ?? null,
-				};
-			}
 		}
 
-		// Lifecycle signals are always available — never 409'd.
-		const summaryEntry = lifecycle.find((e) => e.scheme === "summarize");
-		const updateEntry = lifecycle.find((e) => e.scheme === "update");
+		const summaryEntry = recorded.findLast((e) => e.scheme === "summarize");
+		const updateEntry = recorded.findLast((e) => e.scheme === "update");
 		let summaryText = summaryEntry?.body || null;
 		let updateText = updateEntry?.body || null;
 
-		// If model sent both, update wins — if it can't decide, it's not done
-		if (summaryText && updateText) summaryText = null;
+		// If model sent both, last signal wins — respects the model's final intent
+		if (summaryText && updateText) {
+			const lastLifecycle = recorded.findLast(
+				(e) => e.scheme === "summarize" || e.scheme === "update",
+			);
+			if (lastLifecycle.scheme === "summarize") updateText = null;
+			else summaryText = null;
+		}
 
 		// If model says "done" but actions failed, override — the model's
 		// assertion that it's done is false if it failed to do what it tried.
@@ -598,11 +577,7 @@ export default class TurnExecutor {
 
 		// --- Classify for return value ---
 
-		const actionCalls = recorded.filter((e) =>
-			["get", "set", "rm", "mv", "cp", "sh", "env", "search"].includes(
-				e.scheme,
-			),
-		);
+		const actionCalls = recorded.filter((e) => ACTION_SCHEMES.has(e.scheme));
 		const writeCalls = recorded.filter(
 			(e) =>
 				e.scheme === "known" ||
@@ -610,12 +585,8 @@ export default class TurnExecutor {
 		);
 		const unknownCalls = recorded.filter((e) => e.scheme === "unknown");
 
-		const hasAct = actionCalls.some((c) =>
-			["set", "rm", "sh", "mv", "cp"].includes(c.scheme),
-		);
-		const hasReads = actionCalls.some((c) =>
-			["get", "env", "search"].includes(c.scheme),
-		);
+		const hasAct = actionCalls.some((c) => MUTATION_SCHEMES.has(c.scheme));
+		const hasReads = actionCalls.some((c) => READ_SCHEMES.has(c.scheme));
 		const hasWrites = writeCalls.length > 0 || unknownCalls.length > 0;
 		const flags = { hasAct, hasReads, hasWrites };
 
@@ -651,83 +622,7 @@ export default class TurnExecutor {
 	 * Returns the recorded entry descriptor, or null if rejected/skipped.
 	 */
 	async #record(runId, loopId, turn, mode, cmd) {
-		if (mode === "ask") {
-			if (cmd.name === "sh") {
-				console.warn("[RUMMY] Rejected <sh> in ask mode");
-				return null;
-			}
-			if (cmd.name === "set" && cmd.path && cmd.body) {
-				const scheme = KnownStore.scheme(cmd.path);
-				if (scheme === null) {
-					console.warn(
-						`[RUMMY] Rejected file edit to ${cmd.path} in ${mode} mode`,
-					);
-					return null;
-				}
-			}
-			if (cmd.name === "rm" && cmd.path) {
-				const scheme = KnownStore.scheme(cmd.path);
-				if (scheme === null) {
-					console.warn(`[RUMMY] Rejected file rm of ${cmd.path} in ask mode`);
-					return null;
-				}
-			}
-			if ((cmd.name === "mv" || cmd.name === "cp") && cmd.to) {
-				const destScheme = KnownStore.scheme(cmd.to);
-				if (destScheme === null) {
-					console.warn(
-						`[RUMMY] Rejected ${cmd.name} to file ${cmd.to} in ask mode`,
-					);
-					return null;
-				}
-			}
-		}
-
 		const scheme = cmd.name;
-
-		// Structural tags — recorded like any other entry
-		if (scheme === "summarize" || scheme === "update") {
-			const statusPath = await this.#knownStore.slugPath(
-				runId,
-				scheme,
-				cmd.body,
-			);
-			await this.#knownStore.upsert(runId, turn, statusPath, cmd.body, 200, {
-				loopId,
-			});
-			return {
-				scheme,
-				body: cmd.body,
-				path: statusPath,
-				resultPath: statusPath,
-				attributes: null,
-			};
-		}
-
-		// Unknown — deduplicated, sticky
-		if (scheme === "unknown") {
-			const existingValues = await this.#knownStore.getUnknownValues(runId);
-			if (existingValues.has(cmd.body)) {
-				console.warn(`[RUMMY] Unknown deduped: "${cmd.body.slice(0, 60)}"`);
-				return null;
-			}
-			const unknownPath = await this.#knownStore.slugPath(
-				runId,
-				"unknown",
-				cmd.body,
-			);
-			await this.#knownStore.upsert(runId, turn, unknownPath, cmd.body, 200, {
-				loopId,
-			});
-			return {
-				scheme,
-				path: unknownPath,
-				body: cmd.body,
-				resultPath: unknownPath,
-				attributes: null,
-			};
-		}
-
 		const rawTarget = cmd.path || cmd.command || cmd.question || "";
 		// Reject paths that are likely reasoning bleed — too long or contain non-printing chars
 		if (rawTarget.length > 512 || /\p{Cc}/u.test(rawTarget)) {
@@ -766,108 +661,15 @@ export default class TurnExecutor {
 		const { name: _, ...attributes } = cmd;
 		if (cmd.path) attributes.path = target;
 
-		// known tool or naked write → known:// slug from body
-		if (scheme === "known" || (scheme === "set" && !cmd.path)) {
-			if (!cmd.body) return null;
-
-			// Size gate: reject entries > 512 tokens — force atomic entries
-			const entryTokens = countTokens(cmd.body);
-			const MAX_ENTRY_TOKENS = 512;
-			if (scheme === "known" && entryTokens > MAX_ENTRY_TOKENS) {
-				const rejectPath = await this.#knownStore.slugPath(
-					runId,
-					scheme,
-					cmd.body,
-				);
-				await this.#knownStore.upsert(
-					runId,
-					turn,
-					rejectPath,
-					`Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
-					413,
-					{ loopId },
-				);
-				return {
-					scheme,
-					path: rejectPath,
-					body: "",
-					resultPath: rejectPath,
-					attributes,
-					status: 413,
-				};
-			}
-
-			let knownPath = cmd.path;
-			if (!knownPath) {
-				knownPath = await this.#knownStore.slugPath(
-					runId,
-					"known",
-					cmd.body,
-					cmd.summary,
-				);
-			}
-			// Dedup: if this exact path already exists, update rather than duplicate
-			const existing = await this.#knownStore.getEntriesByPattern(
-				runId,
-				knownPath,
-				null,
-			);
-			if (existing.length > 0) {
-				// Path exists — update body and turn, skip creating a new entry
-				await this.#knownStore.upsert(
-					runId,
-					turn,
-					existing[0].path,
-					cmd.body || existing[0].body,
-					200,
-					{
-						attributes,
-						loopId,
-					},
-				);
-				return {
-					scheme: "known",
-					path: existing[0].path,
-					body: cmd.body || existing[0].body,
-					resultPath: existing[0].path,
-					attributes,
-				};
-			}
-			await this.#knownStore.upsert(runId, turn, knownPath, cmd.body, 200, {
-				attributes,
-				loopId,
-			});
-			return {
-				scheme: "known",
-				path: knownPath,
-				body: cmd.body,
-				resultPath: knownPath,
-				attributes,
-			};
-		}
-
 		const body = cmd.body || cmd.command || cmd.question || "";
 
 		// Filter: plugins can validate/transform before recording
 		const filtered = await this.#hooks.entry.recording.filter(
 			{ scheme, path: resultPath, body, attributes, status: 200 },
-			{ runId, turn, loopId },
+			{ runId, turn, loopId, mode },
 		);
 		if (filtered.status >= 400) return filtered;
 
-		// Record the entry — 200 OK, handlers change status during dispatch
-		await this.#knownStore.upsert(
-			runId,
-			turn,
-			filtered.path,
-			filtered.body,
-			200,
-			{
-				attributes: filtered.attributes,
-				loopId,
-			},
-		);
-
 		return {
 			scheme: filtered.scheme,
 			path: filtered.path,