@possumtech/rummy 0.2.7 → 0.3.0

package/src/agent/TurnExecutor.js

@@ -25,9 +25,11 @@ export default class TurnExecutor {
 		projectId,
 		currentRunId,
 		currentAlias,
+		currentLoopId,
 		requestedModel,
 		loopPrompt,
-		noContext,
+		noRepo,
+		toolSet,
 		contextSize,
 		options,
 		signal,
@@ -36,6 +38,7 @@ export default class TurnExecutor {
 
 		const turnRow = await this.#db.create_turn.get({
 			run_id: currentRunId,
+			loop_id: currentLoopId,
 			sequence: turn,
 		});
 
@@ -67,8 +70,10 @@ export default class TurnExecutor {
 				type: mode,
 				sequence: turn,
 				runId: currentRunId,
+				loopId: currentLoopId,
 				turnId: turnRow.id,
-				noContext,
+				noRepo,
+				toolSet,
 				contextSize,
 				systemPrompt: null,
 				loopPrompt,
@@ -123,11 +128,12 @@ export default class TurnExecutor {
 
 			await this.#db.insert_turn_context.run({
 				run_id: currentRunId,
+				loop_id: currentLoopId,
 				turn,
 				ordinal: row.ordinal,
 				path: row.path,
 				fidelity: row.fidelity,
-				state: row.state,
+				status: row.status,
 				body: projectedBody ?? "",
 				tokens: countTokens(projectedBody ?? ""),
 				attributes: row.attributes,
@@ -136,6 +142,14 @@ export default class TurnExecutor {
 			});
 		}
 
+		const demoted = [];
+
+		await this.#hooks.context.materialized.emit({
+			runId: currentRunId,
+			turn,
+			rowCount: viewRows.length,
+		});
+
 		await this.#hooks.run.progress.emit({
 			projectId,
 			run: currentAlias,
@@ -143,35 +157,86 @@ export default class TurnExecutor {
 			status: "thinking",
 		});
 
-		// Assemble messages from projected system prompt + materialized turn_context
-		const rows = await this.#db.get_turn_context.all({
+		let rows = await this.#db.get_turn_context.all({
 			run_id: currentRunId,
 			turn,
 		});
-		const messages = await ContextAssembler.assembleFromTurnContext(
+		const lastCtx = await this.#db.get_last_context_tokens.get({
+			run_id: currentRunId,
+		});
+		const lastContextTokens = lastCtx?.context_tokens ?? 0;
+
+		let messages = await ContextAssembler.assembleFromTurnContext(
 			rows,
 			{
 				type: mode,
 				systemPrompt,
 				contextSize,
+				demoted,
+				toolSet,
+				lastContextTokens,
 			},
 			this.#hooks,
 		);
 
+		const budgetResult = await this.#hooks.budget.enforce({
+			contextSize,
+			messages,
+			rows,
+		});
+		messages = budgetResult.messages;
+		rows = budgetResult.rows;
+		const assembledTokens =
+			budgetResult.assembledTokens ??
+			messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+
+		// Budget overflow — return 413 to caller without calling LLM.
+		// Panic mode suppresses this — the model must run to free space.
+		if (budgetResult.status === 413 && mode !== "panic") {
+			return {
+				turn,
+				turnId: turnRow.id,
+				status: 413,
+				assembledTokens,
+				contextSize,
+				overflow: budgetResult.overflow,
+			};
+		}
+
 		const filteredMessages = await this.#hooks.llm.messages.filter(messages, {
 			model: requestedModel,
 			projectId,
 			runId: currentRunId,
 		});
 
-		// Store assembled messages as audit
 		// Call LLM
 		await this.#hooks.llm.request.started.emit({ model: requestedModel, turn });
-		const rawResult = await this.#llmProvider.completion(
-			filteredMessages,
-			requestedModel,
-			{ temperature: options?.temperature, signal },
-		);
+		let rawResult;
+		const isTransient = (e) =>
+			/\b(503|429|timeout|ECONNREFUSED|ECONNRESET|unavailable)\b/i.test(
+				e.message,
+			);
+
+		for (let llmAttempt = 0; ; llmAttempt++) {
+			try {
+				rawResult = await this.#llmProvider.completion(
+					filteredMessages,
+					requestedModel,
+					{ temperature: options?.temperature, signal },
+				);
+				break;
+			} catch (err) {
+				if (isTransient(err) && llmAttempt < 3) {
+					const delay = 1000 * 2 ** llmAttempt;
+					console.warn(
+						`[RUMMY] Transient LLM error (attempt ${llmAttempt + 1}/3): ${err.message.slice(0, 120)}. Retrying in ${delay}ms.`,
+					);
+					await new Promise((r) => setTimeout(r, delay));
+					continue;
+				}
+				throw err;
+			}
+		}
 		const result = await this.#hooks.llm.response.filter(rawResult, {
 			model: requestedModel,
 			projectId,
@@ -195,6 +260,19 @@ export default class TurnExecutor {
 		// Parse and emit — plugins handle audit storage
 		const { commands, unparsed } = XmlParser.parse(content);
 
+		// Ensure reasoning_content captures both API field and <think> tag
+		if (responseMessage) {
+			const thinkCmds = commands.filter((c) => c.name === "think");
+			const thinkText = thinkCmds
+				.map((c) => c.body)
+				.filter(Boolean)
+				.join("\n");
+			const apiReasoning = responseMessage.reasoning_content || "";
+			const parts = [apiReasoning, thinkText].filter(Boolean);
+			responseMessage.reasoning_content =
+				parts.length > 0 ? parts.join("\n") : null;
+		}
+
 		const systemMsg = filteredMessages.find((m) => m.role === "system");
 		const userMsg = filteredMessages.find((m) => m.role === "user");
 		await this.#hooks.turn.response.emit({
@@ -205,98 +283,173 @@ export default class TurnExecutor {
 			content,
 			commands,
 			unparsed,
+			assembledTokens,
+			contextSize,
 			systemMsg: systemMsg?.content,
 			userMsg: userMsg?.content,
 		});
 
 		// --- PHASE 1: RECORD ---
-		// Every command becomes an entry. No execution yet.
+		// Split lifecycle signals from action commands.
+		// Lifecycle signals (summarize, update, unknown, known) are state
+		// declarations — always recorded, never 409'd by sequential dispatch.
+		const LIFECYCLE = new Set(["summarize", "update", "unknown", "known"]);
 
 		const recorded = [];
-		let summaryText = null;
-		let updateText = null;
+		const lifecycle = [];
+		const actions = [];
 
 		for (const cmd of commands) {
-			const entry = await this.#record(currentRunId, turn, mode, cmd);
-			if (!entry) continue;
-
-			if (entry.scheme === "summarize") summaryText = entry.body;
-			else if (entry.scheme === "update") updateText = entry.body;
-			else recorded.push(entry);
-		}
-
-		// If model sent both, summary wins
-		if (summaryText && updateText) updateText = null;
-
-		// If model sent neither, heal from content
-		let statusHealed = false;
-		if (!summaryText && !updateText) {
-			const healed = ResponseHealer.healStatus(content, commands);
-			summaryText = healed.summaryText;
-			updateText = healed.updateText;
-			statusHealed = true;
-		}
-
-		// Record healed status
-		if (summaryText) {
-			const summaryPath = await this.#knownStore.slugPath(
-				currentRunId,
-				"summarize",
-				summaryText,
-			);
-			await this.#knownStore.upsert(
-				currentRunId,
-				turn,
-				summaryPath,
-				summaryText,
-				"summary",
-			);
-		} else if (updateText) {
-			const updatePath = await this.#knownStore.slugPath(
-				currentRunId,
-				"update",
-				updateText,
-			);
-			await this.#knownStore.upsert(
+			const entry = await this.#record(
 				currentRunId,
+				currentLoopId,
 				turn,
-				updatePath,
-				updateText,
-				"info",
+				mode,
+				cmd,
 			);
+			if (!entry) continue;
+			recorded.push(entry);
+
+			if (LIFECYCLE.has(entry.scheme)) {
+				lifecycle.push(entry);
+			} else {
+				actions.push(entry);
+			}
 		}
 
 		// --- PHASE 2: DISPATCH ---
-		// Handlers perform side effects: promote, demote, patch, propose.
+		// Budget plugin activates the guard on the store for dispatch.
+		const guard = this.#hooks.budget.activate(
+			this.#knownStore,
+			contextSize,
+			assembledTokens,
+		);
+		const { BudgetExceeded } = this.#hooks.budget;
 
 		let hasErrors = false;
-		for (const entry of recorded) {
-			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
-			await this.#hooks.entry.created.emit(entry);
-		}
+		let hasProposed = false;
+		let abortAfter = null;
+		const dispatched = [...lifecycle];
+
+		try {
+			// Lifecycle signals first — always dispatched, never aborted.
+			for (const entry of lifecycle) {
+				await this.#hooks.tool.before.emit({ entry, rummy });
+				await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
+				await this.#hooks.tool.after.emit({ entry, rummy });
+				await this.#hooks.entry.created.emit(entry);
+			}
 
-		// Materialize proposals (e.g. file plugin applies accumulated revisions)
-		await this.#hooks.turn.proposing.emit({ rummy, recorded });
+			for (const entry of actions) {
+				if (abortAfter || guard.isTripped) {
+					await this.#knownStore.upsert(
+						currentRunId,
+						turn,
+						entry.resultPath || entry.path,
+						"",
+						guard.isTripped ? 413 : 409,
+						{
+							attributes: {
+								error: guard.isTripped
+									? `Budget exceeded by <${guard.tripSource}>.`
+									: `Aborted — preceding <${abortAfter}> requires resolution.`,
+							},
+							loopId: currentLoopId,
+						},
+					);
+					hasErrors = true;
+					continue;
+				}
 
-		// Check if any dispatched entries ended in error state
-		for (const entry of recorded) {
-			const row = await this.#db.get_entry_state.get({
-				run_id: currentRunId,
-				path: entry.resultPath || entry.path,
-			});
-			if (row?.state === "error") hasErrors = true;
+				try {
+					await this.#hooks.tool.before.emit({ entry, rummy });
+					await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
+					await this.#hooks.tool.after.emit({ entry, rummy });
+					await this.#hooks.entry.created.emit(entry);
+					dispatched.push(entry);
+				} catch (err) {
+					if (err instanceof BudgetExceeded) {
+						guard.trip(entry.scheme);
+						await this.#knownStore.upsert(
+							currentRunId,
+							turn,
+							entry.resultPath || entry.path,
+							`Budget exceeded: ${err.requested} tokens requested, ${err.remaining} remaining.`,
+							413,
+							{ attributes: { error: err.message }, loopId: currentLoopId },
+						);
+						hasErrors = true;
+						abortAfter = entry.scheme;
+						continue;
+					}
+					throw err;
+				}
+
+				const row = await this.#db.get_entry_state.get({
+					run_id: currentRunId,
+					path: entry.resultPath || entry.path,
+				});
+				if (row?.status === 202) {
+					hasProposed = true;
+					abortAfter = entry.scheme;
+				} else if (row?.status >= 400) {
+					hasErrors = true;
+					abortAfter = entry.scheme;
+				}
+			}
+
+			// Materialize proposals only if we dispatched actions
+			if (!abortAfter || hasProposed) {
+				await this.#hooks.turn.proposing.emit({ rummy, recorded: dispatched });
+			}
+
+			// Recheck after materialization (set handler may create proposals)
+			if (!hasProposed && !hasErrors) {
+				for (const entry of actions) {
+					const row = await this.#db.get_entry_state.get({
+						run_id: currentRunId,
+						path: entry.resultPath || entry.path,
+					});
+					if (row?.status === 202) hasProposed = true;
+					if (row?.status >= 400) hasErrors = true;
+				}
+			}
+		} finally {
+			this.#hooks.budget.deactivate(this.#knownStore);
 		}
 
-		// Errors override summarize — the model thinks it's done but it's not
-		if (hasErrors && summaryText) {
+		// Lifecycle signals are always available — never 409'd.
+		const summaryEntry = lifecycle.find((e) => e.scheme === "summarize");
+		const updateEntry = lifecycle.find((e) => e.scheme === "update");
+		let summaryText = summaryEntry?.body || null;
+		let updateText = updateEntry?.body || null;
+
+		// If model sent both, update wins — if it can't decide, it's not done
+		if (summaryText && updateText) summaryText = null;
+
+		// If model says "done" but actions failed, override — the model's
+		// assertion that it's done is false if it failed to do what it tried.
+		if (summaryText && hasErrors) {
+			console.warn(
+				"[RUMMY] Overriding <summarize> — actions in this turn failed. Continuing.",
+			);
+			updateText = summaryText;
 			summaryText = null;
-			updateText = "Tool errors detected — retry or investigate.";
+		}
+
+		// If model sent neither, heal from content
+		let statusHealed = false;
+		if (!summaryText && !updateText) {
+			const healed = ResponseHealer.healStatus(content, commands);
+			summaryText = healed.summaryText;
+			updateText = healed.updateText;
+			statusHealed = true;
 		}
 
 		// --- Classify for return value ---
 
 		const actionCalls = recorded.filter((e) =>
-			["get", "store", "set", "rm", "mv", "cp", "sh", "env", "search"].includes(
+			["get", "set", "rm", "mv", "cp", "sh", "env", "search"].includes(
 				e.scheme,
 			),
 		);
@@ -318,7 +471,7 @@ export default class TurnExecutor {
 
 		const askUserEntry = recorded.find((e) => e.scheme === "ask_user");
 
-		return {
+		const turnResult = {
 			turn,
 			turnId: turnRow.id,
 			actionCalls,
@@ -335,25 +488,32 @@ export default class TurnExecutor {
 				options?.temperature ??
 				Number.parseFloat(process.env.RUMMY_TEMPERATURE || "0.7"),
 			contextSize,
+			assembledTokens,
 			usage: result.usage,
 		};
+
+		await this.#hooks.turn.completed.emit(turnResult);
+
+		return turnResult;
 	}
 
 	/**
 	 * Record a parsed command as a known_entries row.
 	 * Returns the recorded entry descriptor, or null if rejected/skipped.
 	 */
-	async #record(runId, turn, mode, cmd) {
-		// Mode enforcement — reject prohibited commands in ask mode
-		if (mode === "ask") {
+	async #record(runId, loopId, turn, mode, cmd) {
+		// Mode enforcement — reject prohibited commands in ask/panic mode
+		if (mode === "ask" || mode === "panic") {
 			if (cmd.name === "sh") {
 				console.warn("[RUMMY] Rejected <sh> in ask mode");
 				return null;
 			}
-			if (cmd.name === "set" && cmd.path) {
+			if (cmd.name === "set" && cmd.path && cmd.body) {
 				const scheme = KnownStore.scheme(cmd.path);
 				if (scheme === null) {
-					console.warn(`[RUMMY] Rejected file set to ${cmd.path} in ask mode`);
+					console.warn(
+						`[RUMMY] Rejected file edit to ${cmd.path} in ${mode} mode`,
+					);
 					return null;
 				}
 			}
@@ -377,21 +537,40 @@ export default class TurnExecutor {
 
 		const scheme = cmd.name;
 
-		// Structural tags — record and return (no handler dispatch)
+		// Structural tags — recorded like any other entry
 		if (scheme === "summarize" || scheme === "update") {
-			return { scheme, body: cmd.body, resultPath: null, attributes: null };
+			const statusPath = await this.#knownStore.slugPath(
+				runId,
+				scheme,
+				cmd.body,
+			);
+			await this.#knownStore.upsert(runId, turn, statusPath, cmd.body, 200, {
+				loopId,
+			});
+			return {
+				scheme,
+				body: cmd.body,
+				path: statusPath,
+				resultPath: statusPath,
+				attributes: null,
+			};
 		}
 
 		// Unknown — deduplicated, sticky
 		if (scheme === "unknown") {
 			const existingValues = await this.#knownStore.getUnknownValues(runId);
-			if (existingValues.has(cmd.body)) return null;
+			if (existingValues.has(cmd.body)) {
+				console.warn(`[RUMMY] Unknown deduped: "${cmd.body.slice(0, 60)}"`);
+				return null;
+			}
 			const unknownPath = await this.#knownStore.slugPath(
 				runId,
 				"unknown",
 				cmd.body,
 			);
-			await this.#knownStore.upsert(runId, turn, unknownPath, cmd.body, "full");
+			await this.#knownStore.upsert(runId, turn, unknownPath, cmd.body, 200, {
+				loopId,
+			});
 			return {
 				scheme,
 				path: unknownPath,
@@ -401,15 +580,14 @@ export default class TurnExecutor {
 			};
 		}
 
-		// Normalize path — encode spaces in scheme:// paths
 		const rawTarget = cmd.path || cmd.command || cmd.question || "";
-		const target = rawTarget.includes("://")
-			? rawTarget.replace(
-					/:\/\/(.*)$/,
-					(_, rest) => `://${encodeURIComponent(decodeURIComponent(rest))}`,
-				)
-			: rawTarget;
-		const resultPath = await this.#knownStore.dedup(runId, scheme, target);
+		const target = rawTarget;
+		const resultPath = await this.#knownStore.dedup(
+			runId,
+			scheme,
+			target,
+			turn,
+		);
 
 		// Pass parsed command fields through as attributes
 		const { name: _, ...attributes } = cmd;
@@ -418,9 +596,72 @@ export default class TurnExecutor {
 		// known tool or naked write → known:// slug from body
 		if (scheme === "known" || (scheme === "set" && !cmd.path)) {
 			if (!cmd.body) return null;
-			const knownPath =
-				cmd.path || (await this.#knownStore.slugPath(runId, "known", cmd.body));
-			await this.#knownStore.upsert(runId, turn, knownPath, cmd.body, "full");
+
+			// Size gate: reject entries > 512 tokens — force atomic entries
+			const entryTokens = countTokens(cmd.body);
+			const MAX_ENTRY_TOKENS = 512;
+			if (scheme === "known" && entryTokens > MAX_ENTRY_TOKENS) {
+				const rejectPath = await this.#knownStore.slugPath(
+					runId,
+					scheme,
+					cmd.body,
+				);
+				await this.#knownStore.upsert(
+					runId,
+					turn,
+					rejectPath,
+					`Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
+					413,
+					{ loopId },
+				);
+				return {
+					scheme,
+					path: rejectPath,
+					body: "",
+					resultPath: rejectPath,
+					attributes,
+					status: 413,
+				};
+			}
+
+			let knownPath = cmd.path;
+			if (!knownPath) {
+				knownPath = await this.#knownStore.slugPath(
+					runId,
+					"known",
+					cmd.body,
+					cmd.summary,
+				);
+			}
+			// Dedup: if this exact path already exists, update rather than duplicate
+			const existing = await this.#knownStore.getEntriesByPattern(
+				runId,
+				knownPath,
+				null,
+			);
+			if (existing.length > 0) {
+				// Path exists — update body and turn, skip creating a new entry
+				await this.#knownStore.upsert(
+					runId,
+					turn,
+					existing[0].path,
+					cmd.body || existing[0].body,
+					200,
+					{
+						loopId,
+					},
+				);
+				return {
+					scheme: "known",
+					path: existing[0].path,
+					body: cmd.body || existing[0].body,
+					resultPath: existing[0].path,
+					attributes,
+				};
+			}
+			await this.#knownStore.upsert(runId, turn, knownPath, cmd.body, 200, {
+				loopId,
+			});
 			return {
 				scheme: "known",
 				path: knownPath,
@@ -430,28 +671,65 @@ export default class TurnExecutor {
 			};
 		}
 
-		// Record the entry
 		const body = cmd.body || cmd.command || cmd.question || "";
-		const state = this.#initialState(scheme);
-		await this.#knownStore.upsert(runId, turn, resultPath, body, state, {
-			attributes,
-		});
+
+		// Filter: plugins can validate/transform before recording
+		const filtered = await this.#hooks.entry.recording.filter(
+			{ scheme, path: resultPath, body, attributes, status: 200 },
+			{ runId, turn, loopId },
+		);
+		if (filtered.status >= 400) return filtered;
+
+		// Record the entry — 200 OK, handlers change status during dispatch
+		await this.#knownStore.upsert(
+			runId,
+			turn,
+			filtered.path,
+			filtered.body,
+			200,
+			{
+				attributes: filtered.attributes,
+				loopId,
+			},
+		);
 
 		return {
-			scheme,
-			path: resultPath,
-			body,
-			attributes,
-			state,
-			resultPath,
+			scheme: filtered.scheme,
+			path: filtered.path,
+			body: filtered.body,
+			attributes: filtered.attributes,
+			status: 200,
+			resultPath: filtered.path,
 		};
 	}
 
-	/**
-	 * Initial state for a recorded command entry.
-	 * All entries start at "full". Handlers change state during dispatch.
-	 */
-	#initialState(_scheme) {
-		return "full";
+	async #rematerialize(runId, loopId, turn) {
+		await this.#db.clear_turn_context.run({ run_id: runId, turn });
+		const viewRows = await this.#db.get_model_context.all({ run_id: runId });
+		for (const row of viewRows) {
+			const scheme = row.scheme || "file";
+			const projectedBody = await this.#hooks.tools.view(scheme, {
+				path: row.path,
+				scheme,
+				body: row.body,
+				attributes: row.attributes ? JSON.parse(row.attributes) : null,
+				fidelity: row.fidelity,
+				category: row.category,
+			});
+			await this.#db.insert_turn_context.run({
+				run_id: runId,
+				loop_id: loopId,
+				turn,
+				ordinal: row.ordinal,
+				path: row.path,
+				fidelity: row.fidelity,
+				status: row.status,
+				body: projectedBody ?? "",
+				tokens: countTokens(projectedBody ?? ""),
+				attributes: row.attributes,
+				category: row.category,
+				source_turn: row.turn,
+			});
+		}
 	}
 }