@possumtech/rummy 0.2.8 → 0.3.1

package/src/agent/TurnExecutor.js

@@ -19,6 +19,68 @@ export default class TurnExecutor {
 		this.#knownStore = knownStore;
 	}
 
+	/**
+	 * Rebuild turn_context from v_model_context, then assemble messages.
+	 * Called at turn start and again after any fidelity demotion within the turn.
+	 */
+	async #materializeTurnContext({
+		runId,
+		loopId,
+		turn,
+		systemPrompt,
+		mode,
+		toolSet,
+		contextSize,
+		demoted,
+	}) {
+		await this.#db.clear_turn_context.run({ run_id: runId, turn });
+		const viewRows = await this.#db.get_model_context.all({ run_id: runId });
+		for (const row of viewRows) {
+			const scheme = row.scheme || "file";
+			const projectedBody = await this.#hooks.tools.view(scheme, {
+				path: row.path,
+				scheme,
+				body: row.body,
+				attributes: row.attributes ? JSON.parse(row.attributes) : null,
+				fidelity: row.fidelity,
+				category: row.category,
+			});
+			await this.#db.insert_turn_context.run({
+				run_id: runId,
+				loop_id: loopId,
+				turn,
+				ordinal: row.ordinal,
+				path: row.path,
+				fidelity: row.fidelity,
+				status: row.status,
+				body: projectedBody ?? "",
+				tokens: countTokens(projectedBody ?? ""),
+				attributes: row.attributes,
+				category: row.category,
+				source_turn: row.turn,
+			});
+		}
+		const rows = await this.#db.get_turn_context.all({ run_id: runId, turn });
+		const lastCtx = await this.#db.get_last_context_tokens.get({
+			run_id: runId,
+		});
+		const lastContextTokens = lastCtx?.context_tokens ?? 0;
+		const messages = await ContextAssembler.assembleFromTurnContext(
+			rows,
+			{
+				type: mode,
+				systemPrompt,
+				contextSize,
+				demoted,
+				toolSet,
+				lastContextTokens,
+				turn,
+			},
+			this.#hooks,
+		);
+		return { rows, messages, lastContextTokens };
+	}
+
 	async execute({
 		mode,
 		project,
@@ -28,11 +90,25 @@ export default class TurnExecutor {
 		currentLoopId,
 		requestedModel,
 		loopPrompt,
-		noContext,
+		loopIteration,
+		noRepo,
+		toolSet,
+		inRecovery = false,
 		contextSize,
 		options,
 		signal,
 	}) {
+		const RECOVERY_EXCLUDED = new Set([
+			"sh",
+			"env",
+			"search",
+			"ask_user",
+			"set",
+		]);
+		const effectiveToolSet = inRecovery
+			? new Set([...toolSet].filter((t) => !RECOVERY_EXCLUDED.has(t)))
+			: toolSet;
+
 		const turn = await this.#knownStore.nextTurn(currentRunId);
 
 		const turnRow = await this.#db.create_turn.get({
@@ -71,7 +147,8 @@ export default class TurnExecutor {
 				runId: currentRunId,
 				loopId: currentLoopId,
 				turnId: turnRow.id,
-				noContext,
+				noRepo,
+				toolSet: effectiveToolSet,
 				contextSize,
 				systemPrompt: null,
 				loopPrompt,
@@ -83,6 +160,7 @@ export default class TurnExecutor {
 			mode,
 			prompt: loopPrompt,
 			isContinuation: options?.isContinuation,
+			loopIteration,
 		});
 
 		await this.#hooks.processTurn(rummy);
@@ -109,38 +187,24 @@ export default class TurnExecutor {
 		});
 
 		// Materialize turn_context: VIEW rows projected through tools
-		await this.#db.clear_turn_context.run({ run_id: currentRunId, turn });
-		const viewRows = await this.#db.get_model_context.all({
-			run_id: currentRunId,
-		});
-		for (const row of viewRows) {
-			const scheme = row.scheme || "file";
-			const projectedBody = await this.#hooks.tools.view(scheme, {
-				path: row.path,
-				scheme,
-				body: row.body,
-				attributes: row.attributes ? JSON.parse(row.attributes) : null,
-				fidelity: row.fidelity,
-				category: row.category,
-			});
-
-			await this.#db.insert_turn_context.run({
-				run_id: currentRunId,
-				loop_id: currentLoopId,
+		const demoted = [];
+		let { rows, messages, lastContextTokens } =
+			await this.#materializeTurnContext({
+				runId: currentRunId,
+				loopId: currentLoopId,
 				turn,
-				ordinal: row.ordinal,
-				path: row.path,
-				fidelity: row.fidelity,
-				status: row.status,
-				body: projectedBody ?? "",
-				tokens: countTokens(projectedBody ?? ""),
-				attributes: row.attributes,
-				category: row.category,
-				source_turn: row.turn,
+				systemPrompt,
+				mode,
+				toolSet: effectiveToolSet,
+				contextSize,
+				demoted,
 			});
-		}
 
-		const demoted = [];
+		await this.#hooks.context.materialized.emit({
+			runId: currentRunId,
+			turn,
+			rowCount: rows.length,
+		});
 
 		await this.#hooks.run.progress.emit({
 			projectId,
@@ -149,94 +213,74 @@ export default class TurnExecutor {
 			status: "thinking",
 		});
 
-		// Assemble messages from projected system prompt + materialized turn_context
-		let rows = await this.#db.get_turn_context.all({
-			run_id: currentRunId,
-			turn,
-		});
-		let messages = await ContextAssembler.assembleFromTurnContext(
+		const budgetResult = await this.#hooks.budget.enforce({
+			contextSize,
+			messages,
 			rows,
-			{
-				type: mode,
-				systemPrompt,
-				contextSize,
-				demoted,
-			},
-			this.#hooks,
-		);
-
-		// Budget check on assembled messages (includes system prompt)
-		if (contextSize && demoted.length === 0) {
-			const assembledTokens = messages.reduce(
-				(sum, m) => sum + countTokens(m.content || ""),
-				0,
-			);
-			const ceiling = contextSize * 0.95;
-			if (assembledTokens > ceiling) {
-				const candidates = rows
-					.filter(
-						(r) =>
-							r.fidelity === "full" &&
-							r.tokens > 0 &&
-							(r.category === "file" || r.category === "known"),
-					)
-					.toSorted((a, b) => a.source_turn - b.source_turn);
-
-				let excess = assembledTokens - ceiling;
-				for (const entry of candidates) {
-					if (excess <= 0) break;
+			lastPromptTokens: lastContextTokens,
+		});
+		messages = budgetResult.messages;
+		rows = budgetResult.rows;
+		let assembledTokens =
+			budgetResult.assembledTokens ??
+			messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+
+		if (budgetResult.status === 413) {
+			if (loopIteration === 1) {
+				// Prompt Demotion: first-turn overflow — demote incoming prompt to summary
+				const promptRow = rows.findLast(
+					(r) => r.category === "prompt" && r.scheme === "prompt",
+				);
+				if (promptRow) {
 					await this.#knownStore.setFidelity(
 						currentRunId,
-						entry.path,
+						promptRow.path,
 						"summary",
 					);
-					excess -= entry.tokens;
-					demoted.push(entry.path);
 				}
-
-				if (demoted.length > 0) {
-					await this.#db.clear_turn_context.run({ run_id: currentRunId, turn });
-					const freshViewRows = await this.#db.get_model_context.all({
-						run_id: currentRunId,
-					});
-					for (const row of freshViewRows) {
-						const scheme = row.scheme || "file";
-						const projectedBody = await this.#hooks.tools.view(scheme, {
-							path: row.path,
-							scheme,
-							body: row.body,
-							attributes: row.attributes ? JSON.parse(row.attributes) : null,
-							fidelity: row.fidelity,
-							category: row.category,
-						});
-						await this.#db.insert_turn_context.run({
-							run_id: currentRunId,
-							loop_id: currentLoopId,
-							turn,
-							ordinal: row.ordinal,
-							path: row.path,
-							fidelity: row.fidelity,
-							status: row.status,
-							body: projectedBody ?? "",
-							tokens: countTokens(projectedBody ?? ""),
-							attributes: row.attributes,
-							category: row.category,
-							source_turn: row.turn,
-						});
-					}
-					rows = await this.#db.get_turn_context.all({
-						run_id: currentRunId,
+				const reMat = await this.#materializeTurnContext({
+					runId: currentRunId,
+					loopId: currentLoopId,
+					turn,
+					systemPrompt,
+					mode,
+					toolSet: effectiveToolSet,
+					contextSize,
+					demoted,
+				});
+				rows = reMat.rows;
+				messages = reMat.messages;
+				const recheck = await this.#hooks.budget.enforce({
+					contextSize,
+					messages,
+					rows,
+					lastPromptTokens: reMat.lastContextTokens,
+				});
+				messages = recheck.messages;
+				rows = recheck.rows;
+				assembledTokens =
+					recheck.assembledTokens ??
+					messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+				if (recheck.status === 413) {
+					return {
 						turn,
-					});
-					messages = await ContextAssembler.assembleFromTurnContext(
-						rows,
-						{ type: mode, systemPrompt, contextSize, demoted },
-						this.#hooks,
-					);
-					console.warn(
-						`[RUMMY] Budget exceeded: demoted ${demoted.length} entries to fit ${contextSize} token limit`,
-					);
+						turnId: turnRow.id,
+						status: 413,
+						assembledTokens,
+						contextSize,
+						overflow: recheck.overflow,
+					};
 				}
+			} else {
+				// Base context too large even without new prompt — genuine failure
+				return {
+					turn,
+					turnId: turnRow.id,
+					status: 413,
+					assembledTokens,
+					contextSize,
+					overflow: budgetResult.overflow,
+				};
 			}
 		}
 
@@ -246,14 +290,34 @@ export default class TurnExecutor {
 			runId: currentRunId,
 		});
 
-		// Store assembled messages as audit
 		// Call LLM
 		await this.#hooks.llm.request.started.emit({ model: requestedModel, turn });
-		const rawResult = await this.#llmProvider.completion(
-			filteredMessages,
-			requestedModel,
-			{ temperature: options?.temperature, signal },
-		);
+		let rawResult;
+		const isTransient = (e) =>
+			/\b(503|429|timeout|ECONNREFUSED|ECONNRESET|unavailable)\b/i.test(
+				e.message,
+			);
+
+		for (let llmAttempt = 0; ; llmAttempt++) {
+			try {
+				rawResult = await this.#llmProvider.completion(
+					filteredMessages,
+					requestedModel,
+					{ temperature: options?.temperature, signal },
+				);
+				break;
+			} catch (err) {
+				if (isTransient(err) && llmAttempt < 3) {
+					const delay = 1000 * 2 ** llmAttempt;
+					console.warn(
+						`[RUMMY] Transient LLM error (attempt ${llmAttempt + 1}/3): ${err.message.slice(0, 120)}. Retrying in ${delay}ms.`,
+					);
+					await new Promise((r) => setTimeout(r, delay));
+					continue;
+				}
+				throw err;
+			}
+		}
 		const result = await this.#hooks.llm.response.filter(rawResult, {
 			model: requestedModel,
 			projectId,
@@ -277,6 +341,19 @@ export default class TurnExecutor {
 		// Parse and emit — plugins handle audit storage
 		const { commands, unparsed } = XmlParser.parse(content);
 
+		// Ensure reasoning_content captures both API field and <think> tag
+		if (responseMessage) {
+			const thinkCmds = commands.filter((c) => c.name === "think");
+			const thinkText = thinkCmds
+				.map((c) => c.body)
+				.filter(Boolean)
+				.join("\n");
+			const apiReasoning = responseMessage.reasoning_content || "";
+			const parts = [apiReasoning, thinkText].filter(Boolean);
+			responseMessage.reasoning_content =
+				parts.length > 0 ? parts.join("\n") : null;
+		}
+
 		const systemMsg = filteredMessages.find((m) => m.role === "system");
 		const userMsg = filteredMessages.find((m) => m.role === "user");
 		await this.#hooks.turn.response.emit({
@@ -287,16 +364,21 @@ export default class TurnExecutor {
 			content,
 			commands,
 			unparsed,
+			assembledTokens,
+			contextSize,
 			systemMsg: systemMsg?.content,
 			userMsg: userMsg?.content,
 		});
 
 		// --- PHASE 1: RECORD ---
-		// Every command becomes an entry. No execution yet.
+		// Split lifecycle signals from action commands.
+		// Lifecycle signals (summarize, update, unknown, known) are state
+		// declarations — always recorded, never 409'd by sequential dispatch.
+		const LIFECYCLE = new Set(["summarize", "update", "unknown", "known"]);
 
 		const recorded = [];
-		let summaryText = null;
-		let updateText = null;
+		const lifecycle = [];
+		const actions = [];
 
 		for (const cmd of commands) {
 			const entry = await this.#record(
@@ -307,94 +389,217 @@ export default class TurnExecutor {
 				cmd,
 			);
 			if (!entry) continue;
+			recorded.push(entry);
 
-			if (entry.scheme === "summarize") summaryText = entry.body;
-			else if (entry.scheme === "update") updateText = entry.body;
-			else recorded.push(entry);
-		}
-
-		// If model sent both, summary wins
-		if (summaryText && updateText) updateText = null;
-
-		// If model sent neither, heal from content
-		let statusHealed = false;
-		if (!summaryText && !updateText) {
-			const healed = ResponseHealer.healStatus(content, commands);
-			summaryText = healed.summaryText;
-			updateText = healed.updateText;
-			statusHealed = true;
-		}
-
-		// Record healed status
-		if (summaryText) {
-			const summaryPath = await this.#knownStore.slugPath(
-				currentRunId,
-				"summarize",
-				summaryText,
-			);
-			await this.#knownStore.upsert(
-				currentRunId,
-				turn,
-				summaryPath,
-				summaryText,
-				200,
-				{ loopId: currentLoopId },
-			);
-		} else if (updateText) {
-			const updatePath = await this.#knownStore.slugPath(
-				currentRunId,
-				"update",
-				updateText,
-			);
-			await this.#knownStore.upsert(
-				currentRunId,
-				turn,
-				updatePath,
-				updateText,
-				200,
-				{ loopId: currentLoopId },
-			);
+			if (LIFECYCLE.has(entry.scheme)) {
+				lifecycle.push(entry);
+			} else {
+				actions.push(entry);
+			}
 		}
 
 		// --- PHASE 2: DISPATCH ---
-		// Handlers perform side effects: promote, demote, patch, propose.
+		let hasErrors = false;
+		let hasProposed = false;
+		let abortAfter = null;
+		const dispatched = [...lifecycle];
 
-		for (const entry of recorded) {
+		// Lifecycle signals first — always dispatched, never aborted.
+		for (const entry of lifecycle) {
+			await this.#hooks.tool.before.emit({ entry, rummy });
 			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
+			await this.#hooks.tool.after.emit({ entry, rummy });
 			await this.#hooks.entry.created.emit(entry);
 		}
 
-		// Materialize proposals (e.g. file plugin applies accumulated revisions)
-		await this.#hooks.turn.proposing.emit({ rummy, recorded });
+		for (const entry of actions) {
+			if (abortAfter) {
+				const errorMsg = `Aborted — preceding <${abortAfter}> requires resolution.`;
+				await this.#knownStore.upsert(
+					currentRunId,
+					turn,
+					entry.resultPath || entry.path,
+					errorMsg,
+					409,
+					{ attributes: { error: errorMsg }, loopId: currentLoopId },
+				);
+				hasErrors = true;
+				continue;
+			}
+
+			await this.#hooks.tool.before.emit({ entry, rummy });
+			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
+			await this.#hooks.tool.after.emit({ entry, rummy });
+			await this.#hooks.entry.created.emit(entry);
+			dispatched.push(entry);
 
-		// Check if any dispatched entries ended in error or proposed state
-		let hasErrors = false;
-		let hasProposed = false;
-		for (const entry of recorded) {
 			const row = await this.#db.get_entry_state.get({
 				run_id: currentRunId,
 				path: entry.resultPath || entry.path,
 			});
-			if (row?.status >= 400) hasErrors = true;
-			if (row?.status === 202) hasProposed = true;
+			if (row?.status === 202) {
+				hasProposed = true;
+				abortAfter = entry.scheme;
+			} else if (row?.status >= 400) {
+				hasErrors = true;
+				abortAfter = entry.scheme;
+			}
 		}
 
-		// Errors override summarize — the model thinks it's done but it's not
-		if (hasErrors && summaryText) {
-			summaryText = null;
-			updateText = "Tool errors detected — retry or investigate.";
+		// Materialize proposals only if we dispatched actions
+		if (!abortAfter || hasProposed) {
+			await this.#hooks.turn.proposing.emit({ rummy, recorded: dispatched });
+		}
+
+		// Recheck after materialization (set handler may create proposals)
+		if (!hasProposed && !hasErrors) {
+			for (const entry of actions) {
+				const row = await this.#db.get_entry_state.get({
+					run_id: currentRunId,
+					path: entry.resultPath || entry.path,
+				});
+				if (row?.status === 202) hasProposed = true;
+				if (row?.status >= 400) hasErrors = true;
+			}
 		}
 
-		// Proposals override summarize — outcome unknown until user resolves
-		if (hasProposed && summaryText) {
+		// Turn Demotion: if end-of-turn context exceeds ceiling, demote this
+		// turn's data entries and the incoming prompt to summary, then force a
+		// budget recovery phase before continuing.
+		let budgetRecovery = null;
+		// Use actual prompt_tokens from this turn's LLM response as the ground-truth
+		// token count for post-turn budget checks — more accurate than the estimate.
+		const currentPromptTokens = result.usage?.prompt_tokens ?? 0;
+		if (contextSize) {
+			const postMat = await this.#materializeTurnContext({
+				runId: currentRunId,
+				loopId: currentLoopId,
+				turn,
+				systemPrompt,
+				mode,
+				toolSet: effectiveToolSet,
+				contextSize,
+				demoted,
+			});
+			const postBudget = await this.#hooks.budget.enforce({
+				contextSize,
+				messages: postMat.messages,
+				rows: postMat.rows,
+				lastPromptTokens: currentPromptTokens,
+			});
+			if (postBudget.status === 413) {
+				// Demote this turn's data entries.
+				const demotedEntries = await this.#db.demote_turn_data_entries.all({
+					run_id: currentRunId,
+					turn,
+				});
+				const paths = demotedEntries.map((r) => r.path).join(", ");
+
+				// Also summarize the prompt — forces the model to earn it back.
+				const promptRow = postMat.rows.find((r) => r.scheme === "prompt");
+				if (promptRow) {
+					await this.#knownStore.setFidelity(
+						currentRunId,
+						promptRow.path,
+						"summary",
+					);
+				}
+
+				// Re-materialize after both demotions for accurate token count.
+				const recoveryMat = await this.#materializeTurnContext({
+					runId: currentRunId,
+					loopId: currentLoopId,
+					turn,
+					systemPrompt,
+					mode,
+					toolSet: effectiveToolSet,
+					contextSize,
+					demoted,
+				});
+				const recoveryBudget = await this.#hooks.budget.enforce({
+					contextSize,
+					messages: recoveryMat.messages,
+					rows: recoveryMat.rows,
+					lastPromptTokens: currentPromptTokens,
+				});
+				const safeLevel = Math.floor(contextSize * 0.9);
+				const tokensToFree = Math.max(
+					0,
+					recoveryBudget.assembledTokens - safeLevel,
+				);
+
+				const promptLine =
+					tokensToFree > 0
+						? `Info: Prompt auto-summarized. Full prompt restores automatically when you free ${tokensToFree} tokens.`
+						: "Info: Prompt auto-summarized. It will restore automatically.";
+				const body = [
+					"Error 413: Context Size Exceeded",
+					"",
+					"Required: YOU MUST demote larger and/or less relevant items to optimize your context.",
+					`Info: ${paths} have been automatically summarized to avoid overflow.`,
+					promptLine,
+					"Info: YOU MAY use bulk patterns to demote and promote entries by pattern.",
+					"Info: Well-designed paths and summaries improve context management.",
+					'Example: <set path="known://people/*" fidelity="summary"/>',
+				].join("\n");
+
+				await this.#knownStore.upsert(
+					currentRunId,
+					turn,
+					`budget://${currentLoopId}/${turn}`,
+					body,
+					413,
+					{ loopId: currentLoopId },
+				);
+
+				budgetRecovery = {
+					target: safeLevel,
+					promptPath: promptRow?.path ?? null,
+				};
+			}
+		}
+
+		// Lifecycle signals are always available — never 409'd.
+		const summaryEntry = lifecycle.find((e) => e.scheme === "summarize");
+		const updateEntry = lifecycle.find((e) => e.scheme === "update");
+		let summaryText = summaryEntry?.body || null;
+		let updateText = updateEntry?.body || null;
+
+		// If model sent both, update wins — if it can't decide, it's not done
+		if (summaryText && updateText) summaryText = null;
+
+		// If model says "done" but actions failed, override — the model's
+		// assertion that it's done is false if it failed to do what it tried.
+		if (summaryText && hasErrors) {
+			console.warn(
+				"[RUMMY] Overriding <summarize> — actions in this turn failed. Continuing.",
+			);
+			// Mark the recorded summarize entry as 409 so the model sees it was rejected
+			if (summaryEntry?.path) {
+				await this.#knownStore.resolve(
+					currentRunId,
+					summaryEntry.path,
+					409,
+					"Overridden — actions in this turn failed. Use <update/> until resolved.",
+				);
+			}
+			updateText = summaryText;
 			summaryText = null;
-			updateText = "Awaiting approval for proposed changes.";
+		}
+
+		// If model sent neither, heal from content
+		let statusHealed = false;
+		if (!summaryText && !updateText) {
+			const healed = ResponseHealer.healStatus(content, commands);
+			summaryText = healed.summaryText;
+			updateText = healed.updateText;
+			statusHealed = true;
 		}
 
 		// --- Classify for return value ---
 
 		const actionCalls = recorded.filter((e) =>
-			["get", "store", "set", "rm", "mv", "cp", "sh", "env", "search"].includes(
+			["get", "set", "rm", "mv", "cp", "sh", "env", "search"].includes(
 				e.scheme,
 			),
 		);
@@ -416,7 +621,7 @@ export default class TurnExecutor {
 
 		const askUserEntry = recorded.find((e) => e.scheme === "ask_user");
 
-		return {
+		const turnResult = {
 			turn,
 			turnId: turnRow.id,
 			actionCalls,
@@ -429,12 +634,16 @@ export default class TurnExecutor {
 			flags,
 			model: result.model || requestedModel,
 			modelAlias: requestedModel,
-			temperature:
-				options?.temperature ??
-				Number.parseFloat(process.env.RUMMY_TEMPERATURE || "0.7"),
+			temperature: options?.temperature,
 			contextSize,
+			assembledTokens,
 			usage: result.usage,
+			budgetRecovery,
 		};
+
+		await this.#hooks.turn.completed.emit(turnResult);
+
+		return turnResult;
 	}
 
 	/**
@@ -442,16 +651,17 @@ export default class TurnExecutor {
 	 * Returns the recorded entry descriptor, or null if rejected/skipped.
 	 */
 	async #record(runId, loopId, turn, mode, cmd) {
-		// Mode enforcement — reject prohibited commands in ask mode
 		if (mode === "ask") {
 			if (cmd.name === "sh") {
 				console.warn("[RUMMY] Rejected <sh> in ask mode");
 				return null;
 			}
-			if (cmd.name === "set" && cmd.path) {
+			if (cmd.name === "set" && cmd.path && cmd.body) {
 				const scheme = KnownStore.scheme(cmd.path);
 				if (scheme === null) {
-					console.warn(`[RUMMY] Rejected file set to ${cmd.path} in ask mode`);
+					console.warn(
+						`[RUMMY] Rejected file edit to ${cmd.path} in ${mode} mode`,
+					);
 					return null;
 				}
 			}
@@ -475,15 +685,32 @@ export default class TurnExecutor {
 
 		const scheme = cmd.name;
 
-		// Structural tags — record and return (no handler dispatch)
+		// Structural tags — recorded like any other entry
 		if (scheme === "summarize" || scheme === "update") {
-			return { scheme, body: cmd.body, resultPath: null, attributes: null };
+			const statusPath = await this.#knownStore.slugPath(
+				runId,
+				scheme,
+				cmd.body,
+			);
+			await this.#knownStore.upsert(runId, turn, statusPath, cmd.body, 200, {
+				loopId,
+			});
+			return {
+				scheme,
+				body: cmd.body,
+				path: statusPath,
+				resultPath: statusPath,
+				attributes: null,
+			};
 		}
 
 		// Unknown — deduplicated, sticky
 		if (scheme === "unknown") {
 			const existingValues = await this.#knownStore.getUnknownValues(runId);
-			if (existingValues.has(cmd.body)) return null;
+			if (existingValues.has(cmd.body)) {
+				console.warn(`[RUMMY] Unknown deduped: "${cmd.body.slice(0, 60)}"`);
+				return null;
+			}
 			const unknownPath = await this.#knownStore.slugPath(
 				runId,
 				"unknown",
@@ -502,8 +729,38 @@ export default class TurnExecutor {
 		}
 
 		const rawTarget = cmd.path || cmd.command || cmd.question || "";
+		// Reject paths that are likely reasoning bleed — too long or contain non-printing chars
+		if (rawTarget.length > 512 || /\p{Cc}/u.test(rawTarget)) {
+			const rejectPath = await this.#knownStore.dedup(
+				runId,
+				scheme,
+				`${scheme}://invalid`,
+				turn,
+			);
+			await this.#knownStore.upsert(
+				runId,
+				turn,
+				rejectPath,
+				`Invalid path: too long or contains non-printing characters`,
+				400,
+				{ loopId },
+			);
+			return {
+				scheme,
+				path: rejectPath,
+				body: "",
+				attributes: {},
+				status: 400,
+				resultPath: rejectPath,
+			};
+		}
 		const target = rawTarget;
-		const resultPath = await this.#knownStore.dedup(runId, scheme, target);
+		const resultPath = await this.#knownStore.dedup(
+			runId,
+			scheme,
+			target,
+			turn,
+		);
 
 		// Pass parsed command fields through as attributes
 		const { name: _, ...attributes } = cmd;
@@ -512,9 +769,72 @@ export default class TurnExecutor {
 		// known tool or naked write → known:// slug from body
 		if (scheme === "known" || (scheme === "set" && !cmd.path)) {
 			if (!cmd.body) return null;
-			const knownPath =
-				cmd.path || (await this.#knownStore.slugPath(runId, "known", cmd.body));
+
+			// Size gate: reject entries > 512 tokens — force atomic entries
+			const entryTokens = countTokens(cmd.body);
+			const MAX_ENTRY_TOKENS = 512;
+			if (scheme === "known" && entryTokens > MAX_ENTRY_TOKENS) {
+				const rejectPath = await this.#knownStore.slugPath(
+					runId,
+					scheme,
+					cmd.body,
+				);
+				await this.#knownStore.upsert(
+					runId,
+					turn,
+					rejectPath,
+					`Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
+					413,
+					{ loopId },
+				);
+				return {
+					scheme,
+					path: rejectPath,
+					body: "",
+					resultPath: rejectPath,
+					attributes,
+					status: 413,
+				};
+			}
+
+			let knownPath = cmd.path;
+			if (!knownPath) {
+				knownPath = await this.#knownStore.slugPath(
+					runId,
+					"known",
+					cmd.body,
+					cmd.summary,
+				);
+			}
+			// Dedup: if this exact path already exists, update rather than duplicate
+			const existing = await this.#knownStore.getEntriesByPattern(
+				runId,
+				knownPath,
+				null,
+			);
+			if (existing.length > 0) {
+				// Path exists — update body and turn, skip creating a new entry
+				await this.#knownStore.upsert(
+					runId,
+					turn,
+					existing[0].path,
+					cmd.body || existing[0].body,
+					200,
+					{
+						attributes,
+						loopId,
+					},
+				);
+				return {
+					scheme: "known",
+					path: existing[0].path,
+					body: cmd.body || existing[0].body,
+					resultPath: existing[0].path,
+					attributes,
+				};
+			}
 			await this.#knownStore.upsert(runId, turn, knownPath, cmd.body, 200, {
+				attributes,
 				loopId,
 			});
 			return {
@@ -526,20 +846,35 @@ export default class TurnExecutor {
 			};
 		}
 
-		// Record the entry — 200 OK, handlers change status during dispatch
 		const body = cmd.body || cmd.command || cmd.question || "";
-		await this.#knownStore.upsert(runId, turn, resultPath, body, 200, {
-			attributes,
-			loopId,
-		});
+
+		// Filter: plugins can validate/transform before recording
+		const filtered = await this.#hooks.entry.recording.filter(
+			{ scheme, path: resultPath, body, attributes, status: 200 },
+			{ runId, turn, loopId },
+		);
+		if (filtered.status >= 400) return filtered;
+
+		// Record the entry — 200 OK, handlers change status during dispatch
+		await this.#knownStore.upsert(
+			runId,
+			turn,
+			filtered.path,
+			filtered.body,
+			200,
+			{
+				attributes: filtered.attributes,
+				loopId,
+			},
+		);
 
 		return {
-			scheme,
-			path: resultPath,
-			body,
-			attributes,
+			scheme: filtered.scheme,
+			path: filtered.path,
+			body: filtered.body,
+			attributes: filtered.attributes,
 			status: 200,
-			resultPath,
+			resultPath: filtered.path,
 		};
 	}
 }