@possumtech/rummy 0.2.6 → 0.2.8

package/src/agent/TurnExecutor.js

@@ -25,6 +25,7 @@ export default class TurnExecutor {
 		projectId,
 		currentRunId,
 		currentAlias,
+		currentLoopId,
 		requestedModel,
 		loopPrompt,
 		noContext,
@@ -36,6 +37,7 @@ export default class TurnExecutor {
 
 		const turnRow = await this.#db.create_turn.get({
 			run_id: currentRunId,
+			loop_id: currentLoopId,
 			sequence: turn,
 		});
 
@@ -67,6 +69,7 @@ export default class TurnExecutor {
 				type: mode,
 				sequence: turn,
 				runId: currentRunId,
+				loopId: currentLoopId,
 				turnId: turnRow.id,
 				noContext,
 				contextSize,
@@ -123,11 +126,12 @@ export default class TurnExecutor {
 
 			await this.#db.insert_turn_context.run({
 				run_id: currentRunId,
+				loop_id: currentLoopId,
 				turn,
 				ordinal: row.ordinal,
 				path: row.path,
 				fidelity: row.fidelity,
-				state: row.state,
+				status: row.status,
 				body: projectedBody ?? "",
 				tokens: countTokens(projectedBody ?? ""),
 				attributes: row.attributes,
@@ -136,6 +140,8 @@ export default class TurnExecutor {
 			});
 		}
 
+		const demoted = [];
+
 		await this.#hooks.run.progress.emit({
 			projectId,
 			run: currentAlias,
@@ -144,20 +150,96 @@ export default class TurnExecutor {
 		});
 
 		// Assemble messages from projected system prompt + materialized turn_context
-		const rows = await this.#db.get_turn_context.all({
+		let rows = await this.#db.get_turn_context.all({
 			run_id: currentRunId,
 			turn,
 		});
-		const messages = await ContextAssembler.assembleFromTurnContext(
+		let messages = await ContextAssembler.assembleFromTurnContext(
 			rows,
 			{
 				type: mode,
 				systemPrompt,
 				contextSize,
+				demoted,
 			},
 			this.#hooks,
 		);
 
+		// Budget check on assembled messages (includes system prompt)
+		if (contextSize && demoted.length === 0) {
+			const assembledTokens = messages.reduce(
+				(sum, m) => sum + countTokens(m.content || ""),
+				0,
+			);
+			const ceiling = contextSize * 0.95;
+			if (assembledTokens > ceiling) {
+				const candidates = rows
+					.filter(
+						(r) =>
+							r.fidelity === "full" &&
+							r.tokens > 0 &&
+							(r.category === "file" || r.category === "known"),
+					)
+					.toSorted((a, b) => a.source_turn - b.source_turn);
+
+				let excess = assembledTokens - ceiling;
+				for (const entry of candidates) {
+					if (excess <= 0) break;
+					await this.#knownStore.setFidelity(
+						currentRunId,
+						entry.path,
+						"summary",
+					);
+					excess -= entry.tokens;
+					demoted.push(entry.path);
+				}
+
+				if (demoted.length > 0) {
+					await this.#db.clear_turn_context.run({ run_id: currentRunId, turn });
+					const freshViewRows = await this.#db.get_model_context.all({
+						run_id: currentRunId,
+					});
+					for (const row of freshViewRows) {
+						const scheme = row.scheme || "file";
+						const projectedBody = await this.#hooks.tools.view(scheme, {
+							path: row.path,
+							scheme,
+							body: row.body,
+							attributes: row.attributes ? JSON.parse(row.attributes) : null,
+							fidelity: row.fidelity,
+							category: row.category,
+						});
+						await this.#db.insert_turn_context.run({
+							run_id: currentRunId,
+							loop_id: currentLoopId,
+							turn,
+							ordinal: row.ordinal,
+							path: row.path,
+							fidelity: row.fidelity,
+							status: row.status,
+							body: projectedBody ?? "",
+							tokens: countTokens(projectedBody ?? ""),
+							attributes: row.attributes,
+							category: row.category,
+							source_turn: row.turn,
+						});
+					}
+					rows = await this.#db.get_turn_context.all({
+						run_id: currentRunId,
+						turn,
+					});
+					messages = await ContextAssembler.assembleFromTurnContext(
+						rows,
+						{ type: mode, systemPrompt, contextSize, demoted },
+						this.#hooks,
+					);
+					console.warn(
+						`[RUMMY] Budget exceeded: demoted ${demoted.length} entries to fit ${contextSize} token limit`,
+					);
+				}
+			}
+		}
+
 		const filteredMessages = await this.#hooks.llm.messages.filter(messages, {
 			model: requestedModel,
 			projectId,
@@ -217,7 +299,13 @@ export default class TurnExecutor {
 		let updateText = null;
 
 		for (const cmd of commands) {
-			const entry = await this.#record(currentRunId, turn, mode, cmd);
+			const entry = await this.#record(
+				currentRunId,
+				currentLoopId,
+				turn,
+				mode,
+				cmd,
+			);
 			if (!entry) continue;
 
 			if (entry.scheme === "summarize") summaryText = entry.body;
@@ -249,7 +337,8 @@ export default class TurnExecutor {
 				turn,
 				summaryPath,
 				summaryText,
-				"summary",
+				200,
+				{ loopId: currentLoopId },
 			);
 		} else if (updateText) {
 			const updatePath = await this.#knownStore.slugPath(
@@ -262,14 +351,14 @@ export default class TurnExecutor {
 				turn,
 				updatePath,
 				updateText,
-				"info",
+				200,
+				{ loopId: currentLoopId },
 			);
 		}
 
 		// --- PHASE 2: DISPATCH ---
 		// Handlers perform side effects: promote, demote, patch, propose.
 
-		let hasErrors = false;
 		for (const entry of recorded) {
 			await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
 			await this.#hooks.entry.created.emit(entry);
@@ -278,13 +367,16 @@ export default class TurnExecutor {
 		// Materialize proposals (e.g. file plugin applies accumulated revisions)
 		await this.#hooks.turn.proposing.emit({ rummy, recorded });
 
-		// Check if any dispatched entries ended in error state
+		// Check if any dispatched entries ended in error or proposed state
+		let hasErrors = false;
+		let hasProposed = false;
 		for (const entry of recorded) {
 			const row = await this.#db.get_entry_state.get({
 				run_id: currentRunId,
 				path: entry.resultPath || entry.path,
 			});
-			if (row?.state === "error") hasErrors = true;
+			if (row?.status >= 400) hasErrors = true;
+			if (row?.status === 202) hasProposed = true;
 		}
 
 		// Errors override summarize — the model thinks it's done but it's not
@@ -293,6 +385,12 @@ export default class TurnExecutor {
 			updateText = "Tool errors detected — retry or investigate.";
 		}
 
+		// Proposals override summarize — outcome unknown until user resolves
+		if (hasProposed && summaryText) {
+			summaryText = null;
+			updateText = "Awaiting approval for proposed changes.";
+		}
+
 		// --- Classify for return value ---
 
 		const actionCalls = recorded.filter((e) =>
@@ -343,7 +441,7 @@ export default class TurnExecutor {
 	 * Record a parsed command as a known_entries row.
 	 * Returns the recorded entry descriptor, or null if rejected/skipped.
 	 */
-	async #record(runId, turn, mode, cmd) {
+	async #record(runId, loopId, turn, mode, cmd) {
 		// Mode enforcement — reject prohibited commands in ask mode
 		if (mode === "ask") {
 			if (cmd.name === "sh") {
@@ -391,7 +489,9 @@ export default class TurnExecutor {
 				"unknown",
 				cmd.body,
 			);
-			await this.#knownStore.upsert(runId, turn, unknownPath, cmd.body, "full");
+			await this.#knownStore.upsert(runId, turn, unknownPath, cmd.body, 200, {
+				loopId,
+			});
 			return {
 				scheme,
 				path: unknownPath,
@@ -401,14 +501,8 @@ export default class TurnExecutor {
 			};
 		}
 
-		// Normalize path — encode spaces in scheme:// paths
 		const rawTarget = cmd.path || cmd.command || cmd.question || "";
-		const target = rawTarget.includes("://")
-			? rawTarget.replace(
-					/:\/\/(.*)$/,
-					(_, rest) => `://${encodeURIComponent(decodeURIComponent(rest))}`,
-				)
-			: rawTarget;
+		const target = rawTarget;
 		const resultPath = await this.#knownStore.dedup(runId, scheme, target);
 
 		// Pass parsed command fields through as attributes
@@ -420,7 +514,9 @@ export default class TurnExecutor {
 			if (!cmd.body) return null;
 			const knownPath =
 				cmd.path || (await this.#knownStore.slugPath(runId, "known", cmd.body));
-			await this.#knownStore.upsert(runId, turn, knownPath, cmd.body, "full");
+			await this.#knownStore.upsert(runId, turn, knownPath, cmd.body, 200, {
+				loopId,
+			});
 			return {
 				scheme: "known",
 				path: knownPath,
@@ -430,11 +526,11 @@ export default class TurnExecutor {
 			};
 		}
 
-		// Record the entry
+		// Record the entry — 200 OK, handlers change status during dispatch
 		const body = cmd.body || cmd.command || cmd.question || "";
-		const state = this.#initialState(scheme);
-		await this.#knownStore.upsert(runId, turn, resultPath, body, state, {
+		await this.#knownStore.upsert(runId, turn, resultPath, body, 200, {
 			attributes,
+			loopId,
 		});
 
 		return {
@@ -442,16 +538,8 @@ export default class TurnExecutor {
 			path: resultPath,
 			body,
 			attributes,
-			state,
+			status: 200,
 			resultPath,
 		};
 	}
-
-	/**
-	 * Initial state for a recorded command entry.
-	 * All entries start at "full". Handlers change state during dispatch.
-	 */
-	#initialState(_scheme) {
-		return "full";
-	}
 }

package/src/agent/XmlParser.js

@@ -190,6 +190,16 @@ export default class XmlParser {
 						return;
 					}
 
+					// Known tool opened while another is still open — close the old one.
+					if (current) {
+						warnings.push(
+							`Unclosed <${current.name}> before <${name}> — recovered`,
+						);
+						commands.push(
+							resolveCommand(current.name, current.attrs, current.rawBody),
+						);
+					}
+
 					current = { name, attrs, rawBody: "" };
 				},
 
@@ -210,6 +220,16 @@ export default class XmlParser {
 							resolveCommand(current.name, current.attrs, current.rawBody),
 						);
 						current = null;
+					} else if (current && ALL_TOOLS.has(name)) {
+						// Mismatched close tag for a known tool — close current tag,
+						// don't swallow subsequent commands as body text.
+						warnings.push(
+							`Mismatched </${name}> closing <${current.name}> — recovered`,
+						);
+						commands.push(
+							resolveCommand(current.name, current.attrs, current.rawBody),
+						);
+						current = null;
 					} else if (current) {
 						current.rawBody += `</${name}>`;
 					} else if (isImplied && ALL_TOOLS.has(name)) {

package/src/agent/known_checks.sql

@@ -17,25 +17,26 @@ SELECT path, body, attributes, turn
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND state = 'proposed';
+	AND status = 202;
 
 -- PREP: has_rejections
 SELECT COUNT(*) AS count
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND state = 'rejected';
+	AND loop_id = :loop_id
+	AND status = 403;
 
 -- PREP: has_accepted_actions
 SELECT COUNT(*) AS count
 FROM known_entries
 WHERE
 	run_id = :run_id
-	AND state = 'pass'
+	AND status = 200
 	AND scheme IN ('set', 'sh', 'rm', 'mv', 'cp');
 
 -- PREP: get_file_entries
-SELECT path, state, hash, updated_at
+SELECT path, status, fidelity, hash, updated_at
 FROM known_entries
 WHERE
 	run_id = :run_id

package/src/agent/known_queries.sql

@@ -1,5 +1,5 @@
 -- PREP: get_known_entries
-SELECT path, scheme, state, body, turn, hash, attributes
+SELECT path, scheme, status, fidelity, body, turn, hash, attributes
 FROM known_entries
 WHERE run_id = :run_id
 ORDER BY path;
@@ -18,7 +18,7 @@ WHERE
 ORDER BY id;
 
 -- PREP: get_turn_audit
-SELECT path, scheme, state, turn, body, attributes
+SELECT path, scheme, status, fidelity, turn, body, attributes
 FROM known_entries
 WHERE
 	run_id = :run_id
@@ -57,12 +57,13 @@ SELECT body
 FROM known_entries
 WHERE
 	run_id = :run_id
+	AND loop_id = :loop_id
 	AND scheme = 'summarize'
 ORDER BY id DESC
 LIMIT 1;
 
 -- PREP: get_history
-SELECT ke.path, ke.state AS status, ke.body, ke.attributes, ke.turn
+SELECT ke.path, ke.status, ke.body, ke.attributes, ke.turn
 FROM known_entries AS ke
 JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
 WHERE

package/src/agent/known_store.sql

@@ -1,19 +1,22 @@
 -- PREP: upsert_known_entry
 INSERT INTO known_entries (
-	run_id, turn, path, body, state, hash, attributes
-	, tokens, tokens_full, updated_at
+	run_id, loop_id, turn, path, body, status, fidelity, hash
+	, attributes, tokens, tokens_full, updated_at
 )
 VALUES (
-	:run_id, :turn, :path, :body, :state, :hash, COALESCE(:attributes, '{}')
+	:run_id, :loop_id, :turn, :path, :body, :status, :fidelity, :hash
+	, COALESCE(:attributes, '{}')
 	, countTokens(:body)
 	, countTokens(:body)
 	, COALESCE(:updated_at, CURRENT_TIMESTAMP)
 )
 ON CONFLICT (run_id, path) DO UPDATE SET
 	body = excluded.body
-	, state = excluded.state
+	, status = excluded.status
+	, fidelity = excluded.fidelity
 	, hash = COALESCE(excluded.hash, known_entries.hash)
 	, attributes = COALESCE(excluded.attributes, known_entries.attributes)
+	, loop_id = excluded.loop_id
 	, turn = excluded.turn
 	, tokens = countTokens(excluded.body)
 	, tokens_full = countTokens(excluded.body)
@@ -43,17 +46,17 @@ WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
 -- PREP: resolve_known_entry
 UPDATE known_entries
 SET
-	state = :state
+	status = :status
 	, body = :body
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
--- PREP: set_file_state
+-- PREP: set_file_fidelity
 UPDATE known_entries
 SET
-	state = :state
+	fidelity = :fidelity
 	, tokens = CASE
-		WHEN :state = 'summary' THEN countTokens(body)
+		WHEN :fidelity = 'summary' THEN countTokens(body)
 		ELSE tokens_full
 	END
 	, updated_at = CURRENT_TIMESTAMP
@@ -62,7 +65,7 @@ WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
 -- PREP: promote_path
 UPDATE known_entries
 SET
-	state = 'full'
+	fidelity = 'full'
 	, turn = :turn
 	, tokens = tokens_full
 	, updated_at = CURRENT_TIMESTAMP
@@ -71,23 +74,34 @@ WHERE run_id = :run_id AND path = :path;
 -- PREP: demote_path
 UPDATE known_entries
 SET
-	state = 'stored'
+	fidelity = 'stored'
 	, tokens = 0
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
+-- PREP: set_fidelity
+UPDATE known_entries
+SET
+	fidelity = :fidelity
+	, tokens = CASE
+		WHEN :fidelity = 'stored' THEN 0
+		ELSE countTokens(body)
+	END
+	, updated_at = CURRENT_TIMESTAMP
+WHERE run_id = :run_id AND path = :path;
+
 -- PREP: get_entry_body
 SELECT body
 FROM known_entries
 WHERE run_id = :run_id AND path = :path;
 
 -- PREP: get_entry_state
-SELECT state, scheme, turn
+SELECT status, fidelity, scheme, turn
 FROM known_entries
 WHERE run_id = :run_id AND path = :path;
 
 -- PREP: get_file_states_by_pattern
-SELECT path, state, turn
+SELECT path, status, fidelity, turn
 FROM known_entries
 WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL
 ORDER BY path;
@@ -107,7 +121,7 @@ WHERE run_id = :run_id AND path = :path;
 -- PREP: promote_by_pattern
 UPDATE known_entries
 SET
-	state = 'full'
+	fidelity = 'full'
 	, turn = :turn
 	, tokens = tokens_full
 	, updated_at = CURRENT_TIMESTAMP
@@ -119,7 +133,7 @@ WHERE
 -- PREP: demote_by_pattern
 UPDATE known_entries
 SET
-	state = 'stored'
+	fidelity = 'stored'
 	, tokens = 0
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
@@ -128,7 +142,7 @@ WHERE
 	AND (:body IS NULL OR hedsearch(:body, body));
 
 -- PREP: get_entries_by_pattern
-SELECT path, body, scheme, state, tokens_full, attributes
+SELECT path, body, scheme, status, fidelity, tokens_full, attributes
 FROM known_entries
 WHERE
 	run_id = :run_id

package/src/agent/loops.sql

@@ -0,0 +1,63 @@
+-- PREP: enqueue_loop
+INSERT INTO loops (run_id, sequence, mode, model, prompt, config)
+VALUES (:run_id, :sequence, :mode, :model, :prompt, :config)
+RETURNING id, sequence;
+
+-- PREP: next_loop
+UPDATE runs
+SET next_loop = next_loop + 1
+WHERE id = :run_id
+RETURNING next_loop - 1 AS sequence;
+
+-- PREP: claim_next_loop
+UPDATE loops
+SET status = 102
+WHERE
+	id = (
+		SELECT
+			id
+		FROM loops
+		WHERE run_id = :run_id AND status = 100
+		ORDER BY id
+		LIMIT 1
+	)
+RETURNING id, run_id, sequence, mode, model, prompt, config;
+
+-- PREP: complete_loop
+UPDATE loops
+SET status = :status, result = :result
+WHERE id = :id;
+
+-- PREP: abort_active_loop
+UPDATE loops
+SET status = 499
+WHERE run_id = :run_id AND status = 102;
+
+-- PREP: get_pending_loops
+SELECT id, sequence, mode, model, prompt, status, created_at
+FROM loops
+WHERE run_id = :run_id AND status IN (100, 102)
+ORDER BY id;
+
+-- PREP: reset_active_loops
+UPDATE loops
+SET status = 100
+WHERE status = 102;
+
+-- PREP: get_current_loop
+SELECT id, sequence, mode, model, prompt, status
+FROM loops
+WHERE run_id = :run_id AND status = 102
+LIMIT 1;
+
+-- PREP: get_loop_by_id
+SELECT id, run_id, sequence, mode, model, prompt, status, config
+FROM loops
+WHERE id = :id;
+
+-- PREP: get_latest_completed_loop
+SELECT id, sequence, mode, status
+FROM loops
+WHERE run_id = :run_id AND status IN (200, 500)
+ORDER BY id DESC
+LIMIT 1;

package/src/agent/runs.sql

@@ -22,14 +22,14 @@ RETURNING id;
 -- PREP: get_run_by_alias
 SELECT
 	id, project_id, parent_run_id, model, status, alias
-	, temperature, persona, context_limit, next_turn, created_at
+	, temperature, persona, context_limit, next_turn, next_loop, created_at
 FROM runs
 WHERE alias = :alias;
 
 -- PREP: get_run_by_id
 SELECT
 	id, project_id, parent_run_id, model, status, alias
-	, temperature, persona, context_limit, next_turn, created_at
+	, temperature, persona, context_limit, next_turn, next_loop, created_at
 FROM runs
 WHERE id = :id;
 
@@ -80,11 +80,11 @@ RETURNING next_turn - 1 AS turn;
 
 -- PREP: fork_known_entries
 INSERT INTO known_entries (
-	run_id, turn, path, body, state
+	run_id, loop_id, turn, path, body, status, fidelity
 	, hash, attributes, tokens, tokens_full, refs, write_count
 )
 SELECT
-	:new_run_id, turn, path, body, state
+	:new_run_id, NULL, turn, path, body, status, fidelity
 	, hash, attributes, tokens, tokens_full, refs, write_count
 FROM known_entries
 WHERE run_id = :parent_run_id;
@@ -94,7 +94,7 @@ SELECT r.id
 FROM runs AS r
 WHERE
 	r.project_id = :project_id
-	AND r.status IN ('queued', 'running', 'proposed');
+	AND r.status IN (100, 102, 202);
 
 -- PREP: get_latest_run
 SELECT r.id
@@ -110,5 +110,5 @@ WHERE r.project_id = :project_id;
 
 -- PREP: abort_stuck_runs
 UPDATE runs
-SET status = 'aborted'
-WHERE status IN ('running', 'queued');
+SET status = 499
+WHERE status IN (100, 102);

package/src/agent/schemes.sql

@@ -1,3 +1,3 @@
 -- PREP: upsert_scheme
-INSERT OR REPLACE INTO schemes (name, fidelity, model_visible, valid_states, category)
-VALUES (:name, :fidelity, :model_visible, :valid_states, :category);
+INSERT OR REPLACE INTO schemes (name, model_visible, category)
+VALUES (:name, :model_visible, :category);

package/src/agent/turns.sql

@@ -1,6 +1,6 @@
 -- PREP: create_turn
-INSERT INTO turns (run_id, sequence)
-VALUES (:run_id, :sequence)
+INSERT INTO turns (run_id, loop_id, sequence)
+VALUES (:run_id, :loop_id, :sequence)
 RETURNING id, sequence;
 
 -- PREP: update_turn_stats
@@ -26,7 +26,7 @@ FROM turns
 WHERE run_id = :run_id;
 
 -- PREP: get_run_log
-SELECT ke.path, ke.state AS status, ke.body, ke.attributes
+SELECT ke.path, ke.status, ke.body, ke.attributes
 FROM known_entries AS ke
 JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
 WHERE

package/src/hooks/PluginContext.js

@@ -48,18 +48,10 @@ export default class PluginContext {
 		return this.#schemes;
 	}
 
-	registerScheme({
-		name,
-		fidelity = "full",
-		modelVisible = 1,
-		validStates = ["full", "proposed", "pass", "rejected", "error"],
-		category = "result",
-	} = {}) {
+	registerScheme({ name, modelVisible = 1, category = "result" } = {}) {
 		this.#schemes.push({
 			name: name || this.#name,
-			fidelity,
 			model_visible: modelVisible,
-			valid_states: JSON.stringify(validStates),
 			category,
 		});
 	}
@@ -78,7 +70,6 @@ export default class PluginContext {
 			return;
 		}
 		if (event === "full" || event === "summary") {
-			this.#hooks.tools.ensureTool(this.#name);
 			this.#hooks.tools.onView(this.#name, callback, event);
 			return;
 		}