@possumtech/rummy 0.3.1 → 0.4.0

package/src/agent/known_store.sql

@@ -1,13 +1,12 @@
 -- PREP: upsert_known_entry
 INSERT INTO known_entries (
 	run_id, loop_id, turn, path, body, status, fidelity, hash
-	, attributes, tokens, tokens_full, updated_at
+	, attributes, tokens, updated_at
 )
 VALUES (
 	:run_id, :loop_id, :turn, :path, :body, :status, :fidelity, :hash
 	, COALESCE(:attributes, '{}')
 	, countTokens(:body)
-	, countTokens(:body)
 	, COALESCE(:updated_at, CURRENT_TIMESTAMP)
 )
 ON CONFLICT (run_id, path) DO UPDATE SET
@@ -19,13 +18,12 @@ ON CONFLICT (run_id, path) DO UPDATE SET
 	, loop_id = excluded.loop_id
 	, turn = excluded.turn
 	, tokens = countTokens(excluded.body)
-	, tokens_full = countTokens(excluded.body)
 	, write_count = known_entries.write_count + 1
 	, updated_at = COALESCE(excluded.updated_at, CURRENT_TIMESTAMP);
 
 -- PREP: recount_tokens
 UPDATE known_entries
-SET tokens = :tokens, tokens_full = :tokens
+SET tokens = :tokens
 WHERE run_id = :run_id AND path = :path;
 
 -- PREP: get_stale_tokens
@@ -55,18 +53,6 @@ WHERE run_id = :run_id AND path = :path;
 UPDATE known_entries
 SET
 	fidelity = :fidelity
-	, tokens = CASE
-		WHEN :fidelity = 'archive'
-			THEN 0
-		WHEN :fidelity = 'index'
-			THEN 0
-		WHEN :fidelity = 'summary'
-			THEN COALESCE(
-				countTokens(json_extract(attributes, '$.summary')),
-				countTokens(substr(body, 1, 80))
-			)
-		ELSE tokens_full
-	END
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
 
@@ -74,8 +60,8 @@ WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
 UPDATE known_entries
 SET
 	fidelity = 'full'
+	, status = 200
 	, turn = :turn
-	, tokens = tokens_full
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
@@ -83,26 +69,14 @@ WHERE run_id = :run_id AND path = :path;
 UPDATE known_entries
 SET
 	fidelity = 'archive'
-	, tokens = 0
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
 -- PREP: set_fidelity
+-- Tokens unchanged — always reflects full body cost.
 UPDATE known_entries
 SET
 	fidelity = :fidelity
-	, tokens = CASE
-		WHEN :fidelity = 'archive'
-			THEN 0
-		WHEN :fidelity = 'index'
-			THEN 0
-		WHEN :fidelity = 'summary'
-			THEN COALESCE(
-				countTokens(json_extract(attributes, '$.summary')),
-				countTokens(substr(body, 1, 80))
-			)
-		ELSE countTokens(body)
-	END
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND path = :path;
 
@@ -138,8 +112,8 @@ WHERE run_id = :run_id AND path = :path;
 UPDATE known_entries
 SET
 	fidelity = 'full'
+	, status = 200
 	, turn = :turn
-	, tokens = tokens_full
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
 	run_id = :run_id
@@ -150,7 +124,6 @@ WHERE
 UPDATE known_entries
 SET
 	fidelity = 'archive'
-	, tokens = 0
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
 	run_id = :run_id
@@ -158,7 +131,7 @@ WHERE
 	AND (:body IS NULL OR hedsearch(:body, body));
 
 -- PREP: get_entries_by_pattern
-SELECT path, body, scheme, status, fidelity, tokens_full, attributes
+SELECT path, body, scheme, status, fidelity, tokens, attributes
 FROM known_entries
 WHERE
 	run_id = :run_id
@@ -182,7 +155,6 @@ UPDATE known_entries
 SET
 	body = :new_body
 	, tokens = countTokens(:new_body)
-	, tokens_full = countTokens(:new_body)
 	, write_count = write_count + 1
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
@@ -197,43 +169,21 @@ WHERE
 UPDATE known_entries
 SET
 	fidelity = 'full'
-	, tokens = tokens_full
 	, updated_at = CURRENT_TIMESTAMP
 WHERE run_id = :run_id AND scheme = 'prompt' AND fidelity = 'summary';
 
--- PREP: demote_previous_loop_logging
--- Demote full logging entries from all other loops to summary.
--- Fires at loop start so <previous> entries are already compact.
-UPDATE known_entries
-SET
-	fidelity = 'summary'
-	, tokens = COALESCE(
-		countTokens(json_extract(attributes, '$.summary'))
-		, countTokens(substr(body, 1, 80))
-	)
-	, updated_at = CURRENT_TIMESTAMP
-WHERE
-	run_id = :run_id
-	AND (loop_id IS NULL OR loop_id != :loop_id)
-	AND fidelity = 'full'
-	AND scheme IN (SELECT name FROM schemes WHERE category = 'logging');
-
--- PREP: demote_turn_data_entries
--- Demote full data entries from a turn to summary with 413 status.
--- Fires when end-of-turn materialization exceeds the context ceiling.
+-- PREP: demote_turn_entries
+-- Demote all full entries from a turn to summary with 413 status.
+-- Tokens unchanged — always reports full cost regardless of fidelity.
 UPDATE known_entries
 SET
 	fidelity = 'summary'
 	, status = 413
-	, tokens = COALESCE(
-		countTokens(json_extract(attributes, '$.summary'))
-		, countTokens(substr(body, 1, 80))
-	)
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
 	run_id = :run_id
 	AND turn = :turn
 	AND fidelity = 'full'
 	AND status < 400
-	AND scheme IN (SELECT name FROM schemes WHERE category = 'data')
-RETURNING path;
+RETURNING path, tokens;
+

package/src/agent/runs.sql

@@ -81,11 +81,11 @@ RETURNING next_turn - 1 AS turn;
 -- PREP: fork_known_entries
 INSERT INTO known_entries (
 	run_id, loop_id, turn, path, body, status, fidelity
-	, hash, attributes, tokens, tokens_full, refs, write_count
+	, hash, attributes, tokens, refs, write_count
 )
 SELECT
 	:new_run_id, NULL, turn, path, body, status, fidelity
-	, hash, attributes, tokens, tokens_full, refs, write_count
+	, hash, attributes, tokens, refs, write_count
 FROM known_entries
 WHERE run_id = :parent_run_id;
 

package/src/hooks/Hooks.js

@@ -56,6 +56,7 @@ export default function createHooks(debug = false) {
 		turn: {
 			started: createEvent("turn.started"),
 			response: createEvent("turn.response"),
+			proposal: createEvent("turn.proposal"),
 			proposing: createEvent("turn.proposing"),
 			completed: createEvent("turn.completed"),
 		},

package/src/hooks/ToolRegistry.js

@@ -1,19 +1,20 @@
 // Tool display order: gather → reason → act → communicate.
 // Position in the list implies priority to the model.
 const TOOL_ORDER = [
+	"think",
+	"unknown",
+	"known",
 	"get",
 	"set",
-	"known",
-	"unknown",
 	"env",
 	"sh",
 	"rm",
 	"cp",
 	"mv",
-	"search",
-	"summarize",
-	"update",
 	"ask_user",
+	"update",
+	"summarize",
+	"search",
 ];
 
 function sortByPriority(names) {

package/src/plugins/ask_user/ask_userDoc.js

@@ -2,27 +2,22 @@
 // Text goes to the model. Rationale stays in source.
 // Changing ANY line requires reading ALL rationales first.
 const LINES = [
-	// --- Syntax: question attr + options in body
 	['## <ask_user question="[Question?]">[option1; option2; ...]</ask_user>'],
-
-	// --- Constraints FIRST: frames correct usage before examples
 	[
 		"* YOU SHOULD use for decisions, preferences, or approvals the user must make",
-		"Positive framing. Shows what ask_user IS for, not just what it isn't.",
+		"Positive framing. Shows what ask_user IS for.",
 	],
 	[
 		"* YOU SHOULD use <get> to find information before asking the user",
-		"Gentle redirect. Encourages self-sufficiency without forbidding interaction.",
+		"Gentle redirect. Encourages self-sufficiency.",
 	],
-
-	// --- Examples: genuine decision points where user input is valuable
 	[
 		'Example: <ask_user question="Which test framework?">Mocha; Jest; Node Native</ask_user>',
 		"Preference decision. Model truly cannot know this without asking.",
 	],
 	[
 		'Example: <ask_user question="Deploy to staging or production?">staging; production</ask_user>',
-		"Consequential action. Shows ask_user for high-stakes choices.",
+		"Consequential action. High-stakes choice.",
 	],
 ];
 

package/src/plugins/budget/README.md

@@ -2,30 +2,38 @@
 
 Context ceiling enforcement.
 
-## Files
+## Design
 
-- **budget.js** — Plugin. Pre-LLM enforce, BudgetGuard activation.
-- **BudgetGuard.js** — Write-layer gate. Installed on KnownStore during
-  dispatch. Checks token delta on every upsert, promote, and body update.
+Ceiling = `floor(contextSize × 0.9)`. The 10% headroom is the system's
+operating room for graceful overflow handling. No per-write gating —
+tools run uninterrupted. Enforcement happens at boundaries.
 
-## Registration
+## Enforcement Points
 
-- **Hook**: `hooks.budget.enforce` — pre-LLM ceiling check.
-- **Hook**: `hooks.budget.activate(store, contextSize, assembledTokens)` — install guard.
-- **Hook**: `hooks.budget.deactivate(store)` — remove guard.
+1. **Pre-LLM enforce** (`budget.enforce`): checks assembled context
+   before the LLM call. If over ceiling → Prompt Demotion (summarize
+   the incoming prompt). Model runs in the headroom.
 
-## Budget Contract
+2. **Post-dispatch Turn Demotion**: after all tools dispatch, check
+   context. If over ceiling → demote ALL entries from this turn to
+   summary (every scheme except `budget`). Write `budget://` entry
+   listing what was demoted. Model sees it next turn and adapts.
 
-`contextSize` is the ceiling. `countTokens()` is the measurement.
-Over = 413. Under = 200. No margins.
+3. **LLM rejection** (`isContextExceeded`): turn-1 token estimate
+   drift causes LLM to reject. Same demotion pattern.
 
-## BudgetGuard
+4. **AgentLoop recovery**: pre-LLM 413 that Prompt Demotion can't
+   resolve. Batch-demote all full entries, budget entry, model gets
+   recovery turns. 3 strikes without progress → hard 413 to client.
+   Only path where 413 reaches the client.
 
-Installed on KnownStore by TurnExecutor before dispatch, cleared in
-`finally`. Gates `upsert()`, `promoteByPattern()`, `updateBodyByPattern()`.
+## Files
 
-Exemptions: `status >= 400` (error entries), `model_visible = 0` (audit),
-`fidelity = "archive"` (not in context).
+- **budget.js** — Plugin. Pre-LLM enforce hook.
+- **BudgetGuard.js** — `BudgetExceeded` error type, `delta` utility.
 
-On first violation: `BudgetExceeded` thrown, guard trips, all subsequent
-writes fail. TurnExecutor catches per-tool, writes 413 result entry.
+## Registration
+
+- **Hook**: `hooks.budget.enforce` — pre-LLM ceiling check.
+- **Scheme**: `budget://` — logging category, model-visible. `onView`
+  renders body at all fidelity levels (summary shows full content).

package/src/plugins/budget/budget.js

@@ -1,5 +1,8 @@
 import { countTokens } from "../../agent/tokens.js";
 
+const CEILING_RATIO = Number(process.env.RUMMY_BUDGET_CEILING);
+if (!CEILING_RATIO) throw new Error("RUMMY_BUDGET_CEILING must be set");
+
 function measureMessages(messages) {
 	return messages.reduce((sum, m) => sum + countTokens(m.content), 0);
 }
@@ -17,6 +20,7 @@ export default class Budget {
 		core.hooks.tools.onView("budget", (entry) => entry.body);
 		core.hooks.budget = {
 			enforce: this.enforce.bind(this),
+			postDispatch: this.postDispatch.bind(this),
 		};
 	}
 
@@ -25,8 +29,6 @@ export default class Budget {
 			return { messages, rows, demoted: [], assembledTokens: 0, status: 200 };
 		}
 
-		// Prefer actual prompt_tokens from the last API response — the estimate
-		// from measureMessages can be wildly off for structured/XML-heavy content.
 		const assembledTokens =
 			lastPromptTokens > 0 ? lastPromptTokens : measureMessages(messages);
 
@@ -34,7 +36,7 @@ export default class Budget {
 			`[RUMMY] Budget enforce: ${assembledTokens} tokens (${lastPromptTokens > 0 ? "actual" : "estimated"}), ceiling ${contextSize}, ${rows.length} rows`,
 		);
 
-		const ceiling = Math.floor(contextSize * 0.9);
+		const ceiling = Math.floor(contextSize * CEILING_RATIO);
 		if (assembledTokens > ceiling) {
 			const overflow = assembledTokens - ceiling;
 			console.warn(
@@ -52,4 +54,59 @@ export default class Budget {
 
 		return { messages, rows, demoted: [], assembledTokens, status: 200 };
 	}
+
+	async postDispatch({
+		contextSize,
+		messages,
+		rows,
+		runId,
+		loopId,
+		turn,
+		db,
+		store,
+	}) {
+		if (!contextSize) return null;
+
+		const postBudget = await this.enforce({
+			contextSize,
+			messages,
+			rows,
+			lastPromptTokens: 0,
+		});
+
+		if (postBudget.status !== 413) return null;
+
+		// Demote this turn's entries
+		const demotedEntries = await db.demote_turn_entries.all({
+			run_id: runId,
+			turn,
+		});
+
+		// Also summarize the prompt
+		const promptRow = rows.find((r) => r.scheme === "prompt");
+		if (promptRow) {
+			await store.setFidelity(runId, promptRow.path, "summary");
+		}
+
+		// Write budget entry
+		const ceiling = Math.floor(contextSize * CEILING_RATIO);
+		const totalDemoted = demotedEntries.reduce((s, r) => s + r.tokens, 0);
+		const pathList = demotedEntries
+			.map((r) => `${r.path} (${r.tokens} tokens)`)
+			.join("\n");
+		const body = [
+			`Error 413: Context overflowed by ${postBudget.overflow} tokens.`,
+			`${demotedEntries.length} entries (${totalDemoted} tokens total) demoted. Budget: ${ceiling} tokens.`,
+			pathList,
+		].join("\n");
+
+		await store.upsert(runId, turn, `budget://${loopId}/${turn}`, body, 413, {
+			loopId,
+		});
+
+		return {
+			target: ceiling,
+			promptPath: promptRow?.path ?? null,
+		};
+	}
 }

package/src/plugins/budget/recovery.js

@@ -0,0 +1,47 @@
+/**
+ * Pure recovery state transition — exported for testing.
+ *
+ * @param {object|null} recovery  Current recovery state.
+ * @param {{ assembledTokens: number, budgetRecovery?: { target: number, promptPath: string|null } }} result
+ * @returns {{ next: object|null, action: null|'restore'|'hard413', promptPath: string|null }}
+ */
+export function advanceRecovery(recovery, result) {
+	// Initialise or update recovery state from a new Turn Demotion event.
+	if (result.budgetRecovery) {
+		if (!recovery) {
+			recovery = {
+				target: result.budgetRecovery.target,
+				promptPath: result.budgetRecovery.promptPath,
+				strikes: 0,
+				lastTokens: result.assembledTokens,
+			};
+		} else {
+			// Re-overflow during recovery: tighten target, don't count as strike.
+			recovery = {
+				...recovery,
+				target: Math.min(recovery.target, result.budgetRecovery.target),
+			};
+		}
+	}
+
+	if (recovery === null) return { next: null, action: null, promptPath: null };
+
+	const current = result.assembledTokens;
+
+	if (current <= recovery.target) {
+		return { next: null, action: "restore", promptPath: recovery.promptPath };
+	}
+
+	const noProgress = current >= recovery.lastTokens && !result.budgetRecovery;
+	const strikes = noProgress ? recovery.strikes + 1 : 0;
+
+	if (strikes >= 3) {
+		return { next: null, action: "hard413", promptPath: null };
+	}
+
+	return {
+		next: { ...recovery, strikes, lastTokens: current },
+		action: null,
+		promptPath: null,
+	};
+}

package/src/plugins/cp/cpDoc.js

@@ -2,27 +2,22 @@
 // Text goes to the model. Rationale stays in source.
 // Changing ANY line requires reading ALL rationales first.
 const LINES = [
-	// --- Syntax: path attr = source, body = destination
 	['## <cp path="[source]">[destination]</cp> - Copy a file or entry'],
-
-	// --- Examples: single copy, glob batch, cross-scheme
 	[
 		'Example: <cp path="src/config.js">src/config.backup.js</cp>',
 		"Simple file copy. Path = source, body = destination.",
 	],
 	[
 		'Example: <cp path="known://plan_*">known://archive_</cp>',
-		"Glob batch copy across known entries. Shows pattern operations on cp.",
+		"Glob batch copy across known entries.",
 	],
-
-	// --- Constraints
 	[
 		"* Source path accepts patterns: `src/*.js`, `known://draft_*`",
-		"Pattern support. Distributes glob teaching beyond get.",
+		"Pattern support consistent with get/rm.",
 	],
 	[
-		"* Use `preview` to check matches before bulk copy",
-		"Safety pattern consistent with get and rm preview.",
+		"* Use `preview` to check matches before pattern-based bulk copy",
+		"Safety pattern consistent with rm.",
 	],
 ];
 

package/src/plugins/env/envDoc.js

@@ -2,10 +2,7 @@
 // Text goes to the model. Rationale stays in source.
 // Changing ANY line requires reading ALL rationales first.
 const LINES = [
-	// --- Syntax
 	["## <env>[command]</env> - Run an exploratory shell command"],
-
-	// --- Examples: version check and git status — safe, read-only commands
 	[
 		"Example: <env>npm --version</env>",
 		"Version check. Safe, no side effects.",
@@ -14,15 +11,13 @@ const LINES = [
 		"Example: <env>git log --oneline -5</env>",
 		"Git history. Shows env for read-only investigation.",
 	],
-
-	// --- Constraints: hard boundaries
 	[
 		'* YOU MUST NOT use <env/> to read or list files — use <get path="*" preview/> instead',
-		"Prevents cat/ls through shell. Forces file access through get for proper tracking.",
+		"Prevents cat/ls through shell. Forces file access through get.",
 	],
 	[
-		"* YOU MUST use <sh/> for commands with side effects",
-		"Separates exploration from action. env = observe, sh = mutate.",
+		"* YOU MUST NOT use <env/> for commands with side effects",
+		"Separates exploration from action. env = observe only.",
 	],
 ];
 

package/src/plugins/get/get.js

@@ -90,9 +90,7 @@ export default class Get {
 		}
 
 		const VALID_FIDELITY = {
-			stored: 1,
 			summary: 1,
-			index: 1,
 			full: 1,
 			archive: 1,
 		};
@@ -118,11 +116,11 @@ export default class Get {
 				{ loopId },
 			);
 		} else {
-			const total = matches.reduce((s, m) => s + m.tokens_full, 0);
+			const total = matches.reduce((s, m) => s + m.tokens, 0);
 			const paths = matches.map((m) => m.path).join(", ");
 			const body =
 				matches.length > 0
-					? `${paths} loaded into <knowns> (${total} tokens)`
+					? `${paths} promoted to full (${total} tokens)`
 					: `${target} not found`;
 			await store.upsert(runId, turn, entry.resultPath, body, 200, {
 				loopId,

package/src/plugins/get/getDoc.js

@@ -2,49 +2,42 @@
 // Text goes to the model. Rationale stays in source.
 // Changing ANY line requires reading ALL rationales first.
 const LINES = [
-	// --- Syntax: body-form is the primary invocation (simplest)
 	["## <get>[path/to/file]</get> - Load a file or entry into context"],
-
-	// --- Examples: 3 examples covering single file, known recall, and content search
 	[
 		"Example: <get>src/app.js</get>",
-		"Simplest form. Body = path. Teaches that get is the default read tool.",
+		"Simplest form. Body = path.",
 	],
 	[
 		'Example: <get path="known://*">auth</get>',
-		"Keyword recall: glob in path, search term in body. Cross-scheme hedberg pattern.",
+		"Keyword recall: glob in path, search term in body.",
 	],
 	[
 		'Example: <get path="src/**/*.js" preview>authentication</get>',
-		"Full pattern: recursive glob + preview + content filter. Shows all 3 features at once. Body is a filter keyword, never file content.",
+		"Full pattern: recursive glob + preview + content filter.",
 	],
-
-	// --- Partial read: line/limit — show before constraints so model sees it as a first-class pattern
 	[
 		'Example: <get path="src/agent/AgentLoop.js" line="644" limit="80"/>',
-		"Partial read. Returns lines 644–723 as the log item without promoting the entry to full. Use summary fidelity to find line numbers, then target the symbol directly.",
+		"Partial read. Returns lines 644–723 without promoting.",
 	],
-
-	// --- Constraints: RFC-style. Each prevents a specific failure mode.
 	[
 		"* Paths accept patterns: `src/**/*.js`, `known://api_*`",
-		"Reinforces picomatch patterns work everywhere, not just in examples.",
+		"Reinforces picomatch patterns work everywhere.",
 	],
 	[
-		"* `preview` shows matches without loading into context",
-		"Budget-awareness. Without this, models load everything and blow context.",
+		"* `preview` lists matches without loading into context",
+		"Budget-awareness. Preview avoids promotion.",
 	],
 	[
 		"* Body text filters results by content match",
-		"Generalizes examples 2-3. Body = filter, not just path.",
+		"Body = filter, not just path.",
 	],
 	[
 		"* `line` and `limit` read a slice without promoting — patterns not allowed",
-		"The no-promotion constraint is what makes partial read safe: context budget is unaffected.",
+		"Partial read is safe: context budget unaffected.",
 	],
 	[
-		'* Use <set path="..." fidelity="archive"/> to remove loaded content from context',
-		"Lifecycle: get→set. Load, read, archive. Prevents context hoarding.",
+		'* Use <set path="src/file.txt" fidelity="summary"/> when the content is irrelevant to save tokens.',
+		"Cross-tool lifecycle: get promotes, set demotes.",
 	],
 ];
 

package/src/plugins/helpers.js

@@ -14,8 +14,8 @@ export async function storePatternResult(
 ) {
 	const slug = await store.slugPath(runId, scheme, path);
 	const filter = bodyFilter ? ` body="${bodyFilter}"` : "";
-	const total = matches.reduce((s, m) => s + m.tokens_full, 0);
-	const listing = matches.map((m) => `${m.path} (${m.tokens_full})`).join("\n");
+	const total = matches.reduce((s, m) => s + m.tokens, 0);
+	const listing = matches.map((m) => `${m.path} (${m.tokens})`).join("\n");
 	const prefix = preview ? "PREVIEW " : "";
 	const body = `${prefix}${scheme} path="${path}"${filter}: ${matches.length} matched (${total} tokens)\n${listing}`;
 	await store.upsert(runId, turn, slug, body, 200, { loopId });

package/src/plugins/instructions/instructions.js

@@ -37,7 +37,6 @@ export default class Instructions {
 			activeTools.has(n),
 		);
 		const tools = sorted.join(", ");
-		let prompt = preamble.replace("[%TOOLS%]", tools);
 		const toolDocs = await this.#core.hooks.instructions.toolDocs.filter(
 			{},
 			{ toolSet: activeTools },
@@ -46,7 +45,9 @@ export default class Instructions {
 			.filter((key) => toolDocs[key])
 			.map((key) => toolDocs[key])
 			.join("\n\n");
-		if (docsText) prompt += `\n\n${docsText}`;
+		let prompt = preamble
+			.replace("[%TOOLS%]", tools)
+			.replace("[%TOOLDOCS%]", docsText);
 		if (attrs.persona) prompt += `\n\n## Persona\n\n${attrs.persona}`;
 		return prompt;
 	}