@possumtech/rummy 0.4.0 → 2.0.0

package/src/plugins/budget/budget.js

@@ -1,10 +1,44 @@
+import { ceiling, computeBudget, measureMessages } from "../../agent/budget.js";
+import materializeContext from "../../agent/materializeContext.js";
 import { countTokens } from "../../agent/tokens.js";
 
-const CEILING_RATIO = Number(process.env.RUMMY_BUDGET_CEILING);
-if (!CEILING_RATIO) throw new Error("RUMMY_BUDGET_CEILING must be set");
+/**
+ * Delta-from-actual baseline. The pre-call <prompt tokenUsage> reports
+ * the prior turn's actual API prompt_tokens; post-dispatch predicts
+ * next turn's packet = this turn's actual tokens + tokens of new rows
+ * written this turn. Keeps the 413 body on the same scale as the
+ * model's <prompt> arithmetic — a 60% divergence between pre-call
+ * (actual) and post-check (conservative estimator) makes the model
+ * dismiss the system as janky and stop following rules.
+ */
+function predictNextPacket(rows, currentTurn, baseline) {
+	let delta = 0;
+	for (const r of rows) {
+		if (r.source_turn === currentTurn) delta += countTokens(r.body);
+	}
+	return baseline + delta;
+}
 
-function measureMessages(messages) {
-	return messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+/**
+ * Format the 413 error body. Names each demoted path with its turn
+ * and token count so the model can avoid re-promoting them next turn.
+ * Exported (not private) so unit tests can assert the exact wire
+ * format — the model reads this string, so its shape is part of the
+ * contract.
+ */
+export function overflowBody(overflow, contextSize, demoted) {
+	const cap = ceiling(contextSize);
+	const size = cap + overflow;
+	const count = demoted.length;
+	const totalTokens = demoted.reduce((s, r) => s + r.tokens, 0);
+	const head = `Token Budget overflow: packet was ${size} tokens, ceiling is ${cap}. ${count} promotion${count === 1 ? "" : "s"} (${totalTokens} tokens) demoted to fit.`;
+	if (count === 0) return head;
+	const lines = demoted.map((d) =>
+		d.turn
+			? `- ${d.path} (turn ${d.turn}, ${d.tokens} tokens)`
+			: `- ${d.path} (${d.tokens} tokens)`,
+	);
+	return `${head}\nDemoted:\n${lines.join("\n")}`;
 }
 
 export default class Budget {
@@ -12,101 +46,259 @@ export default class Budget {
 
 	constructor(core) {
 		this.#core = core;
-		core.registerScheme({
-			name: "budget",
-			modelVisible: 1,
-			category: "logging",
-		});
-		core.hooks.tools.onView("budget", (entry) => entry.body);
 		core.hooks.budget = {
 			enforce: this.enforce.bind(this),
 			postDispatch: this.postDispatch.bind(this),
 		};
+		core.filter("assembly.user", this.assembleBudget.bind(this), 275);
 	}
 
-	async enforce({ contextSize, messages, rows, lastPromptTokens = 0 }) {
-		if (!contextSize) {
-			return { messages, rows, demoted: [], assembledTokens: 0, status: 200 };
-		}
+	/**
+	 * Render the <budget> table between <instructions> and <prompt>.
+	 * See SPEC @token_accounting for the contract: per-row tokens are
+	 * aTokens (the promotion premium = vTokens − sTokens), summarized
+	 * entries collapse into a single aggregate line, system overhead
+	 * (system prompt + tool defs) gets its own line.
+	 */
+	assembleBudget(content, ctx) {
+		const { rows, contextSize, systemPrompt } = ctx;
+		if (!contextSize) return content;
 
-		const assembledTokens =
-			lastPromptTokens > 0 ? lastPromptTokens : measureMessages(messages);
+		const cap = ceiling(contextSize);
 
-		console.warn(
-			`[RUMMY] Budget enforce: ${assembledTokens} tokens (${lastPromptTokens > 0 ? "actual" : "estimated"}), ceiling ${contextSize}, ${rows.length} rows`,
-		);
+		const visibleByScheme = new Map();
+		let visibleCount = 0;
+		let premiumTokens = 0;
+		let summarizedCount = 0;
+		let summarizedTokens = 0;
+		let floorTokens = 0;
 
-		const ceiling = Math.floor(contextSize * CEILING_RATIO);
-		if (assembledTokens > ceiling) {
-			const overflow = assembledTokens - ceiling;
-			console.warn(
-				`[RUMMY] Budget 413: ${assembledTokens} tokens > ${contextSize} ceiling (${overflow} over)`,
-			);
-			return {
-				messages,
-				rows,
-				demoted: [],
-				assembledTokens,
-				status: 413,
-				overflow,
-			};
+		for (const r of rows) {
+			if (r.aTokens == null) continue;
+			const s = r.scheme || "file";
+			if (r.visibility === "visible") {
+				const entry = visibleByScheme.get(s) ?? { count: 0, tokens: 0 };
+				entry.count += 1;
+				entry.tokens += r.aTokens;
+				visibleByScheme.set(s, entry);
+				visibleCount += 1;
+				premiumTokens += r.aTokens;
+				floorTokens += r.sTokens;
+			} else if (r.visibility === "summarized") {
+				summarizedCount += 1;
+				summarizedTokens += r.sTokens;
+				floorTokens += r.sTokens;
+			}
 		}
 
-		return { messages, rows, demoted: [], assembledTokens, status: 200 };
+		const systemTokens = countTokens(systemPrompt || "");
+		const tokenUsage = floorTokens + premiumTokens + systemTokens;
+		const tokensFree = Math.max(0, cap - tokenUsage);
+
+		const schemeRows = [...visibleByScheme.entries()]
+			.toSorted((a, b) => b[1].tokens - a[1].tokens)
+			.map(([scheme, v]) => {
+				const pct = Math.round((v.tokens / cap) * 100);
+				return `| ${scheme} | ${v.count} | ${v.tokens} | ${pct}% |`;
+			});
+
+		const summarizedPct = Math.round((summarizedTokens / cap) * 100);
+		const systemPct = Math.round((systemTokens / cap) * 100);
+
+		const table = [
+			"| scheme | visible | tokens | % |",
+			"|---|---|---|---|",
+			...schemeRows,
+		].join("\n");
+
+		const summarizedLine = `Summarized: ${summarizedCount} entries, ${summarizedTokens} tokens (${summarizedPct}% of budget).`;
+		const systemLine = `System: ${systemTokens} tokens (${systemPct}% of budget).`;
+		const totalLine = `Total: ${visibleCount} visible + ${summarizedCount} summarized entries; tokenUsage ${tokenUsage} / ceiling ${cap}. ${tokensFree} tokens free.`;
+
+		return `${content}<budget tokenUsage="${tokenUsage}" tokensFree="${tokensFree}">\n${table}\n\n${summarizedLine}\n${systemLine}\n${totalLine}\n</budget>\n`;
+	}
+
+	#check({ contextSize, messages, rows, lastPromptTokens = 0 }) {
+		const totalTokens =
+			lastPromptTokens > 0 ? lastPromptTokens : measureMessages(messages);
+		const b = computeBudget({ rows, contextSize, totalTokens });
+		return {
+			messages,
+			rows,
+			assembledTokens: b.totalTokens,
+			overflow: b.overflow,
+			ok: b.ok,
+		};
+	}
+
+	async #emitOverflow({
+		message,
+		runId,
+		turn,
+		loopId,
+		rummy,
+		demotedCount = 0,
+		demotedTokens = 0,
+	}) {
+		await rummy.hooks.error.log.emit({
+			store: rummy.entries,
+			runId,
+			turn,
+			loopId,
+			message,
+			status: 413,
+			attributes: { demotedCount, demotedTokens },
+		});
 	}
 
-	async postDispatch({
+	/**
+	 * Pre-LLM budget enforcement. On first-turn overflow, demotes the
+	 * incoming prompt and re-materializes; re-checks and returns the
+	 * post-demotion result. If overflow persists after demotion (or on
+	 * later iterations), emits a 413 error (strike) and returns !ok so
+	 * TurnExecutor can skip the LLM call this turn.
+	 *
+	 * ctx = { runId, loopId, turn, systemPrompt, mode, toolSet, demoted,
+	 *         loopIteration }
+	 */
+	async enforce({
 		contextSize,
 		messages,
 		rows,
-		runId,
-		loopId,
-		turn,
-		db,
-		store,
+		lastPromptTokens = 0,
+		ctx,
+		rummy,
 	}) {
-		if (!contextSize) return null;
+		if (!contextSize) {
+			return { messages, rows, assembledTokens: 0, ok: true };
+		}
 
-		const postBudget = await this.enforce({
+		const first = this.#check({
 			contextSize,
 			messages,
 			rows,
-			lastPromptTokens: 0,
+			lastPromptTokens,
 		});
+		if (first.ok) return first;
 
-		if (postBudget.status !== 413) return null;
+		if (ctx?.loopIteration !== 1) {
+			const cap = ceiling(contextSize);
+			await this.#emitOverflow({
+				message: `Token Budget overflow: packet was ${cap + first.overflow} tokens, ceiling is ${cap}.`,
+				runId: ctx.runId,
+				turn: ctx.turn,
+				loopId: ctx.loopId,
+				rummy,
+			});
+			return first;
+		}
 
-		// Demote this turn's entries
-		const demotedEntries = await db.demote_turn_entries.all({
-			run_id: runId,
-			turn,
+		const promptRow = rows.findLast(
+			(r) => r.category === "prompt" && r.scheme === "prompt",
+		);
+		if (promptRow) {
+			await rummy.entries.set({
+				runId: ctx.runId,
+				path: promptRow.path,
+				visibility: "summarized",
+			});
+		}
+		const reMat = await materializeContext({
+			db: rummy.db,
+			hooks: rummy.hooks,
+			runId: ctx.runId,
+			loopId: ctx.loopId,
+			turn: ctx.turn,
+			systemPrompt: ctx.systemPrompt,
+			mode: ctx.mode,
+			toolSet: ctx.toolSet,
+			contextSize,
+			demoted: ctx.demoted,
 		});
+		const rechecked = this.#check({
+			contextSize,
+			messages: reMat.messages,
+			rows: reMat.rows,
+			lastPromptTokens: reMat.lastContextTokens,
+		});
+		if (!rechecked.ok) {
+			const cap = ceiling(contextSize);
+			await this.#emitOverflow({
+				message: `Token Budget overflow: packet was ${cap + rechecked.overflow} tokens after demoting the prompt, ceiling is ${cap}.`,
+				runId: ctx.runId,
+				turn: ctx.turn,
+				loopId: ctx.loopId,
+				rummy,
+			});
+		}
+		return rechecked;
+	}
 
-		// Also summarize the prompt
-		const promptRow = rows.find((r) => r.scheme === "prompt");
+	/**
+	 * Post-dispatch Turn Demotion. Re-materializes end-of-turn context and
+	 * checks against the ceiling. On overflow, demotes this turn's promoted
+	 * entries and emits a 413 error (strike) with the descriptive body so
+	 * the model sees it next turn via the unified error channel.
+	 *
+	 * ctx = { runId, loopId, turn, systemPrompt, mode, toolSet, demoted }
+	 */
+	async postDispatch({ contextSize, ctx, rummy }) {
+		if (!contextSize) return { failed: false };
+		const postMat = await materializeContext({
+			db: rummy.db,
+			hooks: rummy.hooks,
+			runId: ctx.runId,
+			loopId: ctx.loopId,
+			turn: ctx.turn,
+			systemPrompt: ctx.systemPrompt,
+			mode: ctx.mode,
+			toolSet: ctx.toolSet,
+			contextSize,
+			demoted: ctx.demoted,
+		});
+		// Baseline from this turn's actual API tokens (telemetry wrote it
+		// before post-dispatch runs). Delta from rows added this turn.
+		// Predicted next-turn packet stays on the tokenUsage scale the
+		// model can verify against its own arithmetic. materializeContext
+		// guarantees a number (0 when no prior API call exists).
+		const baseline = postMat.lastContextTokens;
+		const predicted = predictNextPacket(postMat.rows, ctx.turn, baseline);
+		const cap = ceiling(contextSize);
+		if (predicted <= cap) return { failed: false };
+		const post = { overflow: predicted - cap };
+
+		const store = rummy.entries;
+		let demotedEntries = await store.demoteTurnEntries(ctx.runId, ctx.turn);
+		// Fallback: if this turn had nothing to demote but the packet still
+		// overflows, the pressure is coming from prior-turn promotions the
+		// model never demoted itself. Widen to all currently-visible
+		// entries in the run. Without this fallback, overflow-with-nothing
+		// strikes out runs where the base context has drifted over ceiling
+		// through no fault of the current turn (observed: runs where 3
+		// stale promotions from turns 12–14 saturate every subsequent
+		// turn's budget).
+		if (demotedEntries.length === 0) {
+			demotedEntries = await store.demoteRunVisibleEntries(ctx.runId);
+		}
+		const promptRow = postMat.rows.find((r) => r.scheme === "prompt");
 		if (promptRow) {
-			await store.setFidelity(runId, promptRow.path, "summary");
+			await store.set({
+				runId: ctx.runId,
+				path: promptRow.path,
+				visibility: "summarized",
+			});
 		}
 
-		// Write budget entry
-		const ceiling = Math.floor(contextSize * CEILING_RATIO);
 		const totalDemoted = demotedEntries.reduce((s, r) => s + r.tokens, 0);
-		const pathList = demotedEntries
-			.map((r) => `${r.path} (${r.tokens} tokens)`)
-			.join("\n");
-		const body = [
-			`Error 413: Context overflowed by ${postBudget.overflow} tokens.`,
-			`${demotedEntries.length} entries (${totalDemoted} tokens total) demoted. Budget: ${ceiling} tokens.`,
-			pathList,
-		].join("\n");
-
-		await store.upsert(runId, turn, `budget://${loopId}/${turn}`, body, 413, {
-			loopId,
+		await this.#emitOverflow({
+			message: overflowBody(post.overflow, contextSize, demotedEntries),
+			demotedCount: demotedEntries.length,
+			demotedTokens: totalDemoted,
+			runId: ctx.runId,
+			turn: ctx.turn,
+			loopId: ctx.loopId,
+			rummy,
 		});
-
-		return {
-			target: ceiling,
-			promptPath: promptRow?.path ?? null,
-		};
+		return { failed: true };
 	}
 }

package/src/plugins/cp/README.md

@@ -1,4 +1,4 @@
-# cp
+# cp {#cp_plugin}
 
 Copies an entry from one path to another within the K/V store.
 
@@ -15,5 +15,5 @@ Shows `cp {from} {to}`.
 ## Behavior
 
 Warns if the destination already exists and will be overwritten. Uses
-`KnownStore.scheme()` to determine whether the destination is a scheme
+`Entries.scheme()` to determine whether the destination is a scheme
 path or a file path.

package/src/plugins/cp/cp.js

@@ -1,4 +1,4 @@
-import KnownStore from "../../agent/KnownStore.js";
+import Entries from "../../agent/Entries.js";
 import docs from "./cpDoc.js";
 
 export default class Cp {
@@ -8,8 +8,8 @@ export default class Cp {
 		this.#core = core;
 		core.registerScheme();
 		core.on("handler", this.handler.bind(this));
-		core.on("full", this.full.bind(this));
-		core.on("summary", this.summary.bind(this));
+		core.on("visible", this.full.bind(this));
+		core.on("summarized", this.summary.bind(this));
 		core.filter("instructions.toolDocs", async (docsMap) => {
 			docsMap.cp = docs;
 			return docsMap;
@@ -19,15 +19,15 @@ export default class Cp {
 	async handler(entry, rummy) {
 		const { entries: store, sequence: turn, runId, loopId } = rummy;
 		const { path, to } = entry.attributes;
-		const VALID = { stored: 1, summary: 1, index: 1, full: 1, archive: 1 };
-		const fidelity = VALID[entry.attributes.fidelity]
-			? entry.attributes.fidelity
+		const VALID = { visible: 1, summarized: 1, archived: 1 };
+		const visibility = VALID[entry.attributes.visibility]
+			? entry.attributes.visibility
 			: undefined;
 
 		const source = await store.getBody(runId, path);
 		if (source === null) return;
 
-		const destScheme = KnownStore.scheme(to);
+		const destScheme = Entries.scheme(to);
 		const existing = await store.getBody(runId, to);
 		const warning =
 			existing !== null && destScheme !== null
@@ -36,13 +36,31 @@ export default class Cp {
 
 		const body = `${path} ${to}`;
 		if (destScheme === null) {
-			await store.upsert(runId, turn, entry.resultPath, body, 202, {
+			await store.set({
+				runId,
+				turn,
+				path: entry.resultPath,
+				body,
+				state: "proposed",
 				attributes: { from: path, to, isMove: false, warning },
 				loopId,
 			});
 		} else {
-			await store.upsert(runId, turn, to, source, 200, { fidelity, loopId });
-			await store.upsert(runId, turn, entry.resultPath, body, 200, {
+			await store.set({
+				runId,
+				turn,
+				path: to,
+				body: source,
+				state: "resolved",
+				visibility,
+				loopId,
+			});
+			await store.set({
+				runId,
+				turn,
+				path: entry.resultPath,
+				body,
+				state: "resolved",
 				attributes: { from: path, to, isMove: false, warning },
 				loopId,
 			});
@@ -50,10 +68,10 @@ export default class Cp {
 	}
 
 	full(entry) {
-		return `# cp ${entry.attributes.from || ""} ${entry.attributes.to || ""}`;
+		return `# cp ${entry.attributes.from} ${entry.attributes.to}`;
 	}
 
-	summary(entry) {
-		return this.full(entry);
+	summary() {
+		return "";
 	}
 }

package/src/plugins/cp/cpDoc.js

@@ -1,24 +1,3 @@
-// Tool doc for <cp>. Each entry: [text, rationale].
-// Text goes to the model. Rationale stays in source.
-// Changing ANY line requires reading ALL rationales first.
-const LINES = [
-	['## <cp path="[source]">[destination]</cp> - Copy a file or entry'],
-	[
-		'Example: <cp path="src/config.js">src/config.backup.js</cp>',
-		"Simple file copy. Path = source, body = destination.",
-	],
-	[
-		'Example: <cp path="known://plan_*">known://archive_</cp>',
-		"Glob batch copy across known entries.",
-	],
-	[
-		"* Source path accepts patterns: `src/*.js`, `known://draft_*`",
-		"Pattern support consistent with get/rm.",
-	],
-	[
-		"* Use `preview` to check matches before pattern-based bulk copy",
-		"Safety pattern consistent with rm.",
-	],
-];
+import { loadDoc } from "../helpers.js";
 
-export default LINES.map(([text]) => text).join("\n");
+export default loadDoc(import.meta.url, "cpDoc.md");

package/src/plugins/cp/cpDoc.md

@@ -0,0 +1,7 @@
+## <cp path="[source]">[destination]</cp> - Copy a file or entry
+
+Example: <cp path="src/config.js">src/config.backup.js</cp>
+<!-- Simple file copy. Path = source, body = destination. -->
+
+Example: <cp path="known://plan_*">known://archive_</cp>
+<!-- Glob batch copy across known entries. -->

package/src/plugins/engine/README.md

@@ -1,10 +1,10 @@
-# engine
+# engine {#engine_plugin}
 
 SQL infrastructure for context assembly and turn management. No JS plugin.
 
 ## Files
 
-- **engine.sql** — Queries for retrieving promoted entries by scheme tier, model visibility, and state.
+- **engine.sql** — Queries for retrieving visible entries by scheme tier, model visibility, and state.
 - **turn_context.sql** — Queries for clearing and reading the `turn_context` / `v_model_context` view, which produces the ordered context sent to the model.
 
 ## Behavior

package/src/plugins/engine/engine.sql

@@ -1,13 +1,13 @@
 -- PREP: get_promoted_entries
 SELECT
-	ke.path, ke.scheme, ke.status, ke.fidelity, ke.turn
-	, ke.tokens, ke.refs
+	ke.path, ke.scheme, ke.state, ke.outcome, ke.visibility, ke.turn
+	, countTokens(ke.body) AS tokens, ke.refs
 FROM known_entries AS ke
 JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
 WHERE
 	ke.run_id = :run_id
-	AND ke.fidelity IN ('full', 'summary')
+	AND ke.visibility IN ('visible', 'summarized')
 	AND s.model_visible = 1
-ORDER BY ke.turn, ke.refs, ke.tokens DESC;
+ORDER BY ke.turn, ke.refs, countTokens(ke.body) DESC;
 
 

package/src/plugins/engine/turn_context.sql

@@ -4,33 +4,33 @@ WHERE run_id = :run_id AND turn = :turn;
 
 -- PREP: get_model_context
 SELECT
-	ordinal, path, scheme, fidelity, status, body
-	, tokens, attributes, category, turn
+	ordinal, path, scheme, visibility, state, outcome, body
+	, attributes, category, turn
 FROM v_model_context
 WHERE run_id = :run_id
 ORDER BY ordinal;
 
 -- PREP: insert_turn_context
 INSERT INTO turn_context (
-	run_id, loop_id, turn, ordinal, path, fidelity, status
-	, body, tokens, attributes, category, source_turn
+	run_id, loop_id, turn, ordinal, path, visibility, state, outcome
+	, body, attributes, category, source_turn
 )
 VALUES (
-	:run_id, :loop_id, :turn, :ordinal, :path, :fidelity
-	, :status, :body, :tokens
+	:run_id, :loop_id, :turn, :ordinal, :path, :visibility
+	, :state, :outcome, :body
 	, COALESCE(:attributes, '{}'), :category, :source_turn
 );
 
 -- PREP: get_turn_context
 SELECT
-	ordinal, path, scheme, fidelity, status, body
-	, tokens, attributes, category, source_turn
+	ordinal, path, scheme, visibility, state, outcome, body
+	, attributes, category, source_turn
 FROM turn_context
 WHERE run_id = :run_id AND turn = :turn
 ORDER BY ordinal;
 
 -- PREP: get_turn_budget
-SELECT COALESCE(SUM(tokens), 0) AS total
+SELECT COALESCE(SUM(countTokens(body)), 0) AS total
 FROM turn_context
 WHERE run_id = :run_id AND turn = :turn;
 
@@ -43,7 +43,7 @@ SELECT
 		WHEN 'prompt' THEN 'prompt'
 		ELSE 'system'
 	END AS bucket,
-	COALESCE(SUM(tokens), 0) AS tokens,
+	COALESCE(SUM(countTokens(body)), 0) AS tokens,
 	COUNT(*) AS entries
 FROM turn_context
 WHERE run_id = :run_id AND turn = :turn

package/src/plugins/env/README.md

@@ -1,13 +1,28 @@
-# env
+# env {#env_plugin}
 
-Runs an exploratory shell command and records the output.
+Runs an exploratory shell command and records the output. Streaming
+producer — same entry shape as [sh](../sh/README.md), different scheme
+name so ask-mode policy can admit read-only discovery without allowing
+side effects.
 
 ## Registration
 
 - **Tool**: `env`
-- **Category**: `logging`
-- **Handler**: Upserts the entry at status 202 (proposed) with original attributes preserved.
+- **Scheme**: `env` — `category: "data"` (channels only; see below)
+- **Handler**: Upserts the proposal entry at status 202 (proposed).
+
+## Two namespaces per invocation
+
+- **Log entry**: `log://turn_N/env/{slug}` — scheme=`log`, category=`logging`.
+  The audit record (renders inside `<log>` as `<env>`).
+- **Data channels**: `env://turn_N/{slug}_1` (stdout), `env://turn_N/{slug}_2`
+  (stderr) — scheme=`env`, category=`data`. The captured payload
+  (renders inside `<context>` as `<env>`).
+
+The `env` scheme exists **only** for the data channels. See
+[scheme_category_split](#scheme_category_split).
 
 ## Projection
 
-Shows `env {command}` followed by the entry body.
+- **Visible**: `# env {command}\n{body}`.
+- **Summarized**: empty.