@possumtech/rummy 0.5.0 → 2.0.1

package/src/plugins/budget/budget.js

@@ -1,10 +1,44 @@
+import { ceiling, computeBudget, measureMessages } from "../../agent/budget.js";
+import materializeContext from "../../agent/materializeContext.js";
 import { countTokens } from "../../agent/tokens.js";
 
-const CEILING_RATIO = Number(process.env.RUMMY_BUDGET_CEILING);
-if (!CEILING_RATIO) throw new Error("RUMMY_BUDGET_CEILING must be set");
+/**
+ * Delta-from-actual baseline. The pre-call <prompt tokenUsage> reports
+ * the prior turn's actual API prompt_tokens; post-dispatch predicts
+ * next turn's packet = this turn's actual tokens + tokens of new rows
+ * written this turn. Keeps the 413 body on the same scale as the
+ * model's <prompt> arithmetic — a 60% divergence between pre-call
+ * (actual) and post-check (conservative estimator) makes the model
+ * dismiss the system as janky and stop following rules.
+ */
+function predictNextPacket(rows, currentTurn, baseline) {
+	let delta = 0;
+	for (const r of rows) {
+		if (r.source_turn === currentTurn) delta += countTokens(r.body);
+	}
+	return baseline + delta;
+}
 
-function measureMessages(messages) {
-	return messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+/**
+ * Format the 413 error body. Names each demoted path with its turn
+ * and token count so the model can avoid re-promoting them next turn.
+ * Exported (not private) so unit tests can assert the exact wire
+ * format — the model reads this string, so its shape is part of the
+ * contract.
+ */
+export function overflowBody(overflow, contextSize, demoted) {
+	const cap = ceiling(contextSize);
+	const size = cap + overflow;
+	const count = demoted.length;
+	const totalTokens = demoted.reduce((s, r) => s + r.tokens, 0);
+	const head = `Token Budget overflow: packet was ${size} tokens, ceiling is ${cap}. ${count} promotion${count === 1 ? "" : "s"} (${totalTokens} tokens) demoted to fit.`;
+	if (count === 0) return head;
+	const lines = demoted.map((d) =>
+		d.turn
+			? `- ${d.path} (turn ${d.turn}, ${d.tokens} tokens)`
+			: `- ${d.path} (${d.tokens} tokens)`,
+	);
+	return `${head}\nDemoted:\n${lines.join("\n")}`;
 }
 
 export default class Budget {
@@ -12,121 +46,305 @@ export default class Budget {
 
 	constructor(core) {
 		this.#core = core;
-		core.registerScheme({
-			name: "budget",
-			modelVisible: 1,
-			category: "logging",
-		});
-		core.hooks.tools.onView("budget", (entry) => entry.body);
 		core.hooks.budget = {
 			enforce: this.enforce.bind(this),
 			postDispatch: this.postDispatch.bind(this),
 		};
+		core.filter("assembly.user", this.assembleBudget.bind(this), 275);
 	}
 
-	async enforce({ contextSize, messages, rows, lastPromptTokens = 0 }) {
-		if (!contextSize) {
-			return { messages, rows, demoted: [], assembledTokens: 0, status: 200 };
-		}
+	/**
+	 * Render the <budget> table between <instructions> and <prompt>.
+	 * See SPEC @token_accounting for the contract: per-row tokens are
+	 * aTokens (the promotion premium = vTokens − sTokens), summarized
+	 * entries collapse into a single aggregate line, system overhead
+	 * (system prompt + tool defs) gets its own line.
+	 */
+	assembleBudget(content, ctx) {
+		const { rows, contextSize, systemPrompt } = ctx;
+		if (!contextSize) return content;
 
-		const assembledTokens =
-			lastPromptTokens > 0 ? lastPromptTokens : measureMessages(messages);
+		const cap = ceiling(contextSize);
 
-		console.warn(
-			`[RUMMY] Budget enforce: ${assembledTokens} tokens (${lastPromptTokens > 0 ? "actual" : "estimated"}), ceiling ${contextSize}, ${rows.length} rows`,
-		);
+		// Per-scheme aggregation: counts and costs at each visibility tier
+		// plus the savings (premium) the model would unlock by demoting
+		// visible → summarized. All math derives from per-row vTokens
+		// (cost as visible) / sTokens (cost as summarized) / aTokens
+		// (= vTokens − sTokens, the promotion premium).
+		const byScheme = new Map();
+		let visibleCount = 0;
+		let premiumTokens = 0;
+		let summarizedCount = 0;
+		let _summarizedTokens = 0;
+		let floorTokens = 0;
+		let knownVTokens = 0;
+		let sourceVTokens = 0;
+
+		const schemeEntry = (s) => {
+			let e = byScheme.get(s);
+			if (!e) {
+				e = {
+					vis: 0,
+					sum: 0,
+					visTokens: 0, // current cost of visible entries
+					visIfSumTokens: 0, // sTokens of visible (what they'd cost demoted)
+					sumTokens: 0, // current cost of summarized entries
+					premium: 0, // savings from demoting visible → summarized
+				};
+				byScheme.set(s, e);
+			}
+			return e;
+		};
 
-		const ceiling = Math.floor(contextSize * CEILING_RATIO);
-		if (assembledTokens > ceiling) {
-			const overflow = assembledTokens - ceiling;
-			console.warn(
-				`[RUMMY] Budget 413: ${assembledTokens} tokens > ${contextSize} ceiling (${overflow} over)`,
-			);
-			return {
-				messages,
-				rows,
-				demoted: [],
-				assembledTokens,
-				status: 413,
-				overflow,
-			};
+		for (const r of rows) {
+			if (r.aTokens == null) continue;
+			const s = r.scheme || "file";
+			const entry = schemeEntry(s);
+			if (r.visibility === "visible") {
+				entry.vis += 1;
+				entry.visTokens += r.vTokens || 0;
+				entry.visIfSumTokens += r.sTokens || 0;
+				entry.premium += r.aTokens || 0;
+				visibleCount += 1;
+				premiumTokens += r.aTokens;
+				floorTokens += r.sTokens;
+				const v = r.vTokens || 0;
+				if (s === "known") knownVTokens += v;
+				else if (s === "prompt") sourceVTokens += v;
+				else if (r.category === "data") sourceVTokens += v;
+			} else if (r.visibility === "summarized") {
+				entry.sum += 1;
+				entry.sumTokens += r.sTokens || 0;
+				summarizedCount += 1;
+				_summarizedTokens += r.sTokens;
+				floorTokens += r.sTokens;
+			}
 		}
 
-		return { messages, rows, demoted: [], assembledTokens, status: 200 };
+		const fcrmDenom = knownVTokens + sourceVTokens;
+		const fcrmScore =
+			fcrmDenom > 0 ? (knownVTokens / fcrmDenom).toFixed(2) : "1.00";
+
+		const systemTokens = countTokens(systemPrompt || "");
+		const tokenUsage = floorTokens + premiumTokens + systemTokens;
+		const tokensFree = Math.max(0, cap - tokenUsage);
+
+		// Sort schemes by current cost descending — biggest-impact rows
+		// land at the top, so "what should I demote first?" reads
+		// straight off the table.
+		const schemeRows = [...byScheme.entries()]
+			.toSorted(
+				([, a], [, b]) =>
+					b.visTokens + b.sumTokens - (a.visTokens + a.sumTokens),
+			)
+			.map(([scheme, e]) => {
+				const cost = e.visTokens + e.sumTokens;
+				const ifAllSum = e.visIfSumTokens + e.sumTokens;
+				return `| ${scheme} | ${e.vis} | ${e.sum} | ${cost} | ${ifAllSum} | ${e.premium} |`;
+			});
+
+		const systemPct =
+			tokenUsage > 0 ? Math.round((systemTokens / tokenUsage) * 100) : 0;
+
+		const table = [
+			"| scheme | vis | sum | cost | if-all-sum | premium |",
+			"|---|---|---|---|---|---|",
+			...schemeRows,
+		].join("\n");
+
+		const systemLine = `System: ${systemTokens} tokens (${systemPct}% of budget).`;
+		const totalLine = `Total: ${visibleCount} visible + ${summarizedCount} summarized entries; tokenUsage ${tokenUsage} / ceiling ${cap}. ${tokensFree} tokens free.`;
+		const legend = [
+			"Columns:",
+			"- cost: current cost of this scheme (vTokens for visible + sTokens for summarized)",
+			"- if-all-sum: cost if every entry of this scheme were demoted to summarized",
+			"- premium: savings from demoting visible → summarized (cost − if-all-sum)",
+		].join("\n");
+
+		return `${content}<budget tokenUsage="${tokenUsage}" tokensFree="${tokensFree}" fcrmScore="${fcrmScore}">\n${table}\n\n${legend}\n${systemLine}\n${totalLine}\n</budget>\n`;
+	}
+
+	#check({ contextSize, messages, rows, lastPromptTokens = 0 }) {
+		const totalTokens =
+			lastPromptTokens > 0 ? lastPromptTokens : measureMessages(messages);
+		const b = computeBudget({ rows, contextSize, totalTokens });
+		return {
+			messages,
+			rows,
+			assembledTokens: b.totalTokens,
+			overflow: b.overflow,
+			ok: b.ok,
+		};
+	}
+
+	async #emitOverflow({
+		message,
+		runId,
+		turn,
+		loopId,
+		rummy,
+		demotedCount = 0,
+		demotedTokens = 0,
+	}) {
+		await rummy.hooks.error.log.emit({
+			store: rummy.entries,
+			runId,
+			turn,
+			loopId,
+			message,
+			status: 413,
+			attributes: { demotedCount, demotedTokens },
+		});
 	}
 
-	async postDispatch({
+	/**
+	 * Pre-LLM budget enforcement. On first-turn overflow, demotes the
+	 * incoming prompt and re-materializes; re-checks and returns the
+	 * post-demotion result. If overflow persists after demotion (or on
+	 * later iterations), emits a 413 error (strike) and returns !ok so
+	 * TurnExecutor can skip the LLM call this turn.
+	 *
+	 * ctx = { runId, loopId, turn, systemPrompt, mode, toolSet, demoted,
+	 *         loopIteration }
+	 */
+	async enforce({
 		contextSize,
 		messages,
 		rows,
-		runId,
-		loopId,
-		turn,
-		db,
-		store,
+		lastPromptTokens = 0,
+		ctx,
+		rummy,
 	}) {
-		if (!contextSize) return null;
+		if (!contextSize) {
+			return { messages, rows, assembledTokens: 0, ok: true };
+		}
 
-		const postBudget = await this.enforce({
+		const first = this.#check({
 			contextSize,
 			messages,
 			rows,
-			lastPromptTokens: 0,
+			lastPromptTokens,
 		});
+		if (first.ok) return first;
 
-		if (postBudget.status !== 413) return null;
-
-		// Demote this turn's entries
-		const demotedEntries = await db.demote_turn_entries.all({
-			run_id: runId,
-			turn,
-		});
+		if (ctx?.loopIteration !== 1) {
+			const cap = ceiling(contextSize);
+			await this.#emitOverflow({
+				message: `Token Budget overflow: packet was ${cap + first.overflow} tokens, ceiling is ${cap}.`,
+				runId: ctx.runId,
+				turn: ctx.turn,
+				loopId: ctx.loopId,
+				rummy,
+			});
+			return first;
+		}
 
-		// Also demote the prompt
-		const promptRow = rows.find((r) => r.scheme === "prompt");
+		const promptRow = rows.findLast(
+			(r) => r.category === "prompt" && r.scheme === "prompt",
+		);
 		if (promptRow) {
-			await store.setFidelity(runId, promptRow.path, "demoted");
+			await rummy.entries.set({
+				runId: ctx.runId,
+				path: promptRow.path,
+				visibility: "summarized",
+			});
+		}
+		const reMat = await materializeContext({
+			db: rummy.db,
+			hooks: rummy.hooks,
+			runId: ctx.runId,
+			loopId: ctx.loopId,
+			turn: ctx.turn,
+			systemPrompt: ctx.systemPrompt,
+			mode: ctx.mode,
+			toolSet: ctx.toolSet,
+			contextSize,
+			demoted: ctx.demoted,
+		});
+		const rechecked = this.#check({
+			contextSize,
+			messages: reMat.messages,
+			rows: reMat.rows,
+			lastPromptTokens: reMat.lastContextTokens,
+		});
+		if (!rechecked.ok) {
+			const cap = ceiling(contextSize);
+			await this.#emitOverflow({
+				message: `Token Budget overflow: packet was ${cap + rechecked.overflow} tokens after demoting the prompt, ceiling is ${cap}.`,
+				runId: ctx.runId,
+				turn: ctx.turn,
+				loopId: ctx.loopId,
+				rummy,
+			});
 		}
+		return rechecked;
+	}
+
+	/**
+	 * Post-dispatch Turn Demotion. Re-materializes end-of-turn context and
+	 * checks against the ceiling. On overflow, demotes this turn's promoted
+	 * entries and emits a 413 error (strike) with the descriptive body so
+	 * the model sees it next turn via the unified error channel.
+	 *
+	 * ctx = { runId, loopId, turn, systemPrompt, mode, toolSet, demoted }
+	 */
+	async postDispatch({ contextSize, ctx, rummy }) {
+		if (!contextSize) return { failed: false };
+		const postMat = await materializeContext({
+			db: rummy.db,
+			hooks: rummy.hooks,
+			runId: ctx.runId,
+			loopId: ctx.loopId,
+			turn: ctx.turn,
+			systemPrompt: ctx.systemPrompt,
+			mode: ctx.mode,
+			toolSet: ctx.toolSet,
+			contextSize,
+			demoted: ctx.demoted,
+		});
+		// Baseline from this turn's actual API tokens (telemetry wrote it
+		// before post-dispatch runs). Delta from rows added this turn.
+		// Predicted next-turn packet stays on the tokenUsage scale the
+		// model can verify against its own arithmetic. materializeContext
+		// guarantees a number (0 when no prior API call exists).
+		const baseline = postMat.lastContextTokens;
+		const predicted = predictNextPacket(postMat.rows, ctx.turn, baseline);
+		const cap = ceiling(contextSize);
+		if (predicted <= cap) return { failed: false };
+		const post = { overflow: predicted - cap };
 
-		// Rewrite get-result bodies — the get handler claimed "promoted" success
-		// before this panic ran. Without rewriting, the model reads conflicting
-		// signals next turn (status=413 but body says "promoted").
-		for (const entry of demotedEntries) {
-			if (!entry.path.startsWith("get://")) continue;
-			await db.resolve_known_entry.run({
-				run_id: runId,
-				path: entry.path,
-				body: `Demoted by budget. See budget://${loopId}/${turn}.`,
-				status: 413,
+		const store = rummy.entries;
+		let demotedEntries = await store.demoteTurnEntries(ctx.runId, ctx.turn);
+		// Fallback: if this turn had nothing to demote but the packet still
+		// overflows, the pressure is coming from prior-turn promotions the
+		// model never demoted itself. Widen to all currently-visible
+		// entries in the run. Without this fallback, overflow-with-nothing
+		// strikes out runs where the base context has drifted over ceiling
+		// through no fault of the current turn (observed: runs where 3
+		// stale promotions from turns 12–14 saturate every subsequent
+		// turn's budget).
+		if (demotedEntries.length === 0) {
+			demotedEntries = await store.demoteRunVisibleEntries(ctx.runId);
+		}
+		const promptRow = postMat.rows.find((r) => r.scheme === "prompt");
+		if (promptRow) {
+			await store.set({
+				runId: ctx.runId,
+				path: promptRow.path,
+				visibility: "summarized",
 			});
 		}
 
-		// Write budget entry — terse, actionable. Path list dropped since
-		// demoted entries already render at fidelity="demoted" in <knowns>/<files>.
-		// "tokens remaining" dropped too — the number was over-optimistic (it
-		// treated re-demoted files as freeing their full-body tokens when their
-		// demoted-view renderings return to baseline). Model reads the truthful
-		// remaining in next turn's progress line.
-		//
-		// The 50% rule is the key directive: it forces the model to sum
-		// promotion costs (which is the behavior we want), and the threshold
-		// gives a concrete ceiling for the next try. Twofer — abiding by the
-		// rule requires budget awareness as a side effect.
-		const ceiling = Math.floor(contextSize * CEILING_RATIO);
 		const totalDemoted = demotedEntries.reduce((s, r) => s + r.tokens, 0);
-		const body = [
-			`413 Token Budget Error: overflowed by ${postBudget.overflow} tokens. Token Budget: ${ceiling}.`,
-			`Your ${demotedEntries.length} promotions from last turn (${totalDemoted} tokens total) were demoted to fit.`,
-			`Required: sum the tokens="N" of your promotions and new entries before emitting. A single turn must add no more than 50% of remaining Token Budget.`,
-		].join("\n");
-
-		await store.upsert(runId, turn, `budget://${loopId}/${turn}`, body, 413, {
-			loopId,
+		await this.#emitOverflow({
+			message: overflowBody(post.overflow, contextSize, demotedEntries),
+			demotedCount: demotedEntries.length,
+			demotedTokens: totalDemoted,
+			runId: ctx.runId,
+			turn: ctx.turn,
+			loopId: ctx.loopId,
+			rummy,
 		});
-
-		return {
-			target: ceiling,
-			promptPath: promptRow?.path ?? null,
-		};
+		return { failed: true };
 	}
 }

package/src/plugins/cp/README.md

@@ -1,4 +1,4 @@
-# cp
+# cp {#cp_plugin}
 
 Copies an entry from one path to another within the K/V store.
 
@@ -15,5 +15,5 @@ Shows `cp {from} {to}`.
 ## Behavior
 
 Warns if the destination already exists and will be overwritten. Uses
-`KnownStore.scheme()` to determine whether the destination is a scheme
+`Entries.scheme()` to determine whether the destination is a scheme
 path or a file path.

package/src/plugins/cp/cp.js

@@ -1,4 +1,4 @@
-import KnownStore from "../../agent/KnownStore.js";
+import Entries from "../../agent/Entries.js";
 import docs from "./cpDoc.js";
 
 export default class Cp {
@@ -8,8 +8,8 @@ export default class Cp {
 		this.#core = core;
 		core.registerScheme();
 		core.on("handler", this.handler.bind(this));
-		core.on("promoted", this.full.bind(this));
-		core.on("demoted", this.summary.bind(this));
+		core.on("visible", this.full.bind(this));
+		core.on("summarized", this.summary.bind(this));
 		core.filter("instructions.toolDocs", async (docsMap) => {
 			docsMap.cp = docs;
 			return docsMap;
@@ -19,15 +19,15 @@ export default class Cp {
 	async handler(entry, rummy) {
 		const { entries: store, sequence: turn, runId, loopId } = rummy;
 		const { path, to } = entry.attributes;
-		const VALID = { promoted: 1, demoted: 1, archived: 1 };
-		const fidelity = VALID[entry.attributes.fidelity]
-			? entry.attributes.fidelity
+		const VALID = { visible: 1, summarized: 1, archived: 1 };
+		const visibility = VALID[entry.attributes.visibility]
+			? entry.attributes.visibility
 			: undefined;
 
 		const source = await store.getBody(runId, path);
 		if (source === null) return;
 
-		const destScheme = KnownStore.scheme(to);
+		const destScheme = Entries.scheme(to);
 		const existing = await store.getBody(runId, to);
 		const warning =
 			existing !== null && destScheme !== null
@@ -36,13 +36,31 @@ export default class Cp {
 
 		const body = `${path} ${to}`;
 		if (destScheme === null) {
-			await store.upsert(runId, turn, entry.resultPath, body, 202, {
+			await store.set({
+				runId,
+				turn,
+				path: entry.resultPath,
+				body,
+				state: "proposed",
 				attributes: { from: path, to, isMove: false, warning },
 				loopId,
 			});
 		} else {
-			await store.upsert(runId, turn, to, source, 200, { fidelity, loopId });
-			await store.upsert(runId, turn, entry.resultPath, body, 200, {
+			await store.set({
+				runId,
+				turn,
+				path: to,
+				body: source,
+				state: "resolved",
+				visibility,
+				loopId,
+			});
+			await store.set({
+				runId,
+				turn,
+				path: entry.resultPath,
+				body,
+				state: "resolved",
 				attributes: { from: path, to, isMove: false, warning },
 				loopId,
 			});
@@ -50,7 +68,7 @@ export default class Cp {
 	}
 
 	full(entry) {
-		return `# cp ${entry.attributes.from || ""} ${entry.attributes.to || ""}`;
+		return `# cp ${entry.attributes.from} ${entry.attributes.to}`;
 	}
 
 	summary() {

package/src/plugins/cp/cpDoc.js

@@ -1,16 +1,3 @@
-// Tool doc for <cp>. Each entry: [text, rationale].
-// Text goes to the model. Rationale stays in source.
-// Changing ANY line requires reading ALL rationales first.
-const LINES = [
-	['## <cp path="[source]">[destination]</cp> - Copy a file or entry'],
-	[
-		'Example: <cp path="src/config.js">src/config.backup.js</cp>',
-		"Simple file copy. Path = source, body = destination.",
-	],
-	[
-		'Example: <cp path="known://plan_*">known://archive_</cp>',
-		"Glob batch copy across known entries.",
-	],
-];
+import { loadDoc } from "../helpers.js";
 
-export default LINES.map(([text]) => text).join("\n");
+export default loadDoc(import.meta.url, "cpDoc.md");

package/src/plugins/cp/cpDoc.md

@@ -0,0 +1,7 @@
+## <cp path="[source]">[destination]</cp> - Copy a file or entry
+
+Example: <cp path="src/config.js">src/config.backup.js</cp>
+<!-- Simple file copy. Path = source, body = destination. -->
+
+Example: <cp path="known://plan_*">known://archive_</cp>
+<!-- Glob batch copy across known entries. -->

package/src/plugins/engine/README.md

@@ -1,10 +1,10 @@
-# engine
+# engine {#engine_plugin}
 
 SQL infrastructure for context assembly and turn management. No JS plugin.
 
 ## Files
 
-- **engine.sql** — Queries for retrieving promoted entries by scheme tier, model visibility, and state.
+- **engine.sql** — Queries for retrieving visible entries by scheme tier, model visibility, and state.
 - **turn_context.sql** — Queries for clearing and reading the `turn_context` / `v_model_context` view, which produces the ordered context sent to the model.
 
 ## Behavior

package/src/plugins/engine/engine.sql

@@ -1,13 +1,13 @@
 -- PREP: get_promoted_entries
 SELECT
-	ke.path, ke.scheme, ke.status, ke.fidelity, ke.turn
-	, ke.tokens, ke.refs
+	ke.path, ke.scheme, ke.state, ke.outcome, ke.visibility, ke.turn
+	, countTokens(ke.body) AS tokens, ke.refs
 FROM known_entries AS ke
 JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
 WHERE
 	ke.run_id = :run_id
-	AND ke.fidelity IN ('promoted', 'demoted')
+	AND ke.visibility IN ('visible', 'summarized')
 	AND s.model_visible = 1
-ORDER BY ke.turn, ke.refs, ke.tokens DESC;
+ORDER BY ke.turn, ke.refs, countTokens(ke.body) DESC;
 
 

package/src/plugins/engine/turn_context.sql

@@ -4,33 +4,33 @@ WHERE run_id = :run_id AND turn = :turn;
 
 -- PREP: get_model_context
 SELECT
-	ordinal, path, scheme, fidelity, status, body
-	, tokens, attributes, category, turn
+	ordinal, path, scheme, visibility, state, outcome, body
+	, attributes, category, turn
 FROM v_model_context
 WHERE run_id = :run_id
 ORDER BY ordinal;
 
 -- PREP: insert_turn_context
 INSERT INTO turn_context (
-	run_id, loop_id, turn, ordinal, path, fidelity, status
-	, body, tokens, attributes, category, source_turn
+	run_id, loop_id, turn, ordinal, path, visibility, state, outcome
+	, body, attributes, category, source_turn
 )
 VALUES (
-	:run_id, :loop_id, :turn, :ordinal, :path, :fidelity
-	, :status, :body, :tokens
+	:run_id, :loop_id, :turn, :ordinal, :path, :visibility
+	, :state, :outcome, :body
 	, COALESCE(:attributes, '{}'), :category, :source_turn
 );
 
 -- PREP: get_turn_context
 SELECT
-	ordinal, path, scheme, fidelity, status, body
-	, tokens, attributes, category, source_turn
+	ordinal, path, scheme, visibility, state, outcome, body
+	, attributes, category, source_turn
 FROM turn_context
 WHERE run_id = :run_id AND turn = :turn
 ORDER BY ordinal;
 
 -- PREP: get_turn_budget
-SELECT COALESCE(SUM(tokens), 0) AS total
+SELECT COALESCE(SUM(countTokens(body)), 0) AS total
 FROM turn_context
 WHERE run_id = :run_id AND turn = :turn;
 
@@ -43,7 +43,7 @@ SELECT
 		WHEN 'prompt' THEN 'prompt'
 		ELSE 'system'
 	END AS bucket,
-	COALESCE(SUM(tokens), 0) AS tokens,
+	COALESCE(SUM(countTokens(body)), 0) AS tokens,
 	COUNT(*) AS entries
 FROM turn_context
 WHERE run_id = :run_id AND turn = :turn