@possumtech/rummy 0.3.0 → 0.3.1

package/src/plugins/{current/current.js → performed/performed.js}

@@ -1,25 +1,26 @@
-export default class Current {
+export default class Performed {
 	#core;
 
 	constructor(core) {
 		this.#core = core;
-		core.filter("assembly.user", this.assembleCurrent.bind(this), 100);
+		core.filter("assembly.user", this.assemblePerformed.bind(this), 100);
 	}
 
-	async assembleCurrent(content, ctx) {
+	async assemblePerformed(content, ctx) {
 		const entries = ctx.rows.filter(
-			(r) => r.category === "logging" && r.source_turn >= ctx.loopStartTurn,
+			(r) =>
+				r.category === "logging" &&
+				r.source_turn >= ctx.loopStartTurn &&
+				r.scheme !== "unknown",
 		);
 		if (entries.length === 0) return content;
 
-		const lines = await Promise.all(
-			entries.map((e) => renderToolTag(e, this.#core)),
-		);
-		return `${content}<current>\n${lines.join("\n")}\n</current>\n`;
+		const lines = entries.map((e) => renderToolTag(e));
+		return `${content}<performed>\n${lines.join("\n")}\n</performed>\n`;
 	}
 }
 
-async function renderToolTag(entry, core) {
+function renderToolTag(entry) {
 	const attrs =
 		typeof entry.attributes === "string"
 			? JSON.parse(entry.attributes)
@@ -28,23 +29,17 @@ async function renderToolTag(entry, core) {
 	const target = attrs?.path || attrs?.file || attrs?.command || "";
 	const turn = entry.source_turn ? ` turn="${entry.source_turn}"` : "";
 	const status = entry.status ? ` status="${entry.status}"` : "";
+	const fidelity = entry.fidelity ? ` fidelity="${entry.fidelity}"` : "";
+	const tokens = entry.tokens ? ` tokens="${entry.tokens}"` : "";
 	const summary =
 		typeof attrs?.summary === "string"
 			? ` summary="${attrs.summary.slice(0, 80)}"`
 			: "";
 
-	let body;
-	try {
-		body = await core.hooks.tools.view(entry.scheme, {
-			...entry,
-			attributes: attrs,
-		});
-	} catch {
-		body = entry.body;
-	}
+	const body = entry.body || null;
 
 	if (body) {
-		return `<${entry.scheme} path="${target}"${turn}${status}${summary}>${body}</${entry.scheme}>`;
+		return `<${entry.scheme} path="${target}"${turn}${status}${summary}${fidelity}${tokens}>${body}</${entry.scheme}>`;
 	}
-	return `<${entry.scheme} path="${target}"${turn}${status}${summary}/>`;
+	return `<${entry.scheme} path="${target}"${turn}${status}${summary}${fidelity}${tokens}/>`;
 }

package/src/plugins/previous/README.md

@@ -12,4 +12,5 @@ history from prior ask/act invocations on this run.
 
 Filters turn_context rows where `category` is `logging` or `prompt`
 and `source_turn < loopStartTurn`. Renders each entry chronologically
-with turn number and status.
+with turn, status, summary, fidelity, and tokens. The model can target
+these entries by path with `<set>` or `<rm>` to free context space.

package/src/plugins/previous/previous.js

@@ -9,11 +9,20 @@ export default class Previous {
 	async assemblePrevious(content, ctx) {
 		if (ctx.loopStartTurn <= 1) return content;
 
-		const entries = ctx.rows.filter(
-			(r) =>
-				(r.category === "logging" || r.category === "prompt") &&
-				r.source_turn < ctx.loopStartTurn,
-		);
+		const entries = ctx.rows
+			.filter(
+				(r) =>
+					(r.category === "logging" || r.category === "prompt") &&
+					r.source_turn < ctx.loopStartTurn,
+			)
+			.toSorted((a, b) => {
+				if (a.source_turn !== b.source_turn)
+					return a.source_turn - b.source_turn;
+				// Within the same turn: prompt first (cause before effect)
+				if (a.category === "prompt" && b.category !== "prompt") return -1;
+				if (b.category === "prompt" && a.category !== "prompt") return 1;
+				return 0;
+			});
 		if (entries.length === 0) return content;
 
 		const lines = await Promise.all(
@@ -23,7 +32,7 @@ export default class Previous {
 	}
 }
 
-async function renderToolTag(entry, core) {
+async function renderToolTag(entry, _core) {
 	const attrs =
 		typeof entry.attributes === "string"
 			? JSON.parse(entry.attributes)
@@ -32,23 +41,20 @@ async function renderToolTag(entry, core) {
 	const target = attrs?.path || attrs?.file || attrs?.command || "";
 	const turn = entry.source_turn ? ` turn="${entry.source_turn}"` : "";
 	const status = entry.status ? ` status="${entry.status}"` : "";
-	const summary =
-		typeof attrs?.summary === "string"
-			? ` summary="${attrs.summary.slice(0, 80)}"`
-			: "";
-
-	let body;
-	try {
-		body = await core.hooks.tools.view(entry.scheme, {
-			...entry,
-			attributes: attrs,
-		});
-	} catch {
-		body = entry.body;
-	}
-
-	if (body) {
-		return `<${entry.scheme} path="${target}"${turn}${status}${summary}>${body}</${entry.scheme}>`;
-	}
-	return `<${entry.scheme} path="${target}"${turn}${status}${summary}/>`;
+	const fidelity = entry.fidelity ? ` fidelity="${entry.fidelity}"` : "";
+	const tokens = entry.tokens ? ` tokens="${entry.tokens}"` : "";
+
+	// Previous entries render at summary. Prompts get 512 chars for orientation.
+	const limit = entry.scheme === "prompt" ? 512 : 80;
+	const rawSummary =
+		(typeof attrs?.summary === "string" ? attrs.summary : null) ||
+		entry.body?.slice(0, limit) ||
+		"";
+	// Strip internal dedup namespace prefixes (e.g. "get://turn_3/src/app.js" → "src/app.js")
+	const summaryText = rawSummary.replace(/\b\w+:\/\/turn_\d+\//g, "");
+	const summaryAttr = summaryText
+		? ` summary="${summaryText.replace(/"/g, "'").slice(0, limit)}"`
+		: "";
+
+	return `<${entry.scheme} path="${target}"${turn}${status}${summaryAttr}${fidelity}${tokens}/>`;
 }

package/src/plugins/progress/README.md

@@ -9,9 +9,8 @@ current work log to the active prompt.
 
 ## Behavior
 
-On first turn: "Begin."
+Emits `<progress turn="N">` carrying token budget and fidelity stats.
 On continuation turns with current entries: "The above actions were
 performed in response to the following prompt:"
-If a `progress://` entry exists, uses its body directly.
 
 Progress text is the tuning knob for model orientation between turns.

package/src/plugins/progress/progress.js

@@ -7,38 +7,34 @@ export default class Progress {
 	}
 
 	async assembleProgress(content, ctx) {
-		// Use last turn's real assembled token count when available.
-		// Falls back to row token sum (less accurate — missing system prompt overhead).
-		const rowTokens = ctx.rows.reduce((sum, r) => sum + (r.tokens || 0), 0);
-		const usedTokens = ctx.lastContextTokens || rowTokens;
-		const contextSize = ctx.contextSize || 0;
+		const { lastContextTokens: usedTokens, contextSize } = ctx;
 		const pct = contextSize ? Math.round((usedTokens / contextSize) * 100) : 0;
 
-		// Fidelity distribution across known/file entries
-		const entries = ctx.rows.filter((r) => r.category === "data");
+		// Fidelity distribution across all manageable entries (data + logging)
+		const dataEntries = ctx.rows.filter((r) => r.category === "data");
+		const loggingEntries = ctx.rows.filter((r) => r.category === "logging");
+		const entries = [...dataEntries, ...loggingEntries];
 		const fullEntries = entries.filter((r) => r.fidelity === "full");
 		const summaryEntries = entries.filter((r) => r.fidelity === "summary");
 		const indexEntries = entries.filter((r) => r.fidelity === "index");
-		const fullTokens = fullEntries.reduce((s, r) => s + (r.tokens || 0), 0);
-		const summaryTokens = summaryEntries.reduce(
-			(s, r) => s + (r.tokens || 0),
-			0,
-		);
-		const indexTokens = indexEntries.reduce((s, r) => s + (r.tokens || 0), 0);
+		const fullTokens = fullEntries.reduce((s, r) => s + r.tokens, 0);
+		const summaryTokens = summaryEntries.reduce((s, r) => s + r.tokens, 0);
+		const indexTokens = indexEntries.reduce((s, r) => s + r.tokens, 0);
 
 		const unknownCount = ctx.rows.filter(
 			(r) => r.category === "unknown",
 		).length;
 
-		const hasCurrent = ctx.rows.some(
-			(r) => r.category === "logging" && r.source_turn >= ctx.loopStartTurn,
+		const hasPerformed = loggingEntries.some(
+			(r) => r.source_turn >= ctx.loopStartTurn,
 		);
 
 		const parts = [];
 
-		const knownCount = entries.length;
+		const knownCount = dataEntries.length;
+		const loggingCount = loggingEntries.length;
 		const tokenLine = contextSize
-			? `${usedTokens} of ${contextSize} tokens (${pct}%) · ${knownCount} known${knownCount !== 1 ? "s" : ""} · ${unknownCount} unknown${unknownCount !== 1 ? "s" : ""}`
+			? `${usedTokens} of ${contextSize} tokens (${pct}%) · ${knownCount} known${knownCount !== 1 ? "s" : ""} · ${loggingCount} logging · ${unknownCount} unknown${unknownCount !== 1 ? "s" : ""}`
 			: "";
 		if (tokenLine) parts.push(tokenLine);
 
@@ -55,22 +51,12 @@ export default class Progress {
 		if (fidelityParts.length > 0)
 			parts.push(`Entries: ${fidelityParts.join(" · ")}`);
 
-		if (pct > 75) {
-			parts.push(
-				'Context above 75%. YOU MUST free space: <set fidelity="summary" summary="topic,detail,keyword"/>, <set fidelity="archive"/>, or <rm/>. Target the largest entries.',
-			);
-		} else if (pct > 50) {
-			parts.push(
-				'Context above 50%. You may free space: <set fidelity="summary" summary="topic,detail,keyword"/>, <set fidelity="archive"/>, or <rm/>.',
-			);
-		}
-
-		if (hasCurrent) {
+		if (hasPerformed) {
 			parts.push(
 				"The above actions were performed in response to the following prompt:",
 			);
 		}
 
-		return `${content}<progress>${parts.join("\n")}</progress>\n`;
+		return `${content}<progress turn="${ctx.turn}">${parts.join("\n")}</progress>\n`;
 	}
 }

package/src/plugins/prompt/prompt.js

@@ -4,7 +4,6 @@ export default class Prompt {
 	constructor(core) {
 		this.#core = core;
 		core.hooks.tools.onView("prompt", (entry) => entry.body);
-		core.hooks.tools.onView("progress", (entry) => entry.body);
 		core.on("turn.started", this.onTurnStarted.bind(this));
 		core.filter("assembly.user", this.assemblePrompt.bind(this), 300);
 	}
@@ -17,10 +16,6 @@ export default class Prompt {
 				attributes: { mode },
 				loopId,
 			});
-		} else {
-			await store.upsert(runId, turn, `progress://${turn}`, prompt || "", 200, {
-				loopId,
-			});
 		}
 	}
 
@@ -41,8 +36,6 @@ export default class Prompt {
 		const tools = toolNames.join(",");
 		let warn = "";
 		if (mode === "ask") warn = ' warn="File editing disallowed."';
-		if (mode === "panic")
-			warn = ' warn="Context overflow. Free space to continue."';
 
 		return `${content}<prompt mode="${mode}" tools="${tools}"${warn}>${body}</prompt>`;
 	}

package/src/plugins/rm/rm.js

@@ -41,25 +41,37 @@ export default class Rm {
 			return;
 		}
 
-		for (const match of matches) {
-			const resultPath = `rm://${match.path}`;
-			if (match.scheme === null) {
-				await store.upsert(runId, turn, resultPath, match.path, 202, {
-					attributes: { path: match.path },
-					loopId,
-				});
-			} else {
-				await store.remove(runId, match.path);
-				await store.upsert(runId, turn, resultPath, match.path, 200, {
-					attributes: { path: match.path },
-					loopId,
-				});
-			}
+		const fileMatches = matches.filter((m) => m.scheme === null);
+		const schemeMatches = matches.filter((m) => m.scheme !== null);
+
+		// Scheme entries: remove all, write one aggregate result entry
+		for (const match of schemeMatches) await store.remove(runId, match.path);
+		if (schemeMatches.length > 0) {
+			const paths = schemeMatches.map((m) => m.path).join("\n");
+			await store.upsert(runId, turn, entry.resultPath, paths, 200, {
+				attributes: { path: target },
+				loopId,
+			});
+		}
+
+		// File entries: individual 202 proposals (require user resolution)
+		if (fileMatches.length > 0 && schemeMatches.length > 0)
+			await store.remove(runId, entry.resultPath);
+		for (const match of fileMatches) {
+			const resultPath =
+				schemeMatches.length === 0 && fileMatches.length === 1
+					? entry.resultPath
+					: await store.dedup(runId, "rm", match.path, turn);
+			await store.upsert(runId, turn, resultPath, match.path, 202, {
+				attributes: { path: match.path },
+				loopId,
+			});
 		}
 	}
 
 	full(entry) {
-		return `# rm ${entry.attributes.path || entry.path}`;
+		const header = `# rm ${entry.attributes.path || entry.path}`;
+		return entry.body ? `${header}\n${entry.body}` : header;
 	}
 
 	summary(entry) {

package/src/plugins/rm/rmDoc.js

@@ -8,8 +8,8 @@ const LINES = [
 	// --- Examples: file, known (with slug path), preview safety
 	['Example: <rm path="src/config.js"/>', "File removal. Simplest form."],
 	[
-		'Example: <rm path="known://donald-rumsfeld-was-born-in-1932"/>',
-		"Shows the slugified path convention. Model sees these paths in <knowns> section.",
+		'Example: <rm path="known://config/deprecated_service"/>',
+		"Shows topic-hierarchy path convention. Paths are category/key, not sentence slugs.",
 	],
 	[
 		'Example: <rm path="known://temp_*" preview/>',
@@ -22,7 +22,7 @@ const LINES = [
 		"Nudges toward archive over rm. Archive keeps the key; rm deletes permanently.",
 	],
 	[
-		"* Paths accept globs — use `preview` to check matches first",
+		"* Paths accept patterns — use `preview` to check matches first",
 		"Reinforces preview safety pattern. Prevents accidental bulk deletion.",
 	],
 ];

package/src/plugins/set/set.js

@@ -1,4 +1,5 @@
 import KnownStore from "../../agent/KnownStore.js";
+import { countTokens } from "../../agent/tokens.js";
 import Hedberg, { generatePatch } from "../hedberg/hedberg.js";
 import { storePatternResult } from "../helpers.js";
 import docs from "./setDoc.js";
@@ -66,16 +67,33 @@ export default class Set {
 					}
 				}
 			}
+			if (matches.length === 0) {
+				await store.upsert(
+					runId,
+					turn,
+					entry.resultPath,
+					`${target} not found`,
+					404,
+					{
+						fidelity: "archive",
+						loopId,
+					},
+				);
+				return;
+			}
 			const label =
 				fidelityAttr === "archive" ? "archived" : `set to ${fidelityAttr}`;
-			const body =
-				matches.length > 0
-					? `${matches.map((m) => m.path).join(", ")} ${label}`
-					: `${target} not found`;
-			await store.upsert(runId, turn, entry.resultPath, body, 200, {
-				fidelity: "archive",
-				loopId,
-			});
+			await store.upsert(
+				runId,
+				turn,
+				entry.resultPath,
+				`${matches.map((m) => m.path).join(", ")} ${label}`,
+				200,
+				{
+					fidelity: "archive",
+					loopId,
+				},
+			);
 			return;
 		}
 
@@ -198,7 +216,7 @@ export default class Set {
 					? `<<<<<<< SEARCH\n${searchText}\n=======\n${replaceText}\n>>>>>>> REPLACE`
 					: null;
 			const beforeTokens = match.tokens_full || 0;
-			const afterTokens = patch ? (patch.length / 4) | 0 : beforeTokens;
+			const afterTokens = patch ? countTokens(patch) : beforeTokens;
 
 			await store.upsert(runId, turn, resultPath, match.body, status, {
 				attributes: {
@@ -265,7 +283,7 @@ export default class Set {
 					: null;
 			const merge = mergeBlocks.length > 0 ? mergeBlocks.join("\n") : null;
 			const beforeTokens = fileEntry[0].tokens_full || 0;
-			const afterTokens = current ? (current.length / 4) | 0 : beforeTokens;
+			const afterTokens = current ? countTokens(current) : beforeTokens;
 
 			await store.upsert(runId, turn, entry.path, original, state, {
 				attributes: {
@@ -287,10 +305,7 @@ export default class Set {
 			return { search: attrs.search, replace: attrs.replace ?? "" };
 		}
 		if (attrs.blocks?.length > 0) {
-			return {
-				search: attrs.blocks[0].search,
-				replace: attrs.blocks[0].replace,
-			};
+			return { blocks: attrs.blocks };
 		}
 		return null;
 	}
@@ -312,11 +327,32 @@ export default class Set {
 			};
 		}
 		if (body && attrs.blocks?.length > 0) {
-			const block = attrs.blocks[0];
-			return Hedberg.replace(body, block.search, block.replace, {
-				sed: block.sed,
-				flags: block.flags,
-			});
+			if (attrs.blocks.length === 1) {
+				const block = attrs.blocks[0];
+				return Hedberg.replace(body, block.search, block.replace, {
+					sed: block.sed,
+					flags: block.flags,
+				});
+			}
+			// Multi-block: apply sequentially, no per-hunk merge notation
+			let current = body;
+			let lastWarning = null;
+			for (const block of attrs.blocks) {
+				const result = Hedberg.replace(current, block.search, block.replace, {
+					sed: block.sed,
+					flags: block.flags,
+				});
+				if (result.error) return result;
+				if (result.warning) lastWarning = result.warning;
+				if (result.patch) current = result.patch;
+			}
+			return {
+				patch: current !== body ? current : null,
+				searchText: null,
+				replaceText: null,
+				warning: lastWarning,
+				error: null,
+			};
 		}
 		return {
 			patch: null,

package/src/plugins/set/setDoc.js

@@ -19,8 +19,8 @@ const LINES = [
 		"SEARCH/REPLACE block: literal match and replace. Use when sed escaping is complex.",
 	],
 	[
-		'Example: <set path="known://plan" stored summary="Migration plan for Q2"/>',
-		"Fidelity + summary: archive an entry while preserving a description. Lifecycle endpoint.",
+		'Example: <set path="known://project/milestones" fidelity="summary" summary="milestone,deadline,2026"/> ... <set path="prompt://3" fidelity="index"/>',
+		"Fidelity control: compress a known entry to keywords, demote a previous prompt to index-only. Both free context while keeping paths visible.",
 	],
 
 	// --- Constraints
@@ -28,6 +28,10 @@ const LINES = [
 		'* `fidelity="..."`: `archive`, `summary`, `index`, `full`',
 		"Fidelity control. Archive removes from context but preserves for retrieval.",
 	],
+	[
+		'* `fidelity="summary"` HIDES the body — does NOT require reading or compressing content. Write any short keyword label you already know.',
+		"M-10 fix: model was reading files before compressing to summary, believing it needed semantic content. It does not. The body is preserved on disk; only context visibility changes.",
+	],
 	[
 		'* `summary="..."` (<= 80 chars) persists across fidelity changes',
 		"Model-authored descriptions survive demotion. No janitorial pass needed.",

package/src/plugins/telemetry/telemetry.js

@@ -1,4 +1,4 @@
-import { writeFileSync } from "node:fs";
+import { writeFile } from "node:fs/promises";
 import { join } from "node:path";
 
 export default class Telemetry {
@@ -85,17 +85,20 @@ export default class Telemetry {
 		// assistant://N — the model's raw response
 		await store.upsert(runId, turn, `assistant://${turn}`, content, 200, {
 			loopId,
+			fidelity: "archive",
 		});
 
 		// system://N, user://N — assembled messages as audit
 		if (systemMsg) {
 			await store.upsert(runId, turn, `system://${turn}`, systemMsg, 200, {
 				loopId,
+				fidelity: "archive",
 			});
 		}
 		if (userMsg) {
 			await store.upsert(runId, turn, `user://${turn}`, userMsg, 200, {
 				loopId,
+				fidelity: "archive",
 			});
 		}
 
@@ -112,7 +115,7 @@ export default class Telemetry {
 				model: result.model || null,
 			}),
 			200,
-			{ loopId },
+			{ loopId, fidelity: "archive" },
 		);
 
 		// reasoning://N
@@ -123,7 +126,7 @@ export default class Telemetry {
 				`reasoning://${turn}`,
 				responseMessage.reasoning_content,
 				200,
-				{ loopId },
+				{ loopId, fidelity: "archive" },
 			);
 		}
 
@@ -131,6 +134,7 @@ export default class Telemetry {
 		if (unparsed) {
 			await store.upsert(runId, turn, `content://${turn}`, unparsed, 200, {
 				loopId,
+				fidelity: "archive",
 			});
 		}
 
@@ -147,9 +151,12 @@ export default class Telemetry {
 			usage.completion_tokens_details?.reasoning_tokens ||
 			usage.output_tokens_details?.reasoning_tokens ||
 			0;
+		// Use LLM's actual prompt_tokens as the ground-truth context size when available.
+		// This back-fills context_tokens so get_last_context_tokens reflects reality for the next turn.
+		const actualContextTokens = usage.prompt_tokens || assembledTokens || 0;
 		await rummy.db.update_turn_stats.run({
 			id: rummy.turnId,
-			context_tokens: assembledTokens ?? 0,
+			context_tokens: actualContextTokens,
 			reasoning_content: responseMessage?.reasoning_content || null,
 			prompt_tokens: usage.prompt_tokens ?? 0,
 			cached_tokens: cachedTokens ?? 0,
@@ -189,10 +196,8 @@ export default class Telemetry {
 
 	#flush() {
 		if (!this.#lastRunPath || this.#turnLog.length === 0) return;
-		try {
-			writeFileSync(this.#lastRunPath, `${this.#turnLog.join("\n")}\n`);
-		} catch {
-			// RUMMY_HOME may not exist yet
-		}
+		writeFile(this.#lastRunPath, `${this.#turnLog.join("\n")}\n`).catch(
+			() => {},
+		);
 	}
 }

package/src/plugins/unknown/README.md

@@ -20,4 +20,5 @@ The Rumsfeld mechanism. The model registers what it doesn't know before acting.
 Unknowns are sticky — they persist across turns until the model explicitly
 removes them with `<rm>`. The model investigates unknowns using `<get>`,
 `<env>`, or `<ask_user>`, then removes resolved ones. Server deduplicates
-on insert. Turn numbers shown on each unknown for temporal reasoning.
+on insert. Each unknown renders with turn, fidelity, and tokens for
+temporal reasoning and context management.

package/src/plugins/unknown/unknown.js

@@ -25,10 +25,11 @@ export default class Unknown {
 		const entries = ctx.rows.filter((r) => r.category === "unknown");
 		if (entries.length === 0) return content;
 
-		const lines = entries.map(
-			(u) =>
-				`<unknown path="${u.path}" turn="${u.source_turn || u.turn}">${u.body}</unknown>`,
-		);
+		const lines = entries.map((u) => {
+			const fidelity = u.fidelity ? ` fidelity="${u.fidelity}"` : "";
+			const tokens = u.tokens ? ` tokens="${u.tokens}"` : "";
+			return `<unknown path="${u.path}" turn="${u.source_turn || u.turn}"${fidelity}${tokens}>${u.body}</unknown>`;
+		});
 		return `${content}\n\n<unknowns>\n${lines.join("\n")}\n</unknowns>`;
 	}
 }