@possumtech/rummy 0.3.0 → 0.4.0

package/src/plugins/set/set.js

@@ -1,9 +1,10 @@
 import KnownStore from "../../agent/KnownStore.js";
+import { countTokens } from "../../agent/tokens.js";
 import Hedberg, { generatePatch } from "../hedberg/hedberg.js";
 import { storePatternResult } from "../helpers.js";
 import docs from "./setDoc.js";
 
-const VALID_FIDELITY = { archive: 1, summary: 1, index: 1, full: 1 };
+const VALID_FIDELITY = { archive: 1, summary: 1, full: 1 };
 
 // biome-ignore lint/suspicious/noShadowRestrictedNames: tool name is "set"
 export default class Set {
@@ -25,66 +26,56 @@ export default class Set {
 	async handler(entry, rummy) {
 		const { entries: store, sequence: turn, runId, loopId } = rummy;
 		const attrs = entry.attributes;
-
-		// Fidelity control: <set path="..." fidelity="archive"/>
 		const fidelityAttr = VALID_FIDELITY[attrs.fidelity] ? attrs.fidelity : null;
-		if (fidelityAttr && attrs.path) {
+		const rawSummary =
+			typeof attrs.summary === "string" ? attrs.summary : null;
+		const summaryText = rawSummary ? rawSummary.slice(0, 80) : null;
+
+		// Pure fidelity/metadata change — no body content
+		if (!entry.body && fidelityAttr && attrs.path) {
 			const target = attrs.path;
-			const rawSummary =
-				typeof attrs.summary === "string" ? attrs.summary : null;
-			const summaryText = rawSummary ? rawSummary.slice(0, 80) : null;
 			const matches = await store.getEntriesByPattern(
 				runId,
 				target,
 				attrs.body,
 			);
-			if (entry.body) {
-				// Write content directly at specified fidelity
-				const entryAttrs = summaryText ? { summary: summaryText } : null;
-				for (const match of matches) {
-					await store.upsert(runId, turn, match.path, entry.body, 200, {
-						fidelity: fidelityAttr,
-						attributes: entryAttrs,
-						loopId,
-					});
-				}
-				if (matches.length === 0) {
-					await store.upsert(runId, turn, target, entry.body, 200, {
-						fidelity: fidelityAttr,
-						attributes: entryAttrs,
-						loopId,
+			if (matches.length === 0) {
+				await store.upsert(
+					runId,
+					turn,
+					entry.resultPath,
+					`${target} not found`,
+					404,
+					{ fidelity: "archive", loopId },
+				);
+				return;
+			}
+			for (const match of matches) {
+				await store.setFidelity(runId, match.path, fidelityAttr);
+				if (summaryText) {
+					await store.setAttributes(runId, match.path, {
+						summary: summaryText,
 					});
 				}
-			} else {
-				// No body — change fidelity, attach summary if provided
-				for (const match of matches) {
-					await store.setFidelity(runId, match.path, fidelityAttr);
-					if (summaryText) {
-						await store.setAttributes(runId, match.path, {
-							summary: summaryText,
-						});
-					}
-				}
 			}
 			const label =
 				fidelityAttr === "archive" ? "archived" : `set to ${fidelityAttr}`;
-			const body =
-				matches.length > 0
-					? `${matches.map((m) => m.path).join(", ")} ${label}`
-					: `${target} not found`;
-			await store.upsert(runId, turn, entry.resultPath, body, 200, {
-				fidelity: "archive",
-				loopId,
-			});
+			await store.upsert(
+				runId,
+				turn,
+				entry.resultPath,
+				`${matches.map((m) => m.path).join(", ")} ${label}`,
+				200,
+				{ fidelity: "archive", loopId },
+			);
 			return;
 		}
 
+		// Edit: sed patterns or SEARCH/REPLACE blocks
 		if (attrs.blocks || attrs.search != null) {
 			await this.#processEdit(rummy, entry, attrs);
-			return;
-		}
-
-		if (attrs.preview && attrs.path) {
+		} else if (attrs.preview && attrs.path) {
+			// Preview
 			const matches = await store.getEntriesByPattern(
 				runId,
 				attrs.path,
@@ -101,43 +92,68 @@ export default class Set {
 				{ preview: true, loopId },
 			);
 			return;
-		}
+		} else {
+			// Write content
+			const target = attrs.path;
+			if (!target) return;
 
-		const target = attrs.path;
-		if (!target) return;
+			const scheme = KnownStore.scheme(target);
+			if (scheme === null) {
+				// File write — diff against existing content
+				const existing = await store.getBody(runId, target);
+				const oldContent = existing ?? "";
+				const newContent = entry.body || "";
+				const udiff = generatePatch(target, oldContent, newContent);
+				const merge = oldContent
+					? `<<<<<<< SEARCH\n${oldContent}\n=======\n${newContent}\n>>>>>>> REPLACE`
+					: `<<<<<<< SEARCH\n=======\n${newContent}\n>>>>>>> REPLACE`;
+				await store.upsert(runId, turn, entry.resultPath, oldContent, 202, {
+					attributes: { file: target, patch: udiff, merge },
+					loopId,
+				});
+			} else if (attrs.filter || target.includes("*")) {
+				// Pattern update
+				const matches = await store.getEntriesByPattern(
+					runId,
+					target,
+					attrs.filter,
+				);
+				await store.updateBodyByPattern(
+					runId,
+					target,
+					attrs.filter || null,
+					entry.body,
+				);
+				await storePatternResult(
+					store,
+					runId,
+					turn,
+					"set",
+					target,
+					attrs.filter,
+					matches,
+					{ loopId },
+				);
+			} else {
+				// Direct scheme write
+				await store.upsert(runId, turn, target, entry.body, 200, {
+					fidelity: fidelityAttr || "full",
+					attributes: summaryText ? { summary: summaryText } : null,
+					loopId,
+				});
+			}
+		}
 
-		const scheme = KnownStore.scheme(target);
-		if (scheme === null) {
-			const udiff = generatePatch(target, "", entry.body || "");
-			const merge = `<<<<<<< SEARCH\n=======\n${entry.body || ""}\n>>>>>>> REPLACE`;
-			await store.upsert(runId, turn, entry.resultPath, "", 202, {
-				attributes: { file: target, patch: udiff, merge },
-				loopId,
-			});
-		} else if (attrs.filter || target.includes("*")) {
-			const matches = await store.getEntriesByPattern(
-				runId,
-				target,
-				attrs.filter,
-			);
-			await store.updateBodyByPattern(
-				runId,
-				target,
-				attrs.filter || null,
-				entry.body,
-			);
-			await storePatternResult(
-				store,
-				runId,
-				turn,
-				"set",
-				target,
-				attrs.filter,
-				matches,
-				{ loopId },
-			);
-		} else {
-			await store.upsert(runId, turn, target, entry.body, 200, { loopId });
+		// Apply fidelity after all write operations
+		if (fidelityAttr && attrs.path) {
+			const target = attrs.path;
+			const scheme = KnownStore.scheme(target);
+			if (scheme !== null) {
+				await store.setFidelity(runId, target, fidelityAttr);
+			}
+			if (summaryText) {
+				await store.setAttributes(runId, target, { summary: summaryText });
+			}
 		}
 	}
 
@@ -197,8 +213,8 @@ export default class Set {
 				searchText != null
 					? `<<<<<<< SEARCH\n${searchText}\n=======\n${replaceText}\n>>>>>>> REPLACE`
 					: null;
-			const beforeTokens = match.tokens_full || 0;
-			const afterTokens = patch ? (patch.length / 4) | 0 : beforeTokens;
+			const beforeTokens = match.tokens || 0;
+			const afterTokens = patch ? countTokens(patch) : beforeTokens;
 
 			await store.upsert(runId, turn, resultPath, match.body, status, {
 				attributes: {
@@ -264,8 +280,8 @@ export default class Set {
 					? generatePatch(filePath, original, current)
 					: null;
 			const merge = mergeBlocks.length > 0 ? mergeBlocks.join("\n") : null;
-			const beforeTokens = fileEntry[0].tokens_full || 0;
-			const afterTokens = current ? (current.length / 4) | 0 : beforeTokens;
+			const beforeTokens = fileEntry[0].tokens || 0;
+			const afterTokens = current ? countTokens(current) : beforeTokens;
 
 			await store.upsert(runId, turn, entry.path, original, state, {
 				attributes: {
@@ -287,10 +303,7 @@ export default class Set {
 			return { search: attrs.search, replace: attrs.replace ?? "" };
 		}
 		if (attrs.blocks?.length > 0) {
-			return {
-				search: attrs.blocks[0].search,
-				replace: attrs.blocks[0].replace,
-			};
+			return { blocks: attrs.blocks };
 		}
 		return null;
 	}
@@ -312,11 +325,31 @@ export default class Set {
 			};
 		}
 		if (body && attrs.blocks?.length > 0) {
-			const block = attrs.blocks[0];
-			return Hedberg.replace(body, block.search, block.replace, {
-				sed: block.sed,
-				flags: block.flags,
-			});
+			if (attrs.blocks.length === 1) {
+				const block = attrs.blocks[0];
+				return Hedberg.replace(body, block.search, block.replace, {
+					sed: block.sed,
+					flags: block.flags,
+				});
+			}
+			let current = body;
+			let lastWarning = null;
+			for (const block of attrs.blocks) {
+				const result = Hedberg.replace(current, block.search, block.replace, {
+					sed: block.sed,
+					flags: block.flags,
+				});
+				if (result.error) return result;
+				if (result.warning) lastWarning = result.warning;
+				if (result.patch) current = result.patch;
+			}
+			return {
+				patch: current !== body ? current : null,
+				searchText: null,
+				replaceText: null,
+				warning: lastWarning,
+				error: null,
+			};
 		}
 		return {
 			patch: null,

package/src/plugins/set/setDoc.js

@@ -2,44 +2,35 @@
 // Text goes to the model. Rationale stays in source.
 // Changing ANY line requires reading ALL rationales first.
 const LINES = [
-	// --- Syntax: path attr + body = edit content
-	['## <set path="[path/to/file]">[edit]</set> - Edit a file or entry'],
-
-	// --- Examples: sed, SEARCH/REPLACE, fidelity control
-	[
-		'Example: <set path="src/config.js">s/port = 3000/port = 8080/g</set>',
-		"Sed syntax: most common edit pattern. Shows s/old/new/ with g flag.",
-	],
-	[
-		`Example: <set path="src/app.js"><<<<<<< SEARCH
-// TODO: add error handling
+  [
+    '## <set path="[path/to/file]">[content or edit]</set> - Create, edit, or update a file or entry',
+  ],
+  [
+    'Example: <set path="known://project/milestones" fidelity="summary" summary="milestone,deadline,2026"/>',
+    "Fidelity control first — most unique capability of set.",
+  ],
+  [
+    `Example: <set path="src/app.js">
+<<<<<<< SEARCH
+old text
 =======
-// error handler configured
->>>>>>> REPLACE</set>`,
-		"SEARCH/REPLACE block: literal match and replace. Use when sed escaping is complex.",
-	],
-	[
-		'Example: <set path="known://plan" stored summary="Migration plan for Q2"/>',
-		"Fidelity + summary: archive an entry while preserving a description. Lifecycle endpoint.",
-	],
-
-	// --- Constraints
-	[
-		'* `fidelity="..."`: `archive`, `summary`, `index`, `full`',
-		"Fidelity control. Archive removes from context but preserves for retrieval.",
-	],
-	[
-		'* `summary="..."` (<= 80 chars) persists across fidelity changes',
-		"Model-authored descriptions survive demotion. No janitorial pass needed.",
-	],
-	[
-		"* YOU MUST NOT use <sh/> or <env/> to read, create, or edit files",
-		"Forces file operations through set/get. Prevents untracked mutations.",
-	],
-	[
-		"* Editing: s/old/new/ sed patterns and literal SEARCH/REPLACE blocks",
-		"Both syntaxes supported. Hedberg normalizes either form.",
-	],
+new text
+>>>>>>> REPLACE
+</set>`,
+    "SEARCH/REPLACE block — primary edit pattern for existing files.",
+  ],
+  [
+    'Example: <set path="src/config.js">s/port = 3000/port = 8080/g;s/host = 127.0.0.1/host = localhost/g;</set>',
+    "Sed syntax: chained s/old/new/ patterns with semicolons.",
+  ],
+  [
+    'Example: <set path="example.md">Full file content here</set>',
+    "Create: body contents are entire file.",
+  ],
+  [
+    "* YOU MUST NOT use <sh/> or <env/> to list, create, read, or edit files. Use the Tool Commands.",
+    "Forces file operations through set/get.",
+  ],
 ];
 
 export default LINES.map(([text]) => text).join("\n");

package/src/plugins/sh/shDoc.js

@@ -2,23 +2,18 @@
 // Text goes to the model. Rationale stays in source.
 // Changing ANY line requires reading ALL rationales first.
 const LINES = [
-	// --- Syntax
 	["## <sh>[command]</sh> - Run a shell command with side effects"],
-
-	// --- Examples: install and test — real mutations
 	[
 		"Example: <sh>npm install express</sh>",
-		"Package install. Shows a real side-effect command.",
+		"Package install. Real side-effect command.",
 	],
 	[
 		"Example: <sh>npm test</sh>",
 		"Test execution. Another common side-effect action.",
 	],
-
-	// --- Constraints
 	[
 		"* YOU MUST NOT use <sh/> to read, create, or edit files — use <get/> and <set/>",
-		"Forces file operations through the entry system. Prevents untracked mutations.",
+		"Forces file operations through the entry system.",
 	],
 	[
 		"* YOU MUST use <env/> for commands without side effects",

package/src/plugins/summarize/summarize.js

@@ -7,6 +7,7 @@ export default class Summarize {
 		this.#core = core;
 		core.ensureTool();
 		core.registerScheme({ category: "logging" });
+		core.on("handler", this.handler.bind(this));
 		core.on("full", this.full.bind(this));
 		core.on("summary", this.summary.bind(this));
 		core.filter("instructions.toolDocs", async (docsMap) => {
@@ -15,6 +16,12 @@ export default class Summarize {
 		});
 	}
 
+	async handler(entry, rummy) {
+		const { entries: store, sequence: turn, runId, loopId } = rummy;
+		const statusPath = await store.slugPath(runId, "summarize", entry.body);
+		await store.upsert(runId, turn, statusPath, entry.body, 200, { loopId });
+	}
+
 	full(entry) {
 		return `# summarize\n${entry.body}`;
 	}

package/src/plugins/summarize/summarizeDoc.js

@@ -2,31 +2,26 @@
 // Text goes to the model. Rationale stays in source.
 // Changing ANY line requires reading ALL rationales first.
 const LINES = [
-	// --- Syntax
 	["## <summarize>[answer or summary]</summarize> - Signal completion"],
-
-	// --- Examples: answer and task completion
 	[
 		"Example: <summarize>The port is 8080</summarize>",
-		"Direct answer. Shows summarize as the vehicle for delivering answers.",
+		"Direct answer. Summarize delivers answers.",
 	],
 	[
 		"Example: <summarize>Installed express, updated config</summarize>",
-		"Task summary. Shows summarize for action completion.",
+		"Task summary. Action completion.",
 	],
-
-	// --- Constraints: RFC-style MUST/MUST NOT
 	[
-		"* YOU MUST use <summarize> when done — describes the final state",
-		"Completion signal. Without this, the loop continues indefinitely.",
+		"* YOU MUST use <summarize></summarize> when done — describes the final state",
+		"Completion signal.",
 	],
 	[
 		"* YOU MUST NOT use <summarize> if still working — use <update/> instead",
-		"Mutual exclusion with update. Prevents premature completion.",
+		"Mutual exclusion with update.",
 	],
 	[
 		"* YOU MUST keep <summarize> to <= 80 characters",
-		"Length cap. Matches the summary attribute constraint. Prevents verbose output.",
+		"Length cap.",
 	],
 ];
 

package/src/plugins/telemetry/telemetry.js

@@ -1,4 +1,4 @@
-import { writeFileSync } from "node:fs";
+import { writeFile } from "node:fs/promises";
 import { join } from "node:path";
 
 export default class Telemetry {
@@ -85,17 +85,20 @@ export default class Telemetry {
 		// assistant://N — the model's raw response
 		await store.upsert(runId, turn, `assistant://${turn}`, content, 200, {
 			loopId,
+			fidelity: "archive",
 		});
 
 		// system://N, user://N — assembled messages as audit
 		if (systemMsg) {
 			await store.upsert(runId, turn, `system://${turn}`, systemMsg, 200, {
 				loopId,
+				fidelity: "archive",
 			});
 		}
 		if (userMsg) {
 			await store.upsert(runId, turn, `user://${turn}`, userMsg, 200, {
 				loopId,
+				fidelity: "archive",
 			});
 		}
 
@@ -112,7 +115,7 @@ export default class Telemetry {
 				model: result.model || null,
 			}),
 			200,
-			{ loopId },
+			{ loopId, fidelity: "archive" },
 		);
 
 		// reasoning://N
@@ -123,7 +126,7 @@ export default class Telemetry {
 				`reasoning://${turn}`,
 				responseMessage.reasoning_content,
 				200,
-				{ loopId },
+				{ loopId, fidelity: "archive" },
 			);
 		}
 
@@ -131,6 +134,7 @@ export default class Telemetry {
 		if (unparsed) {
 			await store.upsert(runId, turn, `content://${turn}`, unparsed, 200, {
 				loopId,
+				fidelity: "archive",
 			});
 		}
 
@@ -147,9 +151,12 @@ export default class Telemetry {
 			usage.completion_tokens_details?.reasoning_tokens ||
 			usage.output_tokens_details?.reasoning_tokens ||
 			0;
+		// Use LLM's actual prompt_tokens as the ground-truth context size when available.
+		// This back-fills context_tokens so get_last_context_tokens reflects reality for the next turn.
+		const actualContextTokens = usage.prompt_tokens || assembledTokens || 0;
 		await rummy.db.update_turn_stats.run({
 			id: rummy.turnId,
-			context_tokens: assembledTokens ?? 0,
+			context_tokens: actualContextTokens,
 			reasoning_content: responseMessage?.reasoning_content || null,
 			prompt_tokens: usage.prompt_tokens ?? 0,
 			cached_tokens: cachedTokens ?? 0,
@@ -189,10 +196,8 @@ export default class Telemetry {
 
 	#flush() {
 		if (!this.#lastRunPath || this.#turnLog.length === 0) return;
-		try {
-			writeFileSync(this.#lastRunPath, `${this.#turnLog.join("\n")}\n`);
-		} catch {
-			// RUMMY_HOME may not exist yet
-		}
+		writeFile(this.#lastRunPath, `${this.#turnLog.join("\n")}\n`).catch(
+			() => {},
+		);
 	}
 }

package/src/plugins/think/think.js

@@ -1,5 +1,17 @@
+import docs from "./thinkDoc.js";
+
+const THINK_ENABLED = process.env.RUMMY_THINK;
+if (THINK_ENABLED === undefined) throw new Error("RUMMY_THINK must be set (1 or 0)");
+
 export default class Think {
 	constructor(core) {
 		core.registerScheme({ modelVisible: 0, category: "logging" });
+		if (THINK_ENABLED === "1") {
+			core.ensureTool();
+			core.filter("instructions.toolDocs", async (docsMap) => {
+				docsMap.think = docs;
+				return docsMap;
+			});
+		}
 	}
 }

package/src/plugins/think/thinkDoc.js

@@ -0,0 +1,18 @@
+// Tool doc for <think/>. Each entry: [text, rationale].
+// Text goes to the model. Rationale stays in source.
+// Changing ANY line requires reading ALL rationales first.
+const LINES = [
+	[
+		"## <think>[reasoning]</think> - Think before acting",
+	],
+	[
+		"* Use <think> before any other tools to plan your approach",
+		"Positioning: think first, then act. Prevents degenerate tool-call storms.",
+	],
+	[
+		"* Reasoning inside <think> is private — it does not appear in your context",
+		"Frees the model to reason without consuming context budget.",
+	],
+];
+
+export default LINES.map(([text]) => text).join("\n");

package/src/plugins/unknown/README.md

@@ -20,4 +20,5 @@ The Rumsfeld mechanism. The model registers what it doesn't know before acting.
 Unknowns are sticky — they persist across turns until the model explicitly
 removes them with `<rm>`. The model investigates unknowns using `<get>`,
 `<env>`, or `<ask_user>`, then removes resolved ones. Server deduplicates
-on insert. Turn numbers shown on each unknown for temporal reasoning.
+on insert. Each unknown renders with turn, fidelity, and tokens for
+temporal reasoning and context management.

package/src/plugins/unknown/unknown.js

@@ -9,7 +9,9 @@ export default class Unknown {
 		core.registerScheme({
 			category: "unknown",
 		});
+		core.on("handler", this.handler.bind(this));
 		core.on("full", this.full.bind(this));
+		core.on("summary", this.summary.bind(this));
 		core.filter("assembly.system", this.assembleUnknowns.bind(this), 300);
 		core.filter("instructions.toolDocs", async (docsMap) => {
 			docsMap.unknown = docs;
@@ -17,18 +19,38 @@ export default class Unknown {
 		});
 	}
 
+	async handler(entry, rummy) {
+		const { entries: store, sequence: turn, runId, loopId } = rummy;
+
+		// Deduplicate — if this exact body already exists, skip
+		const existingValues = await store.getUnknownValues(runId);
+		if (existingValues.has(entry.body)) {
+			console.warn(`[RUMMY] Unknown deduped: "${entry.body.slice(0, 60)}"`);
+			return;
+		}
+
+		// Generate slug path and upsert
+		const unknownPath = await store.slugPath(runId, "unknown", entry.body);
+		await store.upsert(runId, turn, unknownPath, entry.body, 200, { loopId });
+	}
+
 	full(entry) {
 		return `# unknown\n${entry.body}`;
 	}
 
+	summary(entry) {
+		return this.full(entry);
+	}
+
 	async assembleUnknowns(content, ctx) {
 		const entries = ctx.rows.filter((r) => r.category === "unknown");
 		if (entries.length === 0) return content;
 
-		const lines = entries.map(
-			(u) =>
-				`<unknown path="${u.path}" turn="${u.source_turn || u.turn}">${u.body}</unknown>`,
-		);
+		const lines = entries.map((u) => {
+			const fidelity = u.fidelity ? ` fidelity="${u.fidelity}"` : "";
+			const tokens = u.tokens ? ` tokens="${u.tokens}"` : "";
+			return `<unknown path="${u.path}" turn="${u.source_turn || u.turn}"${fidelity}${tokens}>${u.body}</unknown>`;
+		});
 		return `${content}\n\n<unknowns>\n${lines.join("\n")}\n</unknowns>`;
 	}
 }

package/src/plugins/unknown/unknownDoc.js

@@ -2,29 +2,24 @@
 // Text goes to the model. Rationale stays in source.
 // Changing ANY line requires reading ALL rationales first.
 const LINES = [
-	// --- Syntax: body = what you need to learn
 	[
-		`## <unknown>[specific thing I need to learn]</unknown> - Track open questions`,
+		"## <unknown>[specific thing I need to learn]</unknown> - Track open questions",
 	],
-
-	// --- Examples: concrete unknowns, not abstract
 	[
-		`Example: <unknown path="unknown://answer">contents of answer.txt</unknown>`,
-		`Specific and actionable. Shows that unknowns are concrete investigation targets.`,
+		'Example: <unknown path="unknown://answer">contents of answer.txt</unknown>',
+		"Path form: explicit unknown path for structured tracking.",
 	],
 	[
-		`Example: <unknown>which database adapter is configured</unknown>`,
-		`Domain question. Shows unknowns for configuration/architecture questions.`,
+		"Example: <unknown>which database adapter is configured</unknown>",
+		"Body form: question as body, path auto-generated.",
 	],
-
-	// --- Lifecycle: register → investigate → resolve
 	[
-		`* Investigate with Tool Commands`,
-		`Cross-tool lifecycle: unknowns drive get/env/ask_user actions.`,
+		"* Investigate with Tool Commands",
+		"Unknowns drive action — get, env, search, ask_user.",
 	],
 	[
-		`* When resolved or irrelevant, remove with <rm path="unknown://..."/>`,
-		`Cross-tool lifecycle: rm cleans resolved unknowns from context.`,
+		'* When resolved or irrelevant, remove with <set path="unknown://..." fidelity="archive"/>',
+		"Archive instead of delete — preserves the question for context history.",
 	],
 ];