@possumtech/rummy 0.4.0 → 2.0.0

package/src/plugins/set/README.md

@@ -1,4 +1,4 @@
-# set
+# set {#set_plugin}
 
 Writes or edits entry content. Handles new files, full overwrites,
 SEARCH/REPLACE edits, and pattern updates.

package/src/plugins/set/set.js

@@ -1,10 +1,17 @@
-import KnownStore from "../../agent/KnownStore.js";
+import Entries from "../../agent/Entries.js";
 import { countTokens } from "../../agent/tokens.js";
+import File from "../file/file.js";
 import Hedberg, { generatePatch } from "../hedberg/hedberg.js";
 import { storePatternResult } from "../helpers.js";
 import docs from "./setDoc.js";
 
-const VALID_FIDELITY = { archive: 1, summary: 1, full: 1 };
+const VALID_VISIBILITY = { archived: 1, summarized: 1, visible: 1 };
+const LOG_ACTION_RE = /^log:\/\/turn_\d+\/(\w+)\//;
+
+function isSetProposal(path) {
+	const m = LOG_ACTION_RE.exec(path);
+	return m?.[1] === "set";
+}
 
 // biome-ignore lint/suspicious/noShadowRestrictedNames: tool name is "set"
 export default class Set {
@@ -14,25 +21,121 @@ export default class Set {
 		this.#core = core;
 		core.registerScheme();
 		core.on("handler", this.handler.bind(this));
-		core.on("full", this.full.bind(this));
-		core.on("summary", this.summary.bind(this));
-		core.on("turn.proposing", this.#materializeRevisions.bind(this));
+		core.on("visible", this.full.bind(this));
+		core.on("summarized", this.summary.bind(this));
+		core.on("proposal.prepare", this.#materializeRevisions.bind(this));
 		core.filter("instructions.toolDocs", async (docsMap) => {
 			docsMap.set = docs;
 			return docsMap;
 		});
+		core.filter("proposal.accepting", this.#vetoReadonly.bind(this));
+		core.filter("proposal.content", this.#preferExistingBody.bind(this));
+		core.on("proposal.accepted", this.#materializeFile.bind(this));
+	}
+
+	async #vetoReadonly(current, ctx) {
+		if (current) return current;
+		if (!isSetProposal(ctx.path)) return current;
+		if (!ctx.attrs?.path) return current;
+		const blocked = await File.isReadonly(
+			ctx.db,
+			ctx.projectId,
+			ctx.attrs.path,
+		);
+		if (!blocked) return current;
+		return {
+			allow: false,
+			outcome: "readonly",
+			body: `refused: ${ctx.attrs.path} is readonly`,
+		};
+	}
+
+	async #preferExistingBody(defaultBody, ctx) {
+		if (!isSetProposal(ctx.path)) return defaultBody;
+		const existing = await ctx.entries.getBody(ctx.runId, ctx.path);
+		if (existing) return existing;
+		return defaultBody;
+	}
+
+	async #materializeFile(ctx) {
+		if (!isSetProposal(ctx.path)) return;
+		const { attrs, runId, projectId, projectRoot, db, entries } = ctx;
+		if (!attrs?.path || !attrs?.merge) return;
+
+		const existing = await entries.getBody(runId, attrs.path);
+		const isNewFile = existing === null;
+		const fileBody = isNewFile ? "" : existing;
+		const blocks = attrs.merge.split(/(?=<<<<<<< SEARCH)/);
+		let patched = fileBody;
+		for (const block of blocks) {
+			const m = block.match(
+				/<<<<<<< SEARCH\n?([\s\S]*?)\n?=======\n?([\s\S]*?)\n?>>>>>>> REPLACE/,
+			);
+			if (!m) continue;
+			if (m[1] === "") {
+				patched = m[2];
+			} else {
+				patched = patched.replace(m[1], m[2]);
+			}
+		}
+		const turn = (await db.get_run_by_id.get({ id: runId })).next_turn;
+		// Preserve the file entry's current visibility — a <get>
+		// earlier in the run may have promoted it. Updating the
+		// body without specifying visibility falls through to
+		// the data-category default ("summarized") and wipes
+		// the promotion, making the model re-get the file next
+		// turn (then cycle-strike out).
+		const existingState = await entries.getState(runId, attrs.path);
+		await entries.set({
+			runId,
+			turn,
+			path: attrs.path,
+			body: patched,
+			visibility: existingState?.visibility,
+		});
+		if (projectRoot) {
+			const { writeFile } = await import("node:fs/promises");
+			const { join } = await import("node:path");
+			await writeFile(join(projectRoot, attrs.path), patched).catch(() => {});
+		}
+		if (isNewFile && projectId) {
+			await File.setConstraint(db, projectId, attrs.path, "active");
+		}
 	}
 
 	async handler(entry, rummy) {
 		const { entries: store, sequence: turn, runId, loopId } = rummy;
 		const attrs = entry.attributes;
-		const fidelityAttr = VALID_FIDELITY[attrs.fidelity] ? attrs.fidelity : null;
-		const rawSummary =
-			typeof attrs.summary === "string" ? attrs.summary : null;
+		const visibilityAttr = VALID_VISIBILITY[attrs.visibility]
+			? attrs.visibility
+			: null;
+		const rawSummary = typeof attrs.summary === "string" ? attrs.summary : null;
 		const summaryText = rawSummary ? rawSummary.slice(0, 80) : null;
 
-		// Pure fidelity/metadata change — no body content
-		if (!entry.body && fidelityAttr && attrs.path) {
+		// Invalid visibility value on a body-less set: reject with an
+		// error instead of falling through to the write path. Without
+		// this guard, a typo like visibility="promoted" (pre-migration
+		// terminology) silently body-wiped the target — the fidelity
+		// regression that cost us multiple demo runs.
+		if (
+			!entry.body &&
+			attrs.path &&
+			attrs.visibility !== undefined &&
+			!visibilityAttr
+		) {
+			await rummy.hooks.error.log.emit({
+				store,
+				runId,
+				turn,
+				loopId,
+				message: `Invalid visibility "${attrs.visibility}" on <set path="${attrs.path}"/>. Use visibility="visible|summarized|archived".`,
+				status: 400,
+			});
+			return;
+		}
+
+		// Pure visibility/metadata change — no body content
+		if (!entry.body && visibilityAttr && attrs.path) {
 			const target = attrs.path;
 			const matches = await store.getEntriesByPattern(
 				runId,
@@ -40,34 +143,44 @@ export default class Set {
 				attrs.body,
 			);
 			if (matches.length === 0) {
-				await store.upsert(
+				await store.set({
 					runId,
 					turn,
-					entry.resultPath,
-					`${target} not found`,
-					404,
-					{ fidelity: "archive", loopId },
-				);
+					path: entry.resultPath,
+					body: `${target} not found`,
+					state: "failed",
+					outcome: "not_found",
+					visibility: "archived",
+					loopId,
+				});
 				return;
 			}
 			for (const match of matches) {
-				await store.setFidelity(runId, match.path, fidelityAttr);
+				await store.set({
+					runId: runId,
+					path: match.path,
+					visibility: visibilityAttr,
+				});
 				if (summaryText) {
-					await store.setAttributes(runId, match.path, {
-						summary: summaryText,
+					await store.set({
+						runId: runId,
+						path: match.path,
+						attributes: {
+							summary: summaryText,
+						},
 					});
 				}
 			}
-			const label =
-				fidelityAttr === "archive" ? "archived" : `set to ${fidelityAttr}`;
-			await store.upsert(
+			const label = `set to ${visibilityAttr}`;
+			await store.set({
 				runId,
 				turn,
-				entry.resultPath,
-				`${matches.map((m) => m.path).join(", ")} ${label}`,
-				200,
-				{ fidelity: "archive", loopId },
-			);
+				path: entry.resultPath,
+				body: `${matches.map((m) => m.path).join(", ")} ${label}`,
+				state: "resolved",
+				visibility: "archived",
+				loopId,
+			});
 			return;
 		}
 
@@ -97,18 +210,32 @@ export default class Set {
 			const target = attrs.path;
 			if (!target) return;
 
-			const scheme = KnownStore.scheme(target);
+			const scheme = Entries.scheme(target);
 			if (scheme === null) {
 				// File write — diff against existing content
 				const existing = await store.getBody(runId, target);
-				const oldContent = existing ?? "";
-				const newContent = entry.body || "";
+				const oldContent = existing === null ? "" : existing;
+				const newContent = entry.body;
 				const udiff = generatePatch(target, oldContent, newContent);
 				const merge = oldContent
 					? `<<<<<<< SEARCH\n${oldContent}\n=======\n${newContent}\n>>>>>>> REPLACE`
 					: `<<<<<<< SEARCH\n=======\n${newContent}\n>>>>>>> REPLACE`;
-				await store.upsert(runId, turn, entry.resultPath, oldContent, 202, {
-					attributes: { file: target, patch: udiff, merge },
+				const beforeTokens = oldContent ? countTokens(oldContent) : 0;
+				const afterTokens = countTokens(newContent);
+				await store.set({
+					runId,
+					turn,
+					path: entry.resultPath,
+					body: newContent,
+					state: "proposed",
+					attributes: {
+						path: target,
+						patch: udiff,
+						merge,
+						beforeTokens,
+						afterTokens,
+						summary: summaryText,
+					},
 					loopId,
 				});
 			} else if (attrs.filter || target.includes("*")) {
@@ -118,12 +245,12 @@ export default class Set {
 					target,
 					attrs.filter,
 				);
-				await store.updateBodyByPattern(
-					runId,
-					target,
-					attrs.filter || null,
-					entry.body,
-				);
+				await store.set({
+					runId: runId,
+					path: target,
+					body: entry.body,
+					bodyFilter: attrs.filter === undefined ? null : attrs.filter,
+				});
 				await storePatternResult(
 					store,
 					runId,
@@ -135,42 +262,91 @@ export default class Set {
 					{ loopId },
 				);
 			} else {
-				// Direct scheme write
-				await store.upsert(runId, turn, target, entry.body, 200, {
-					fidelity: fidelityAttr || "full",
+				// Direct scheme write (known://, unknown://, etc.)
+				// Same result shape as file writes — diff against existing.
+				const existing = await store.getBody(runId, target);
+				const oldContent = existing === null ? "" : existing;
+				const newContent = entry.body;
+				const udiff = generatePatch(target, oldContent, newContent);
+				const merge = oldContent
+					? `<<<<<<< SEARCH\n${oldContent}\n=======\n${newContent}\n>>>>>>> REPLACE`
+					: `<<<<<<< SEARCH\n=======\n${newContent}\n>>>>>>> REPLACE`;
+				const beforeTokens = oldContent ? countTokens(oldContent) : 0;
+				const afterTokens = countTokens(newContent);
+
+				await store.set({
+					runId,
+					turn,
+					path: target,
+					body: newContent,
+					state: "resolved",
+					// Scheme writes default to promoted — the model wrote it, so
+					// it's material unless they explicitly demote/archive.
+					visibility: visibilityAttr ? visibilityAttr : "visible",
 					attributes: summaryText ? { summary: summaryText } : null,
 					loopId,
 				});
+				await store.set({
+					runId,
+					turn,
+					path: entry.resultPath,
+					body: newContent,
+					state: "resolved",
+					loopId,
+					attributes: {
+						path: target,
+						patch: udiff,
+						merge,
+						beforeTokens,
+						afterTokens,
+						summary: summaryText,
+					},
+				});
 			}
 		}
 
-		// Apply fidelity after all write operations
-		if (fidelityAttr && attrs.path) {
+		// Apply visibility after all write operations
+		if (visibilityAttr && attrs.path) {
 			const target = attrs.path;
-			const scheme = KnownStore.scheme(target);
+			const scheme = Entries.scheme(target);
 			if (scheme !== null) {
-				await store.setFidelity(runId, target, fidelityAttr);
+				await store.set({
+					runId: runId,
+					path: target,
+					visibility: visibilityAttr,
+				});
 			}
 			if (summaryText) {
-				await store.setAttributes(runId, target, { summary: summaryText });
+				await store.set({
+					runId: runId,
+					path: target,
+					attributes: { summary: summaryText },
+				});
 			}
 		}
 	}
 
 	full(entry) {
 		const attrs = entry.attributes;
-		const file = attrs.file || entry.path;
-		if (attrs.error) return `# set ${file}\n${attrs.error}`;
+		const target = attrs.path || entry.path;
+		if (attrs.error) return `# set ${target}\n${attrs.error}`;
 		const tokens =
 			attrs.beforeTokens != null
 				? ` ${attrs.beforeTokens}→${attrs.afterTokens} tokens`
 				: "";
-		if (!attrs.merge) return `# set ${file}${tokens}`;
-		return `# set ${file}${tokens}\n${attrs.merge}`;
+		if (!attrs.merge) return `# set ${target}${tokens}`;
+		return `# set ${target}${tokens}\n${attrs.merge}`;
 	}
 
 	summary(entry) {
-		return entry.attributes.merge || "";
+		if (!entry.body) return "";
+		// Preserve SEARCH/REPLACE merge blocks intact — truncating them
+		// drops the before/after the model needs to recognize its edit.
+		if (/<<<<<<< SEARCH[\s\S]*>>>>>>> REPLACE/.test(entry.body)) {
+			return entry.body;
+		}
+		const flat = entry.body.replace(/\s+/g, " ").trim();
+		return flat.length <= 80 ? flat : `${flat.slice(0, 77)}...`;
 	}
 
 	async #processEdit(rummy, entry, attrs) {
@@ -179,8 +355,14 @@ export default class Set {
 		const matches = await store.getEntriesByPattern(runId, target, attrs.body);
 
 		if (matches.length === 0) {
-			await store.upsert(runId, turn, entry.resultPath, "", 404, {
-				attributes: { file: target, error: `${target} not found in context` },
+			await store.set({
+				runId,
+				turn,
+				path: entry.resultPath,
+				body: "",
+				state: "failed",
+				outcome: "not_found",
+				attributes: { path: target, error: `${target} not found in context` },
 				loopId,
 			});
 			return;
@@ -188,37 +370,78 @@ export default class Set {
 
 		for (const match of matches) {
 			if (match.scheme === null) {
+				// Bare file path — apply the edit immediately against the
+				// match body so the log carries a concrete before/after
+				// merge. #materializeRevisions still runs at turn-end to
+				// consolidate the set:// proposal for client acceptance.
 				const canonicalPath = `set://${match.path}`;
 				const revision = Set.#buildRevision(attrs);
 				const existingAttrs = await rummy.getAttributes(canonicalPath);
-				const revisions = existingAttrs?.revisions || [];
+				const revisions = existingAttrs?.revisions
+					? existingAttrs.revisions
+					: [];
 				revisions.push(revision);
-				await store.upsert(runId, turn, canonicalPath, "", 200, {
-					attributes: { file: match.path, revisions },
+				await store.set({
+					runId,
+					turn,
+					path: canonicalPath,
+					body: "",
+					state: "resolved",
+					attributes: { path: match.path, revisions },
+					loopId,
+				});
+				const { patch, searchText, replaceText, warning, error } =
+					Set.#applyRevision(match.body, attrs);
+				const merge =
+					searchText != null
+						? `<<<<<<< SEARCH\n${searchText}\n=======\n${replaceText}\n>>>>>>> REPLACE`
+						: null;
+				const beforeTokens = match.tokens;
+				const afterTokens = patch ? countTokens(patch) : beforeTokens;
+				const logState = error ? "failed" : "resolved";
+				await store.set({
+					runId,
+					turn,
+					path: entry.resultPath,
+					body: merge ?? (patch || `edit to ${match.path}`),
+					state: logState,
+					outcome: error ? "conflict" : null,
+					attributes: {
+						path: match.path,
+						merge,
+						beforeTokens,
+						afterTokens,
+						warning,
+						error,
+					},
 					loopId,
 				});
-				if (KnownStore.normalizePath(entry.resultPath) !== canonicalPath) {
-					await store.remove(runId, entry.resultPath);
-				}
 				return;
 			}
 
 			const { patch, searchText, replaceText, warning, error } =
 				Set.#applyRevision(match.body, attrs);
 
-			const status = error ? 409 : 200;
-			const resultPath = `set://${match.path}`;
+			const state = error ? "failed" : "resolved";
+			const outcome = error ? "conflict" : null;
 			const udiff = patch ? generatePatch(match.path, match.body, patch) : null;
 			const merge =
 				searchText != null
 					? `<<<<<<< SEARCH\n${searchText}\n=======\n${replaceText}\n>>>>>>> REPLACE`
 					: null;
-			const beforeTokens = match.tokens || 0;
+			const beforeTokens = match.tokens;
 			const afterTokens = patch ? countTokens(patch) : beforeTokens;
 
-			await store.upsert(runId, turn, resultPath, match.body, status, {
+			// Log entry at log://turn_N/set/<target> records the action.
+			await store.set({
+				runId,
+				turn,
+				path: entry.resultPath,
+				body: patch ?? match.body,
+				state,
+				outcome,
 				attributes: {
-					file: match.path,
+					path: match.path,
 					patch: udiff,
 					merge,
 					beforeTokens,
@@ -229,8 +452,13 @@ export default class Set {
 				loopId,
 			});
 
-			if (status === 200 && patch) {
-				await store.upsert(runId, turn, match.path, patch, match.status, {
+			if (state === "resolved" && patch) {
+				await store.set({
+					runId,
+					turn,
+					path: match.path,
+					body: patch,
+					state: match.state,
 					loopId,
 				});
 			}
@@ -248,11 +476,11 @@ export default class Set {
 					: entry.attributes;
 			if (!attrs?.revisions?.length) continue;
 
-			const filePath = attrs.file;
-			const fileEntry = await store.getEntriesByPattern(runId, filePath);
-			if (fileEntry.length === 0) continue;
+			const entryPath = attrs.path;
+			const targetEntry = await store.getEntriesByPattern(runId, entryPath);
+			if (targetEntry.length === 0) continue;
 
-			const original = fileEntry[0].body;
+			const original = targetEntry[0].body;
 			let current = original;
 			const mergeBlocks = [];
 			let lastError = null;
@@ -274,18 +502,25 @@ export default class Set {
 				}
 			}
 
-			const state = lastError ? 409 : 202;
+			const state = lastError ? "failed" : "proposed";
+			const outcome = lastError ? "conflict" : null;
 			const udiff =
 				current !== original
-					? generatePatch(filePath, original, current)
+					? generatePatch(entryPath, original, current)
 					: null;
 			const merge = mergeBlocks.length > 0 ? mergeBlocks.join("\n") : null;
-			const beforeTokens = fileEntry[0].tokens || 0;
+			const beforeTokens = targetEntry[0].tokens;
 			const afterTokens = current ? countTokens(current) : beforeTokens;
 
-			await store.upsert(runId, turn, entry.path, original, state, {
+			await store.set({
+				runId,
+				turn,
+				path: entry.path,
+				body: current,
+				state,
+				outcome,
 				attributes: {
-					file: filePath,
+					path: entryPath,
 					patch: udiff,
 					merge,
 					beforeTokens,
@@ -298,9 +533,15 @@ export default class Set {
 		}
 	}
 
+	// `replace` attr is optional in search/replace form — absence means
+	// "delete the match"; normalize to empty string at this boundary.
+	static #resolveReplace(attrs) {
+		return attrs.replace === undefined ? "" : attrs.replace;
+	}
+
 	static #buildRevision(attrs) {
 		if (attrs.search != null) {
-			return { search: attrs.search, replace: attrs.replace ?? "" };
+			return { search: attrs.search, replace: Set.#resolveReplace(attrs) };
 		}
 		if (attrs.blocks?.length > 0) {
 			return { blocks: attrs.blocks };
@@ -310,7 +551,7 @@ export default class Set {
 
 	static #applyRevision(body, attrs) {
 		if (attrs.search != null) {
-			return Hedberg.replace(body, attrs.search, attrs.replace ?? "", {
+			return Hedberg.replace(body, attrs.search, Set.#resolveReplace(attrs), {
 				sed: attrs.sed,
 				flags: attrs.flags,
 			});

package/src/plugins/set/setDoc.js

@@ -1,36 +1,3 @@
-// Tool doc for <set>. Each entry: [text, rationale].
-// Text goes to the model. Rationale stays in source.
-// Changing ANY line requires reading ALL rationales first.
-const LINES = [
-  [
-    '## <set path="[path/to/file]">[content or edit]</set> - Create, edit, or update a file or entry',
-  ],
-  [
-    'Example: <set path="known://project/milestones" fidelity="summary" summary="milestone,deadline,2026"/>',
-    "Fidelity control first — most unique capability of set.",
-  ],
-  [
-    `Example: <set path="src/app.js">
-<<<<<<< SEARCH
-old text
-=======
-new text
->>>>>>> REPLACE
-</set>`,
-    "SEARCH/REPLACE block — primary edit pattern for existing files.",
-  ],
-  [
-    'Example: <set path="src/config.js">s/port = 3000/port = 8080/g;s/host = 127.0.0.1/host = localhost/g;</set>',
-    "Sed syntax: chained s/old/new/ patterns with semicolons.",
-  ],
-  [
-    'Example: <set path="example.md">Full file content here</set>',
-    "Create: body contents are entire file.",
-  ],
-  [
-    "* YOU MUST NOT use <sh/> or <env/> to list, create, read, or edit files. Use the Tool Commands.",
-    "Forces file operations through set/get.",
-  ],
-];
+import { loadDoc } from "../helpers.js";
 
-export default LINES.map(([text]) => text).join("\n");
+export default loadDoc(import.meta.url, "setDoc.md");

package/src/plugins/set/setDoc.md

@@ -0,0 +1,22 @@
+## <set path="[path/to/file]">[content or edit]</set> - Create, edit, or update a file or entry
+
+Example: <set path="known://project/milestones" visibility="summarized" summary="milestone,deadline,2026"/>
+<!-- Visibility control first — most unique capability of set. -->
+
+Example: <set path="src/app.js">
+<<<<<<< SEARCH
+old text
+=======
+new text
+>>>>>>> REPLACE
+</set>
+<!-- SEARCH/REPLACE block — primary edit pattern for existing files. -->
+
+Example: <set path="src/config.js">s/port = 3000/port = 8080/g;s/We're almost done/We're done./g;</set>
+<!-- Sed syntax: chained s/old/new/ patterns with semicolons. -->
+
+Example: <set path="example.md">Full file content here</set>
+<!-- Create: body contents are entire file. -->
+
+* YOU MUST NOT use <sh></sh> or <env></env> to list, create, read, or edit files — use <get></get> and <set></set>
+<!-- Reinforces at the decision point — model reading setDoc for file ops sees the prohibition here, not just buried in shDoc/envDoc which it may not be reading. -->

package/src/plugins/sh/README.md

@@ -1,16 +1,39 @@
-# sh
+# sh {#sh_plugin}
 
-Proposes shell command execution for client approval.
+Proposes shell command execution for client approval. Streaming
+producer: the actual stdout/stderr arrive as separate data entries
+after the proposal is accepted.
 
 ## Registration
 
 - **Tool**: `sh`
-- **Category**: `logging`
-- **Handler**: Upserts the entry at status 202 (proposed). The client must approve execution.
+- **Scheme**: `sh` — `category: "data"` (channels only; see below)
+- **Handler**: Upserts the proposal entry at status 202 (proposed). The
+  client must approve execution.
+
+## Two namespaces per invocation
+
+A single `<sh>` emission produces entries in two namespaces — one audit
+record, one data payload:
+
+- **Log entry**: `log://turn_N/sh/{slug}` — scheme=`log`, category=`logging`.
+  This is the proposal the client sees and resolves. On accept, body is
+  rewritten to `ran '{cmd}' (in progress). Output: {dataBase}_1, {dataBase}_2`
+  and finalized by `stream/completed` with exit code + duration. Renders
+  inside the `<log>` block as `<sh>`.
+- **Data channels**: `sh://turn_N/{slug}_1` (stdout), `sh://turn_N/{slug}_2`
+  (stderr) — scheme=`sh`, category=`data`. Created at status=102 on
+  proposal acceptance, grow via the `stream` RPC, transition to 200/500
+  via `stream/completed`. Render inside the `<context>` block as `<sh>`.
+
+The `sh` scheme exists **only** for the data channels. The proposal/log
+entry itself is in the unified `log://` namespace along with every
+other action record. See [scheme_category_split](#scheme_category_split).
 
 ## Projection
 
-Shows `sh {command}` followed by the entry body.
+- **Visible**: `# sh {command}\n{body}` (channel body is the captured stream).
+- **Summarized**: empty (the command + path are already shown via attrs).
 
 ## Behavior