@possumtech/rummy 0.4.0 → 2.0.0

package/src/plugins/sh/sh.js

@@ -1,33 +1,77 @@
+import { logPathToDataBase } from "../helpers.js";
 import docs from "./shDoc.js";
 
+const LOG_ACTION_RE = /^log:\/\/turn_\d+\/(\w+)\//;
+
 export default class Sh {
 	#core;
 
 	constructor(core) {
 		this.#core = core;
-		core.registerScheme();
+		// `sh` scheme holds the streamed stdout/stderr payload — that's
+		// data the model reads, not an audit record. The log entry at
+		// log://turn_N/sh/{slug} (scheme=log, category=logging) is the
+		// audit record; it lives in a separate namespace by design.
+		// See SPEC §streaming_entries and the scheme/category invariant.
+		core.registerScheme({ category: "data" });
 		core.on("handler", this.handler.bind(this));
-		core.on("full", this.full.bind(this));
-		core.on("summary", this.summary.bind(this));
+		core.on("visible", this.full.bind(this));
+		core.on("summarized", this.summary.bind(this));
 		core.filter("instructions.toolDocs", async (docsMap) => {
 			docsMap.sh = docs;
 			return docsMap;
 		});
+		core.on("proposal.accepted", this.#onAccepted.bind(this));
+	}
+
+	async #onAccepted(ctx) {
+		const m = LOG_ACTION_RE.exec(ctx.path);
+		if (m?.[1] !== "sh") return;
+		let command = "";
+		if (ctx.attrs?.command) command = ctx.attrs.command;
+		else if (ctx.attrs?.summary) command = ctx.attrs.summary;
+		const turn = (await ctx.db.get_run_by_id.get({ id: ctx.runId })).next_turn;
+		const dataBase = logPathToDataBase(ctx.path);
+		for (const ch of [1, 2]) {
+			await ctx.entries.set({
+				runId: ctx.runId,
+				turn,
+				path: `${dataBase}_${ch}`,
+				body: "",
+				state: "streaming",
+				visibility: "summarized",
+				attributes: { command, summary: command, channel: ch },
+			});
+		}
+		await ctx.entries.set({
+			runId: ctx.runId,
+			path: ctx.path,
+			state: "resolved",
+			body: `ran '${command}' (in progress). Output: ${dataBase}_1, ${dataBase}_2`,
+		});
 	}
 
 	async handler(entry, rummy) {
 		const { entries: store, sequence: turn, runId, loopId } = rummy;
-		await store.upsert(runId, turn, entry.resultPath, entry.body, 202, {
-			attributes: entry.attributes,
+		// Proposal at 202 with the command as summary and empty body — the
+		// body fills in on accept (log message about the action). Data
+		// entries with stdout/stderr are created on accept in resolve().
+		await store.set({
+			runId,
+			turn,
+			path: entry.resultPath,
+			body: "",
+			state: "proposed",
+			attributes: { ...entry.attributes, summary: entry.attributes.command },
 			loopId,
 		});
 	}
 
 	full(entry) {
-		return `# sh ${entry.attributes.command || ""}\n${entry.body}`;
+		return `# sh ${entry.attributes.command}\n${entry.body}`;
 	}
 
-	summary(entry) {
-		return entry.attributes.command || "";
+	summary() {
+		return "";
 	}
 }

package/src/plugins/sh/shDoc.js

@@ -1,24 +1,3 @@
-// Tool doc for <sh>. Each entry: [text, rationale].
-// Text goes to the model. Rationale stays in source.
-// Changing ANY line requires reading ALL rationales first.
-const LINES = [
-	["## <sh>[command]</sh> - Run a shell command with side effects"],
-	[
-		"Example: <sh>npm install express</sh>",
-		"Package install. Real side-effect command.",
-	],
-	[
-		"Example: <sh>npm test</sh>",
-		"Test execution. Another common side-effect action.",
-	],
-	[
-		"* YOU MUST NOT use <sh/> to read, create, or edit files — use <get/> and <set/>",
-		"Forces file operations through the entry system.",
-	],
-	[
-		"* YOU MUST use <env/> for commands without side effects",
-		"Reinforces the env/sh split. Read = env, mutate = sh.",
-	],
-];
+import { loadDoc } from "../helpers.js";
 
-export default LINES.map(([text]) => text).join("\n");
+export default loadDoc(import.meta.url, "shDoc.md");

package/src/plugins/sh/shDoc.md

@@ -0,0 +1,13 @@
+## <sh>[command]</sh> - Run a shell command with side effects
+
+Example: <sh>npm install express</sh>
+<!-- Package install. Real side-effect command. -->
+
+Example: <sh>npm test</sh>
+<!-- Test execution. Another common side-effect action. -->
+
+* YOU MUST NOT use <sh></sh> to read, create, or edit files — use <get></get> and <set></set>
+<!-- Forces file operations through the entry system. -->
+
+* YOU MUST use <env></env> for commands without side effects
+<!-- Reinforces the env/sh split. Read = env, mutate = sh. -->

package/src/plugins/skill/README.md

@@ -0,0 +1,23 @@
+# skill {#skill_plugin}
+
+Runtime skill management. A skill is a markdown file that gets
+attached to a run as a `skill://<name>` entry. Models see skills
+like any other entry.
+
+## Files
+
+- **skill.js** — RPC registration and skill file loading.
+
+## Registration
+
+- **Scheme**: `skill` (category: `data`)
+- **Projections**: visible → body; summarized → empty.
+
+## RPC Methods
+
+| Method | Params | Notes |
+|--------|--------|-------|
+| `skill/add` | `{ run, name }` | Load `${RUMMY_HOME}/skills/<name>.md` and attach it to the run as `skill://<name>`. |
+| `skill/remove` | `{ run, name }` | Remove the skill entry from the run. |
+| `getSkills` | `{ run }` | Skills active on a run. |
+| `listSkills` | — | Available skill files on disk. |

package/src/plugins/skill/skill.js

@@ -10,7 +10,8 @@ export default class Skill {
 			name: "skill",
 			category: "data",
 		});
-		core.hooks.tools.onView("skill", (entry) => entry.body);
+		core.hooks.tools.onView("skill", (entry) => entry.body, "visible");
+		core.hooks.tools.onView("skill", () => "", "summarized");
 
 		const r = core.hooks.rpc.registry;
 
@@ -24,7 +25,12 @@ export default class Skill {
 
 				const body = await loadFile("skills", params.name);
 				const store = ctx.projectAgent.entries;
-				await store.upsert(runRow.id, 0, `skill://${params.name}`, body, 200, {
+				await store.set({
+					runId: runRow.id,
+					turn: 0,
+					path: `skill://${params.name}`,
+					body,
+					state: "resolved",
 					attributes: {
 						name: params.name,
 						source: filePath("skills", params.name),
@@ -51,7 +57,7 @@ export default class Skill {
 				if (!runRow) throw new Error(`Run not found: ${params.run}`);
 
 				const store = ctx.projectAgent.entries;
-				await store.remove(runRow.id, `skill://${params.name}`);
+				await store.rm({ runId: runRow.id, path: `skill://${params.name}` });
 
 				return { status: "ok" };
 			},
@@ -111,23 +117,14 @@ function filePath(subfolder, name) {
 async function loadFile(subfolder, name) {
 	const path = filePath(subfolder, name);
 	if (!path) throw new Error("RUMMY_HOME not configured");
-	try {
-		return await fs.readFile(path, "utf8");
-	} catch (err) {
-		if (err.code === "ENOENT") throw new Error(`Not found: ${path}`);
-		throw err;
-	}
+	return fs.readFile(path, "utf8");
 }
 
 async function listAvailable(subfolder) {
 	const dir = configDir(subfolder);
 	if (!dir) return [];
-	try {
-		const files = await fs.readdir(dir);
-		return files
-			.filter((f) => f.endsWith(".md"))
-			.map((f) => ({ name: f.replace(".md", ""), path: join(dir, f) }));
-	} catch {
-		return [];
-	}
+	const files = await fs.readdir(dir);
+	return files
+		.filter((f) => f.endsWith(".md"))
+		.map((f) => ({ name: f.replace(".md", ""), path: join(dir, f) }));
 }

package/src/plugins/stream/README.md

@@ -0,0 +1,101 @@
+# stream {#stream_plugin}
+
+Generic streaming entry infrastructure. Provides RPC methods that any
+producer plugin (sh, env, future: search, fetch, watch) can use to
+populate data entries over time.
+
+## Namespace split
+
+A streaming action lives in **two namespaces** by design:
+
+- **Log entry** (audit record): `log://turn_N/{action}/{slug}` —
+  scheme=`log`, category=`logging`. Created by the producer's dispatch
+  handler (via `TurnExecutor` → `logPath`). This is the proposal the
+  client resolves. Renders inside `<log>`.
+- **Data channels** (payload): `{action}://turn_N/{slug}_1`,
+  `{action}://turn_N/{slug}_2`, ... — scheme=`{action}` (sh, env, ...),
+  category=`data`. Created at status=102 on proposal acceptance. Grow
+  via `stream`; terminal via `stream/completed` / `stream/aborted` /
+  `stream/cancel`. Render inside `<context>`.
+
+The stream RPC `path` param is always the **log-entry path** (that's
+what clients receive on `run/proposal`). The server derives the data
+base path internally via `logPathToDataBase`. See
+[scheme_category_split](#scheme_category_split).
+
+## RPC Methods
+
+### `stream { run, path, channel, chunk }`
+
+Append `chunk` to the data channel entry at `{dataBase}_{channel}`,
+where `dataBase` is derived from the log path. Entry must exist
+(created by the producer plugin on proposal acceptance, at status=102).
+
+Unix FD convention for the channel number: 1=stdout, 2=stderr, higher
+numbers for additional producer channels.
+
+### `stream/completed { run, path, exit_code?, duration? }`
+
+Transition all `{dataBase}_*` data channels to terminal status:
+- `exit_code=0` (or omitted) → status=200
+- `exit_code≠0` → status=500
+
+Rewrite the log entry at `path` with a summary: command, exit code,
+duration, and channel sizes.
+
+### `stream/aborted { run, path, reason?, duration? }`
+
+Client-initiated cancellation. Transition all `{dataBase}_*` data
+channels to status **499 (Client Closed Request)** — the de-facto HTTP
+status for a request terminated by the client. Rewrite the log entry
+body to note the abort (with optional `reason` and `duration`).
+
+Client contract: kill the underlying process first, then call
+`stream/aborted`. Body of each data channel is preserved at whatever
+content was streamed before the kill.
+
+### `stream/cancel { run, path, reason? }`
+
+Server-initiated cancellation. Any client (or internal server code) can
+cancel a streaming producer — the server transitions channels to **499**
+immediately and pushes a `stream/cancelled` notification to all connected
+clients so they can kill their local processes.
+
+Also serves as **stale 102 cleanup**: if the originating client died
+mid-stream (`stream/completed` never arrived), any client can call
+`stream/cancel` to mark orphaned entries terminal.
+
+## Producer Plugin Contract
+
+A streaming producer plugin:
+
+1. On model dispatch, writes the **proposal/log entry** at
+   `log://turn_N/{action}/{slug}` at status=202 (this is automatic —
+   `TurnExecutor` builds the path via `logPath`; the producer's
+   `handler` just persists it).
+2. On `proposal.accepted`, derives the data base
+   (`logPathToDataBase(ctx.path)`) and creates **data entries** at
+   `{dataBase}_1`, `{dataBase}_2`, etc. at status=102, category=data,
+   visibility=summarized, empty body. Then rewrites the log entry body
+   to reference the channel paths.
+3. Client or external producer calls the `stream` RPC with chunks as
+   they arrive.
+4. When the producer is done, the client/producer calls
+   `stream/completed`.
+
+Current producers:
+- **sh** — shell commands with side effects (stdout ch1, stderr ch2)
+- **env** — safe shell (stdout ch1, stderr ch2)
+
+Future producers that could adopt this pattern:
+- **search** — web search results streaming in (primary ch1, warnings ch2)
+- **fetch** — large page fetch (body ch1, redirects/headers ch2)
+- **tail** — log file following (lines ch1)
+- **watch** — file system events (events ch1)
+
+## Not a Model-Facing Tool
+
+No scheme registration, no tooldoc, no dispatch handler. The model
+interacts with streamed output via `<get>` on the data entries; the
+stream plugin is purely RPC infrastructure that clients and producer
+plugins use.

package/src/plugins/stream/stream.js

@@ -0,0 +1,290 @@
+import { logPathToDataBase } from "../helpers.js";
+
+/**
+ * Stream plugin — generic streaming entry infrastructure.
+ *
+ * Receives chunks from the client (or any producer) and appends them to
+ * existing data entries. Producers (sh/env handlers) create the data
+ * entries at status=102 on proposal acceptance; this plugin handles the
+ * subsequent append + terminal-status transition via two RPC methods.
+ *
+ * RPC `path` param is the **log-entry path** (log://turn_N/{action}/{slug}
+ * — that's what the client sees on `run/proposal`). Channels live under
+ * the producer scheme ({action}://turn_N/{slug}_N) for a clean
+ * data-vs-logging namespace split; this plugin derives the data base from
+ * the log path on every RPC call.
+ *
+ * Not a model-facing tool. No scheme, no tooldoc, no dispatch handler.
+ * Pure RPC plumbing that any streaming-producer plugin can leverage.
+ */
+export default class Stream {
+	#core;
+
+	constructor(core) {
+		this.#core = core;
+		const hooks = core.hooks;
+		const r = hooks.rpc.registry;
+
+		// stream: append a chunk to a streaming entry.
+		// Entry path is constructed as `${path}_${channel}` per the Unix FD
+		// convention (1=stdout, 2=stderr, higher=other producer channels).
+		r.register("stream", {
+			handler: async (params, ctx) => {
+				if (!params.run) throw new Error("run is required");
+				if (!params.path) throw new Error("path is required");
+				if (params.channel == null)
+					throw new Error("channel is required (numeric)");
+				if (params.chunk == null) throw new Error("chunk is required");
+
+				const runRow = await ctx.db.get_run_by_alias.get({
+					alias: params.run,
+				});
+				if (!runRow) throw new Error(`run not found: ${params.run}`);
+
+				const dataBase = logPathToDataBase(params.path);
+				if (!dataBase) {
+					throw new Error(
+						`path must be a log entry (log://turn_N/...); got: ${params.path}`,
+					);
+				}
+				const entryPath = `${dataBase}_${params.channel}`;
+				await ctx.projectAgent.entries.set({
+					runId: runRow.id,
+					path: entryPath,
+					body: params.chunk,
+					append: true,
+				});
+				return { status: "ok" };
+			},
+			description:
+				"Append a chunk to a streaming entry channel. Used by clients and producers to grow a 102 entry's body.",
+			params: {
+				run: "string — run alias",
+				path: "string — log-entry path (log://turn_N/{action}/{slug}); server derives the data channel path",
+				channel: "number — channel index (Unix FD: 1=stdout, 2=stderr)",
+				chunk: "string — content to append to the entry body",
+			},
+			requiresInit: true,
+		});
+
+		// stream/completed: transition all data channels for this producer
+		// to their terminal status and finalize the log entry body.
+		r.register("stream/completed", {
+			handler: async (params, ctx) => {
+				if (!params.run) throw new Error("run is required");
+				if (!params.path) throw new Error("path is required");
+
+				const runRow = await ctx.db.get_run_by_alias.get({
+					alias: params.run,
+				});
+				if (!runRow) throw new Error(`run not found: ${params.run}`);
+				const runId = runRow.id;
+
+				const { exit_code: exitCode = 0, duration = null } = params;
+				const terminalState = exitCode === 0 ? "resolved" : "failed";
+				const terminalOutcome = exitCode === 0 ? null : `exit:${exitCode}`;
+
+				const dataBase = logPathToDataBase(params.path);
+				if (!dataBase) {
+					throw new Error(
+						`path must be a log entry (log://turn_N/...); got: ${params.path}`,
+					);
+				}
+				// Find all `{dataBase}_*` data entries (channels 1, 2, ...).
+				const store = ctx.projectAgent.entries;
+				const channels = await store.getEntriesByPattern(
+					runId,
+					`${dataBase}_*`,
+					null,
+				);
+				for (const ch of channels) {
+					await store.set({
+						runId,
+						path: ch.path,
+						state: terminalState,
+						body: ch.body,
+						outcome: terminalOutcome,
+					});
+				}
+
+				// Update the log entry body with final stats. Keep it terse —
+				// one line summarizing exit code, duration, and channel sizes.
+				const logEntry = await store.getAttributes(runId, params.path);
+				let command = "";
+				if (logEntry?.command) command = logEntry.command;
+				else if (logEntry?.summary) command = logEntry.summary;
+				const channelSummary = channels
+					.map((c) => {
+						const size = c.body ? `${c.tokens} tokens` : "empty";
+						return `${c.path} (${size})`;
+					})
+					.join(", ");
+				const dur = duration ? ` (${duration})` : "";
+				const exitLabel = exitCode === 0 ? "exit=0" : `exit=${exitCode}`;
+				const body = `ran '${command}', ${exitLabel}${dur}. Output: ${channelSummary}`;
+				await store.set({ runId, path: params.path, state: "resolved", body });
+
+				return { ok: true, channels: channels.length };
+			},
+			description:
+				"Finalize a streaming producer. Transitions all `{path}_*` data channels to terminal status (200 on exit_code=0, 500 otherwise) and rewrites the log entry body with exit code, duration, and channel sizes.",
+			params: {
+				run: "string — run alias",
+				path: "string — log-entry path (log://turn_N/{action}/{slug}); server derives the data channel path",
+				exit_code:
+					"number? — exit code (0=success→200, non-zero=failure→500). Defaults to 0 for non-process producers.",
+				duration: "string? — human-readable duration for the log entry",
+			},
+			requiresInit: true,
+		});
+
+		// stream/aborted: client-initiated cancellation. Transitions all data
+		// channels to status 499 (Client Closed Request — the de-facto HTTP
+		// status for client-terminated requests) and rewrites the log entry
+		// body to note the abort. Shape mirrors stream/completed for client
+		// symmetry: same run/path addressing, same channel sweep.
+		r.register("stream/aborted", {
+			handler: async (params, ctx) => {
+				if (!params.run) throw new Error("run is required");
+				if (!params.path) throw new Error("path is required");
+
+				const runRow = await ctx.db.get_run_by_alias.get({
+					alias: params.run,
+				});
+				if (!runRow) throw new Error(`run not found: ${params.run}`);
+				const runId = runRow.id;
+
+				const { duration = null, reason = null } = params;
+
+				const dataBase = logPathToDataBase(params.path);
+				if (!dataBase) {
+					throw new Error(
+						`path must be a log entry (log://turn_N/...); got: ${params.path}`,
+					);
+				}
+				const store = ctx.projectAgent.entries;
+				const channels = await store.getEntriesByPattern(
+					runId,
+					`${dataBase}_*`,
+					null,
+				);
+				for (const ch of channels) {
+					await store.set({
+						runId,
+						path: ch.path,
+						state: "cancelled",
+						body: ch.body,
+						outcome: reason ? reason : "aborted",
+					});
+				}
+
+				const logEntry = await store.getAttributes(runId, params.path);
+				let command = "";
+				if (logEntry?.command) command = logEntry.command;
+				else if (logEntry?.summary) command = logEntry.summary;
+				const channelSummary = channels
+					.map((c) => {
+						const size = c.body ? `${c.tokens} tokens` : "empty";
+						return `${c.path} (${size})`;
+					})
+					.join(", ");
+				const qualifiers = [];
+				if (reason) qualifiers.push(reason);
+				if (duration) qualifiers.push(duration);
+				const qualifier = qualifiers.length
+					? ` (${qualifiers.join(", ")})`
+					: "";
+				const body = `aborted '${command}'${qualifier}. Output: ${channelSummary}`;
+				await store.set({ runId, path: params.path, state: "resolved", body });
+
+				return { status: "ok", channels: channels.length };
+			},
+			description:
+				"Abort a streaming producer. Transitions all `{path}_*` data channels to status 499 (Client Closed Request) and rewrites the log entry body to note the abort.",
+			params: {
+				run: "string — run alias",
+				path: "string — log-entry path (log://turn_N/{action}/{slug}); server derives the data channel path",
+				reason:
+					"string? — human-readable abort reason (e.g. 'user cancelled', 'timeout')",
+				duration: "string? — human-readable duration at abort time",
+			},
+			requiresInit: true,
+		});
+
+		// stream/cancel: server-initiated cancellation. Any client (or
+		// internal server code) can cancel a streaming producer — the server
+		// transitions channels to 499 immediately and pushes a
+		// stream/cancelled notification so connected clients can kill their
+		// local processes. Also serves as stale 102 cleanup: if the client
+		// died mid-stream, call stream/cancel to mark orphaned entries terminal.
+		r.register("stream/cancel", {
+			handler: async (params, ctx) => {
+				if (!params.run) throw new Error("run is required");
+				if (!params.path) throw new Error("path is required");
+
+				const runRow = await ctx.db.get_run_by_alias.get({
+					alias: params.run,
+				});
+				if (!runRow) throw new Error(`run not found: ${params.run}`);
+				const runId = runRow.id;
+
+				const { reason = null } = params;
+
+				const dataBase = logPathToDataBase(params.path);
+				if (!dataBase) {
+					throw new Error(
+						`path must be a log entry (log://turn_N/...); got: ${params.path}`,
+					);
+				}
+				const store = ctx.projectAgent.entries;
+				const channels = await store.getEntriesByPattern(
+					runId,
+					`${dataBase}_*`,
+					null,
+				);
+				for (const ch of channels) {
+					await store.set({
+						runId,
+						path: ch.path,
+						state: "cancelled",
+						body: ch.body,
+						outcome: reason ? reason : "cancelled",
+					});
+				}
+
+				const logEntry = await store.getAttributes(runId, params.path);
+				let command = "";
+				if (logEntry?.command) command = logEntry.command;
+				else if (logEntry?.summary) command = logEntry.summary;
+				const channelSummary = channels
+					.map((c) => {
+						const size = c.body ? `${c.tokens} tokens` : "empty";
+						return `${c.path} (${size})`;
+					})
+					.join(", ");
+				const qualifier = reason ? ` (${reason})` : "";
+				const body = `cancelled '${command}'${qualifier}. Output: ${channelSummary}`;
+				await store.set({ runId, path: params.path, state: "resolved", body });
+
+				// Notify connected clients so they can kill local processes.
+				hooks.stream.cancelled.emit({
+					projectId: ctx.projectId,
+					run: params.run,
+					path: params.path,
+					reason,
+				});
+
+				return { ok: true, channels: channels.length };
+			},
+			description:
+				"Server-initiated cancellation. Transitions all `{path}_*` data channels to status 499 and pushes a stream/cancelled notification to connected clients. Also used for stale 102 cleanup when the originating client is gone.",
+			params: {
+				run: "string — run alias",
+				path: "string — log-entry path (log://turn_N/{action}/{slug}); server derives the data channel path",
+				reason:
+					"string? — cancellation reason (e.g. 'budget exceeded', 'stale cleanup', 'user cancelled from another client')",
+			},
+			requiresInit: true,
+		});
+	}
+}

package/src/plugins/telemetry/README.md

@@ -1,4 +1,4 @@
-# telemetry
+# telemetry {#telemetry_plugin}
 
 Console logging for RPC lifecycle and turn events.