@possumtech/rummy 0.2.1

package/src/plugins/telemetry/rpc_log.sql

@@ -0,0 +1,28 @@
+-- PREP: log_rpc_call
+INSERT INTO rpc_log (project_id, method, rpc_id, params)
+VALUES (:project_id, :method, :rpc_id, :params)
+RETURNING id;
+
+-- PREP: log_rpc_result
+UPDATE rpc_log
+SET result = :result
+WHERE id = :id;
+
+-- PREP: log_rpc_error
+UPDATE rpc_log
+SET error = :error
+WHERE id = :id;
+
+-- PREP: get_rpc_log
+SELECT id, project_id, method, rpc_id, params, result, error, created_at
+FROM rpc_log
+WHERE project_id = :project_id
+ORDER BY id DESC
+LIMIT :limit;
+
+-- PREP: get_rpc_log_by_method
+SELECT id, project_id, method, rpc_id, params, result, error, created_at
+FROM rpc_log
+WHERE project_id = :project_id AND method = :method
+ORDER BY id DESC
+LIMIT :limit;

package/src/plugins/telemetry/telemetry.js

@@ -0,0 +1,186 @@
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+export default class Telemetry {
+	#core;
+	#starts = new Map();
+	#lastRunPath = null;
+	#turnLog = [];
+
+	constructor(core) {
+		this.#core = core;
+
+		const home = process.env.RUMMY_HOME;
+		if (home) this.#lastRunPath = join(home, "last_run.txt");
+
+		core.on("rpc.started", this.#onRpcStarted.bind(this));
+		core.on("rpc.completed", this.#onRpcCompleted.bind(this));
+		core.on("rpc.error", this.#onRpcError.bind(this));
+		core.on("run.step.completed", this.#onStepCompleted.bind(this));
+		core.on("turn.response", this.#onTurnResponse.bind(this));
+		core.filter("llm.messages", this.#logMessages.bind(this), 999);
+		core.filter("llm.response", this.#logResponse.bind(this), 999);
+	}
+
+	async #onRpcStarted({ method, id, params }) {
+		this.#starts.set(id, Date.now());
+		const summary =
+			method === "ask" || method === "act"
+				? `prompt="${(params?.prompt || "").slice(0, 60)}"`
+				: method === "run/abort"
+					? `run=${params?.run}`
+					: method === "run/resolve"
+						? `run=${params?.run} action=${params?.resolution?.action}`
+						: "";
+		console.log(`[RPC] → ${method}(${id})${summary ? ` ${summary}` : ""}`);
+
+		if (method === "ask" || method === "act") {
+			this.#turnLog = [];
+		}
+	}
+
+	async #onRpcCompleted({ method, id, result }) {
+		const elapsed = this.#starts.has(id)
+			? `${((Date.now() - this.#starts.get(id)) / 1000).toFixed(1)}s`
+			: "";
+		this.#starts.delete(id);
+		const summary = result?.run
+			? `run=${result.run} status=${result.status || "ok"}`
+			: result?.status
+				? `status=${result.status}`
+				: "";
+		console.log(
+			`[RPC] ← ${method}(${id}) ${elapsed}${summary ? ` ${summary}` : ""}`,
+		);
+	}
+
+	async #onRpcError({ id, error }) {
+		const elapsed = this.#starts.has(id)
+			? `${((Date.now() - this.#starts.get(id)) / 1000).toFixed(1)}s`
+			: "";
+		this.#starts.delete(id);
+		console.error(`[RPC] ✗ (${id}) ${elapsed} ${error?.message || error}`);
+	}
+
+	async #onStepCompleted(payload) {
+		if (process.env.RUMMY_DEBUG !== "true") return;
+		console.log(
+			`[DEBUG] Turn ${payload.turn} completed for run ${payload.run}`,
+		);
+	}
+
+	async #onTurnResponse({
+		rummy,
+		turn,
+		result,
+		responseMessage,
+		content,
+		commands,
+		unparsed,
+		systemMsg,
+		userMsg,
+	}) {
+		const { entries: store, runId } = rummy;
+
+		// assistant://N — the model's raw response
+		await store.upsert(runId, turn, `assistant://${turn}`, content, "info");
+
+		// system://N, user://N — assembled messages as audit
+		if (systemMsg) {
+			await store.upsert(runId, turn, `system://${turn}`, systemMsg, "info");
+		}
+		if (userMsg) {
+			await store.upsert(runId, turn, `user://${turn}`, userMsg, "info");
+		}
+
+		// model://N — raw API response diagnostics
+		await store.upsert(
+			runId,
+			turn,
+			`model://${turn}`,
+			JSON.stringify({
+				keys: responseMessage ? Object.keys(responseMessage) : [],
+				reasoning_content: responseMessage?.reasoning_content || null,
+				content: content.slice(0, 4096),
+				usage: result.usage || null,
+				model: result.model || null,
+			}),
+			"info",
+		);
+
+		// reasoning://N
+		if (responseMessage?.reasoning_content) {
+			await store.upsert(
+				runId,
+				turn,
+				`reasoning://${turn}`,
+				responseMessage.reasoning_content,
+				"info",
+			);
+		}
+
+		// content://N — unparsed text
+		if (unparsed) {
+			await store.upsert(runId, turn, `content://${turn}`, unparsed, "info");
+		}
+
+		// Commit usage stats
+		const usage = result.usage || {};
+		const cachedTokens =
+			usage.cached_tokens ||
+			usage.prompt_tokens_details?.cached_tokens ||
+			usage.input_tokens_details?.cached_tokens ||
+			usage.cache_read_input_tokens ||
+			0;
+		const reasoningTokens =
+			usage.reasoning_tokens ||
+			usage.completion_tokens_details?.reasoning_tokens ||
+			usage.output_tokens_details?.reasoning_tokens ||
+			0;
+		await rummy.db.update_turn_stats.run({
+			id: rummy.turnId,
+			prompt_tokens: usage.prompt_tokens ?? 0,
+			cached_tokens: cachedTokens ?? 0,
+			completion_tokens: usage.completion_tokens ?? 0,
+			reasoning_tokens: reasoningTokens ?? 0,
+			total_tokens: usage.total_tokens ?? 0,
+			cost: usage.cost ?? 0,
+		});
+	}
+
+	async #logMessages(messages, context) {
+		this.#turnLog.push(
+			`\n${"=".repeat(60)}\nTURN — model=${context.model} run=${context.runId}\n${"=".repeat(60)}`,
+		);
+		for (const msg of messages) {
+			const label = msg.role.toUpperCase();
+			const body =
+				typeof msg.content === "string"
+					? msg.content
+					: JSON.stringify(msg.content);
+			this.#turnLog.push(`\n--- ${label} ---\n${body}`);
+		}
+		return messages;
+	}
+
+	async #logResponse(response) {
+		const msg = response.choices?.[0]?.message;
+		this.#turnLog.push(`\n--- ASSISTANT ---\n${msg?.content || "(empty)"}`);
+		if (msg?.reasoning_content) {
+			this.#turnLog.push(`\n--- REASONING ---\n${msg.reasoning_content}`);
+		}
+		const usage = response.usage || {};
+		this.#turnLog.push(`\n--- USAGE ---\n${JSON.stringify(usage)}`);
+		this.#flush();
+		return response;
+	}
+
+	#flush() {
+		if (!this.#lastRunPath || this.#turnLog.length === 0) return;
+		try {
+			writeFileSync(this.#lastRunPath, `${this.#turnLog.join("\n")}\n`);
+		} catch {
+			// RUMMY_HOME may not exist yet
+		}
+	}
+}

package/src/plugins/unknown/README.md

@@ -0,0 +1,23 @@
+# unknown
+
+The Rumsfeld mechanism. The model registers what it doesn't know before acting.
+
+`<unknown>which database adapter is configured</unknown>`
+
+## Registration
+
+- **Tool**: `unknown`
+- **Modes**: ask, act
+- **Category**: structural
+- **Handler**: None — recorded by TurnExecutor, deduplicated against existing unknowns.
+
+## Projection
+
+`# unknown\n{body}`
+
+## Behavior
+
+Unknowns are sticky — they persist across turns until the model explicitly
+stores or removes them. The model investigates unknowns using `<get>`,
+`<env>`, or `<ask_user>`, then removes resolved ones with `<rm>`.
+Server deduplicates on insert.

package/src/plugins/unknown/docs.md

@@ -0,0 +1,5 @@
+## <unknown>[what you need to learn]</unknown> - Track open questions
+Example: <unknown>contents of answer.txt</unknown>
+Example: <unknown>which database adapter is configured</unknown>
+* Use get, env, or ask_user to investigate unknowns
+* When irrelevant or resolved, use <rm/> to remove from context.

package/src/plugins/unknown/unknown.js

@@ -0,0 +1,31 @@
+import { readFileSync } from "node:fs";
+
+export default class Unknown {
+	#core;
+
+	constructor(core) {
+		this.#core = core;
+		core.registerScheme({
+			validStates: ["full", "stored"],
+			category: "knowledge",
+		});
+		core.on("full", this.full.bind(this));
+		core.filter("assembly.system", this.assembleUnknowns.bind(this), 300);
+		const docs = readFileSync(new URL("./docs.md", import.meta.url), "utf8");
+		core.filter("instructions.toolDocs", async (content) =>
+			content ? `${content}\n\n${docs}` : docs,
+		);
+	}
+
+	full(entry) {
+		return `# unknown\n${entry.body}`;
+	}
+
+	async assembleUnknowns(content, ctx) {
+		const entries = ctx.rows.filter((r) => r.category === "unknown");
+		if (entries.length === 0) return content;
+
+		const lines = entries.map((u) => `<unknown>${u.body}</unknown>`);
+		return `${content}\n\n<unknowns>\n${lines.join("\n")}\n</unknowns>`;
+	}
+}

package/src/plugins/update/README.md

@@ -0,0 +1,18 @@
+# update
+
+Structural tool for model-generated progress updates.
+
+## Registration
+
+- **Tool**: `update`
+- **Modes**: ask, act
+- **Category**: structural
+- **Handler**: None — projection only.
+
+## Projection
+
+Shows `update` followed by the entry body.
+
+## Behavior
+
+No handler logic. Allows the model to emit progress/status entries that appear in context via projection.

package/src/plugins/update/docs.md

@@ -0,0 +1,4 @@
+## <update>[Brief update]</update>
+* Describe the current state
+* DO NOT use if done
+* Keep brief (<= 80 characters)

package/src/plugins/update/update.js

@@ -0,0 +1,24 @@
+import { readFileSync } from "node:fs";
+
+export default class Update {
+	#core;
+
+	constructor(core) {
+		this.#core = core;
+		core.registerScheme({ validStates: ["info"], category: "structural" });
+		core.on("full", this.full.bind(this));
+		core.on("summary", this.summary.bind(this));
+		const docs = readFileSync(new URL("./docs.md", import.meta.url), "utf8");
+		core.filter("instructions.toolDocs", async (content) =>
+			content ? `${content}\n\n${docs}` : docs,
+		);
+	}
+
+	full(entry) {
+		return `# update\n${entry.body}`;
+	}
+
+	summary(entry) {
+		return this.full(entry);
+	}
+}

package/src/server/ClientConnection.js

@@ -0,0 +1,228 @@
+import msg from "../agent/messages.js";
+import ProjectAgent from "../agent/ProjectAgent.js";
+
+export default class ClientConnection {
+	#ws;
+	#db;
+	#projectAgent;
+	#hooks;
+	#rpcRegistry;
+	#rpcLogPending = new Map();
+	#context = {
+		projectId: null,
+		projectRoot: null,
+	};
+
+	constructor(ws, db, hooks) {
+		this.#ws = ws;
+		this.#db = db;
+		this.#hooks = hooks;
+		this.#rpcRegistry = hooks.rpc.registry;
+		this.#projectAgent = new ProjectAgent(db, hooks);
+
+		this.#ws.on("message", (data) => this.#handleMessage(data));
+
+		this.#setupNotifications();
+	}
+
+	#setupNotifications() {
+		this.#hooks.run.progress.on((payload) => {
+			if (payload.projectId === this.#context.projectId) {
+				this.#sendNotification("run/progress", {
+					run: payload.run,
+					turn: payload.turn,
+					status: payload.status,
+				});
+			}
+		});
+
+		this.#hooks.ui.render.on((payload) => {
+			if (payload.projectId === this.#context.projectId) {
+				this.#sendNotification("ui/render", {
+					text: payload.text,
+					append: payload.append,
+				});
+			}
+		});
+
+		this.#hooks.ui.notify.on((payload) => {
+			if (payload.projectId === this.#context.projectId) {
+				this.#sendNotification("ui/notify", {
+					text: payload.text,
+					level: payload.level,
+				});
+			}
+		});
+
+		this.#hooks.run.state.on((payload) => {
+			if (payload.projectId === this.#context.projectId) {
+				this.#sendNotification("run/state", {
+					run: payload.run,
+					turn: payload.turn,
+					status: payload.status,
+					summary: payload.summary,
+					history: payload.history,
+					unknowns: payload.unknowns,
+					proposed: payload.proposed,
+					telemetry: payload.telemetry,
+				});
+			}
+		});
+	}
+
+	#buildHandlerContext() {
+		return {
+			projectAgent: this.#projectAgent,
+			db: this.#db,
+			rpcRegistry: this.#rpcRegistry,
+			projectId: this.#context.projectId,
+			projectRoot: this.#context.projectRoot,
+			setContext: (projectId, projectRoot) => {
+				this.#context.projectId = projectId;
+				this.#context.projectRoot = projectRoot;
+			},
+		};
+	}
+
+	async handleMessageForTest(data) {
+		return this.#handleMessage(data);
+	}
+
+	async #handleMessage(data) {
+		let id = null;
+		const debug = process.env.RUMMY_DEBUG === "true";
+		try {
+			const rawMessage = await this.#hooks.socket.message.raw.filter(data);
+			if (debug) console.log(`[SOCKET] IN: ${rawMessage.toString()}`);
+
+			const message = JSON.parse(rawMessage.toString());
+
+			const filteredRequest = await this.#hooks.rpc.request.filter(message);
+			const { method, params, id: msgId } = filteredRequest;
+			id = msgId;
+
+			await this.#hooks.rpc.started.emit({
+				method,
+				params,
+				id,
+				projectId: this.#context.projectId,
+			});
+
+			try {
+				const logRow = await this.#db.log_rpc_call.get({
+					project_id: this.#context.projectId || null,
+					method,
+					rpc_id: id,
+					params: params ? JSON.stringify(params) : null,
+				});
+				if (logRow) this.#rpcLogPending.set(id, logRow.id);
+			} catch {}
+
+			const resolvedMethod = method === "rpc/discover" ? "discover" : method;
+			const registration = this.#rpcRegistry.get(resolvedMethod);
+			if (!registration)
+				throw new Error(msg("error.method_not_found", { method }));
+
+			if (registration.requiresInit && !this.#context.projectId) {
+				throw new Error(msg("error.not_initialized"));
+			}
+
+			let result;
+			if (registration.longRunning) {
+				result = await registration.handler(
+					params || {},
+					this.#buildHandlerContext(),
+				);
+			} else {
+				const timeout = Number(process.env.RUMMY_RPC_TIMEOUT) || 10_000;
+				result = await Promise.race([
+					registration.handler(params || {}, this.#buildHandlerContext()),
+					new Promise((_, reject) =>
+						setTimeout(
+							() =>
+								reject(
+									new Error(
+										msg("error.rpc_timeout", {
+											method: resolvedMethod,
+											timeout,
+										}),
+									),
+								),
+							timeout,
+						),
+					),
+				]);
+			}
+
+			const finalResult = await this.#hooks.rpc.response.result.filter(result, {
+				method,
+				id,
+			});
+
+			this.#send({
+				jsonrpc: "2.0",
+				result: finalResult,
+				id,
+			});
+
+			await this.#hooks.rpc.completed.emit({
+				method,
+				id,
+				result: finalResult,
+			});
+
+			const logId = this.#rpcLogPending.get(id);
+			if (logId) {
+				this.#rpcLogPending.delete(id);
+				try {
+					await this.#db.log_rpc_result.run({
+						id: logId,
+						result: finalResult
+							? JSON.stringify(finalResult).slice(0, 4096)
+							: null,
+					});
+				} catch {}
+			}
+		} catch (error) {
+			if (debug) {
+				console.error(`[SOCKET] ERR: ${error.message}`);
+				console.error(`[DEBUG] Stack: ${error.stack}`);
+			}
+			this.#send({
+				jsonrpc: "2.0",
+				error: { code: -32603, message: error.message },
+				id: id || null,
+			});
+			await this.#hooks.rpc.error.emit({ id, error });
+
+			const errLogId = this.#rpcLogPending.get(id);
+			if (errLogId) {
+				this.#rpcLogPending.delete(id);
+				try {
+					await this.#db.log_rpc_error.run({
+						id: errLogId,
+						error: error.message,
+					});
+				} catch {}
+			}
+		}
+	}
+
+	#send(payload) {
+		const debug = process.env.RUMMY_DEBUG === "true";
+		if (debug) {
+			console.log(`[SOCKET] OUT: ${JSON.stringify(payload, null, 2)}`);
+		}
+		if (this.#ws.readyState === 1) {
+			this.#ws.send(JSON.stringify(payload));
+		}
+	}
+
+	#sendNotification(method, params) {
+		this.#send({
+			jsonrpc: "2.0",
+			method,
+			params,
+		});
+	}
+}

package/src/server/RpcRegistry.js

@@ -0,0 +1,52 @@
+export default class RpcRegistry {
+	#methods = new Map();
+	#notifications = new Map();
+
+	register(
+		name,
+		{
+			handler,
+			description = "",
+			params = {},
+			requiresInit = false,
+			longRunning = false,
+		},
+	) {
+		if (this.#methods.has(name))
+			throw new Error(`RPC method '${name}' already registered.`);
+		this.#methods.set(
+			name,
+			Object.freeze({
+				handler,
+				description,
+				params,
+				requiresInit,
+				longRunning,
+			}),
+		);
+	}
+
+	registerNotification(name, description = "") {
+		this.#notifications.set(name, Object.freeze({ description }));
+	}
+
+	get(name) {
+		return this.#methods.get(name);
+	}
+
+	has(name) {
+		return this.#methods.has(name);
+	}
+
+	discover() {
+		const methods = {};
+		for (const [name, def] of this.#methods) {
+			methods[name] = { description: def.description, params: def.params };
+		}
+		const notifications = {};
+		for (const [name, def] of this.#notifications) {
+			notifications[name] = { description: def.description };
+		}
+		return { methods, notifications };
+	}
+}

package/src/server/SocketServer.js

@@ -0,0 +1,43 @@
+import { WebSocketServer } from "ws";
+import ClientConnection from "./ClientConnection.js";
+
+export default class SocketServer {
+	#db;
+	#wss;
+	#hooks;
+
+	constructor(db, options) {
+		this.#db = db;
+		this.#hooks = options.hooks;
+		this.#wss = new WebSocketServer(options);
+
+		this.#wss.on("connection", (ws, req) => {
+			if (process.env.RUMMY_DEBUG === "true") {
+				console.log(`[SOCKET] New connection from ${req.socket.remoteAddress}`);
+			}
+			new ClientConnection(ws, this.#db, this.#hooks);
+		});
+
+		this.#wss.on("error", (_err) => {
+			// Proxy to registry or handle locally
+		});
+	}
+
+	address() {
+		return this.#wss.address();
+	}
+
+	on(event, handler) {
+		this.#wss.on(event, handler);
+	}
+
+	close() {
+		return new Promise((resolve) => {
+			if (!this.#wss) return resolve();
+			for (const client of this.#wss.clients) {
+				client.terminate();
+			}
+			this.#wss.close(resolve);
+		});
+	}
+}

package/src/sql/file_constraints.sql

@@ -0,0 +1,15 @@
+-- PREP: upsert_file_constraint
+INSERT INTO file_constraints (project_id, pattern, visibility)
+VALUES (:project_id, :pattern, :visibility)
+ON CONFLICT (project_id, pattern) DO UPDATE SET
+	visibility = excluded.visibility;
+
+-- PREP: delete_file_constraint
+DELETE FROM file_constraints
+WHERE project_id = :project_id AND pattern = :pattern;
+
+-- PREP: get_file_constraints
+SELECT pattern, visibility
+FROM file_constraints
+WHERE project_id = :project_id
+ORDER BY pattern;

package/src/sql/functions/countTokens.js

@@ -0,0 +1,7 @@
+import { countTokens } from "../../agent/tokens.js";
+
+export const deterministic = true;
+
+export default function (text) {
+	return text ? countTokens(text) : 0;
+}

package/src/sql/functions/hedmatch.js

@@ -0,0 +1,8 @@
+import { hedmatch } from "../../plugins/hedberg/patterns.js";
+
+export const deterministic = true;
+
+export default function (pattern, string) {
+	if (string === null) return 0;
+	return hedmatch(pattern, string) ? 1 : 0;
+}

package/src/sql/functions/hedreplace.js

@@ -0,0 +1,8 @@
+import { hedreplace } from "../../plugins/hedberg/patterns.js";
+
+export const deterministic = true;
+
+export default function (pattern, replacement, string) {
+	if (string === null) return null;
+	return hedreplace(pattern, replacement, string);
+}

package/src/sql/functions/hedsearch.js

@@ -0,0 +1,8 @@
+import { hedsearch } from "../../plugins/hedberg/patterns.js";
+
+export const deterministic = true;
+
+export default function (pattern, string) {
+	if (string === null) return 0;
+	return hedsearch(pattern, string).found ? 1 : 0;
+}