@possumtech/rummy 0.2.8 → 0.3.1

package/src/llm/OpenRouterClient.js

@@ -72,7 +72,29 @@ export default class OpenRouterClient {
 		return data;
 	}
 
-	async getContextSize(_model) {
-		return Number(process.env.RUMMY_CONTEXT_SIZE) || DEFAULT_CONTEXT_SIZE;
+	#contextCache = new Map();
+
+	async getContextSize(model) {
+		if (process.env.RUMMY_CONTEXT_SIZE)
+			return Number(process.env.RUMMY_CONTEXT_SIZE);
+
+		if (this.#contextCache.has(model)) return this.#contextCache.get(model);
+
+		try {
+			const res = await fetch(`${this.#baseUrl}/models`, {
+				headers: { Authorization: `Bearer ${this.#apiKey}` },
+				signal: AbortSignal.timeout(5000),
+			});
+			if (res.ok) {
+				const data = await res.json();
+				const entry = data.data?.find((m) => m.id === model);
+				if (entry?.context_length) {
+					this.#contextCache.set(model, entry.context_length);
+					return entry.context_length;
+				}
+			}
+		} catch {}
+
+		return DEFAULT_CONTEXT_SIZE;
 	}
 }

package/src/llm/XaiClient.js

@@ -3,6 +3,7 @@ import msg from "../agent/messages.js";
 export default class XaiClient {
 	#baseUrl;
 	#apiKey;
+	#contextCache = new Map();
 
 	constructor(baseUrl, apiKey) {
 		this.#baseUrl = baseUrl;
@@ -107,7 +108,51 @@ export default class XaiClient {
 		);
 	}
 
-	async getContextSize(_model) {
-		return Number(process.env.RUMMY_CONTEXT_SIZE) || 131072;
+	async getContextSize(model) {
+		if (this.#contextCache.has(model)) return this.#contextCache.get(model);
+
+		if (!this.#apiKey) throw new Error(msg("error.xai_api_key_missing"));
+
+		// Query xAI models endpoint
+		const modelsUrl = this.#baseUrl.replace(/\/responses$/, "/models");
+		const res = await fetch(modelsUrl, {
+			headers: { Authorization: `Bearer ${this.#apiKey}` },
+			signal: AbortSignal.timeout(5000),
+		});
+
+		if (res.ok) {
+			const data = await res.json();
+			const models = data.data || data.models || [];
+			const entry = models.find(
+				(m) => m.id === model || `${m.id}-latest` === model,
+			);
+			if (entry?.context_length) {
+				this.#contextCache.set(model, entry.context_length);
+				return entry.context_length;
+			}
+		}
+
+		// Try /v1/language-models for richer metadata
+		const langUrl = this.#baseUrl.replace(
+			/\/responses$/,
+			`/language-models/${model}`,
+		);
+		const langRes = await fetch(langUrl, {
+			headers: { Authorization: `Bearer ${this.#apiKey}` },
+			signal: AbortSignal.timeout(5000),
+		}).catch(() => null);
+
+		if (langRes?.ok) {
+			const langData = await langRes.json();
+			if (langData?.context_length) {
+				this.#contextCache.set(model, langData.context_length);
+				return langData.context_length;
+			}
+		}
+
+		throw new Error(
+			`Cannot determine context size for xAI model "${model}". ` +
+				"Register the model with addModel(contextLength) or set context_length in the models table.",
+		);
 	}
 }

package/src/plugins/ask_user/README.md

@@ -5,9 +5,8 @@ Presents a question to the user with optional multiple-choice answers.
 ## Registration
 
 - **Tool**: `ask_user`
-- **Modes**: ask, act
-- **Category**: act
-- **Handler**: Parses options (semicolon or comma delimited) and upserts a `proposed` entry awaiting user response.
+- **Category**: `logging`
+- **Handler**: Parses options (semicolon or comma delimited) and upserts at status 202 (proposed) awaiting user response.
 
 ## Projection
 
@@ -15,4 +14,5 @@ Shows the question and answer attributes.
 
 ## Behavior
 
-Options are split by semicolons first, falling back to commas. The entry stays in `proposed` state until resolved by the client.
+Options are split by semicolons first, falling back to commas. The entry
+stays at status 202 until resolved by the client via `run/resolve`.

package/src/plugins/ask_user/ask_user.js

@@ -1,4 +1,4 @@
-import { readFileSync } from "node:fs";
+import docs from "./ask_userDoc.js";
 
 export default class AskUser {
 	#core;
@@ -9,10 +9,10 @@ export default class AskUser {
 		core.on("handler", this.handler.bind(this));
 		core.on("full", this.full.bind(this));
 		core.on("summary", this.summary.bind(this));
-		const docs = readFileSync(new URL("./docs.md", import.meta.url), "utf8");
-		core.filter("instructions.toolDocs", async (content) =>
-			content ? `${content}\n\n${docs}` : docs,
-		);
+		core.filter("instructions.toolDocs", async (docsMap) => {
+			docsMap.ask_user = docs;
+			return docsMap;
+		});
 	}
 
 	async handler(entry, rummy) {

package/src/plugins/ask_user/ask_userDoc.js

@@ -0,0 +1,29 @@
+// Tool doc for <ask_user>. Each entry: [text, rationale].
+// Text goes to the model. Rationale stays in source.
+// Changing ANY line requires reading ALL rationales first.
+const LINES = [
+	// --- Syntax: question attr + options in body
+	['## <ask_user question="[Question?]">[option1; option2; ...]</ask_user>'],
+
+	// --- Constraints FIRST: frames correct usage before examples
+	[
+		"* YOU SHOULD use for decisions, preferences, or approvals the user must make",
+		"Positive framing. Shows what ask_user IS for, not just what it isn't.",
+	],
+	[
+		"* YOU SHOULD use <get> to find information before asking the user",
+		"Gentle redirect. Encourages self-sufficiency without forbidding interaction.",
+	],
+
+	// --- Examples: genuine decision points where user input is valuable
+	[
+		'Example: <ask_user question="Which test framework?">Mocha; Jest; Node Native</ask_user>',
+		"Preference decision. Model truly cannot know this without asking.",
+	],
+	[
+		'Example: <ask_user question="Deploy to staging or production?">staging; production</ask_user>',
+		"Consequential action. Shows ask_user for high-stakes choices.",
+	],
+];
+
+export default LINES.map(([text]) => text).join("\n");

package/src/plugins/budget/README.md

@@ -0,0 +1,31 @@
+# budget
+
+Context ceiling enforcement.
+
+## Files
+
+- **budget.js** — Plugin. Pre-LLM enforce, BudgetGuard activation.
+- **BudgetGuard.js** — Write-layer gate. Installed on KnownStore during
+  dispatch. Checks token delta on every upsert, promote, and body update.
+
+## Registration
+
+- **Hook**: `hooks.budget.enforce` — pre-LLM ceiling check.
+- **Hook**: `hooks.budget.activate(store, contextSize, assembledTokens)` — install guard.
+- **Hook**: `hooks.budget.deactivate(store)` — remove guard.
+
+## Budget Contract
+
+`contextSize` is the ceiling. `countTokens()` is the measurement.
+Over = 413. Under = 200. No margins.
+
+## BudgetGuard
+
+Installed on KnownStore by TurnExecutor before dispatch, cleared in
+`finally`. Gates `upsert()`, `promoteByPattern()`, `updateBodyByPattern()`.
+
+Exemptions: `status >= 400` (error entries), `model_visible = 0` (audit),
+`fidelity = "archive"` (not in context).
+
+On first violation: `BudgetExceeded` thrown, guard trips, all subsequent
+writes fail. TurnExecutor catches per-tool, writes 413 result entry.

package/src/plugins/budget/budget.js

@@ -0,0 +1,55 @@
+import { countTokens } from "../../agent/tokens.js";
+
+function measureMessages(messages) {
+	return messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+}
+
+export default class Budget {
+	#core;
+
+	constructor(core) {
+		this.#core = core;
+		core.registerScheme({
+			name: "budget",
+			modelVisible: 1,
+			category: "logging",
+		});
+		core.hooks.tools.onView("budget", (entry) => entry.body);
+		core.hooks.budget = {
+			enforce: this.enforce.bind(this),
+		};
+	}
+
+	async enforce({ contextSize, messages, rows, lastPromptTokens = 0 }) {
+		if (!contextSize) {
+			return { messages, rows, demoted: [], assembledTokens: 0, status: 200 };
+		}
+
+		// Prefer actual prompt_tokens from the last API response — the estimate
+		// from measureMessages can be wildly off for structured/XML-heavy content.
+		const assembledTokens =
+			lastPromptTokens > 0 ? lastPromptTokens : measureMessages(messages);
+
+		console.warn(
+			`[RUMMY] Budget enforce: ${assembledTokens} tokens (${lastPromptTokens > 0 ? "actual" : "estimated"}), ceiling ${contextSize}, ${rows.length} rows`,
+		);
+
+		const ceiling = Math.floor(contextSize * 0.9);
+		if (assembledTokens > ceiling) {
+			const overflow = assembledTokens - ceiling;
+			console.warn(
+				`[RUMMY] Budget 413: ${assembledTokens} tokens > ${contextSize} ceiling (${overflow} over)`,
+			);
+			return {
+				messages,
+				rows,
+				demoted: [],
+				assembledTokens,
+				status: 413,
+				overflow,
+			};
+		}
+
+		return { messages, rows, demoted: [], assembledTokens, status: 200 };
+	}
+}

package/src/plugins/cp/README.md

@@ -5,9 +5,8 @@ Copies an entry from one path to another within the K/V store.
 ## Registration
 
 - **Tool**: `cp`
-- **Modes**: ask, act
-- **Category**: act
-- **Handler**: Reads source body, writes to destination. K/V destinations resolve immediately (`pass`); file destinations produce a `proposed` entry.
+- **Category**: `logging`
+- **Handler**: Reads source body, writes to destination. Scheme destinations resolve immediately (status 200); file destinations produce status 202 (proposed).
 
 ## Projection
 
@@ -15,4 +14,6 @@ Shows `cp {from} {to}`.
 
 ## Behavior
 
-Warns if the destination already exists and will be overwritten. Uses `KnownStore.scheme()` to determine whether the destination is a K/V path or a file path.
+Warns if the destination already exists and will be overwritten. Uses
+`KnownStore.scheme()` to determine whether the destination is a scheme
+path or a file path.

package/src/plugins/cp/cp.js

@@ -1,5 +1,5 @@
-import { readFileSync } from "node:fs";
 import KnownStore from "../../agent/KnownStore.js";
+import docs from "./cpDoc.js";
 
 export default class Cp {
 	#core;
@@ -10,15 +10,19 @@ export default class Cp {
 		core.on("handler", this.handler.bind(this));
 		core.on("full", this.full.bind(this));
 		core.on("summary", this.summary.bind(this));
-		const docs = readFileSync(new URL("./docs.md", import.meta.url), "utf8");
-		core.filter("instructions.toolDocs", async (content) =>
-			content ? `${content}\n\n${docs}` : docs,
-		);
+		core.filter("instructions.toolDocs", async (docsMap) => {
+			docsMap.cp = docs;
+			return docsMap;
+		});
 	}
 
 	async handler(entry, rummy) {
 		const { entries: store, sequence: turn, runId, loopId } = rummy;
 		const { path, to } = entry.attributes;
+		const VALID = { stored: 1, summary: 1, index: 1, full: 1, archive: 1 };
+		const fidelity = VALID[entry.attributes.fidelity]
+			? entry.attributes.fidelity
+			: undefined;
 
 		const source = await store.getBody(runId, path);
 		if (source === null) return;
@@ -37,7 +41,7 @@ export default class Cp {
 				loopId,
 			});
 		} else {
-			await store.upsert(runId, turn, to, source, 200, { loopId });
+			await store.upsert(runId, turn, to, source, 200, { fidelity, loopId });
 			await store.upsert(runId, turn, entry.resultPath, body, 200, {
 				attributes: { from: path, to, isMove: false, warning },
 				loopId,

package/src/plugins/cp/cpDoc.js

@@ -0,0 +1,29 @@
+// Tool doc for <cp>. Each entry: [text, rationale].
+// Text goes to the model. Rationale stays in source.
+// Changing ANY line requires reading ALL rationales first.
+const LINES = [
+	// --- Syntax: path attr = source, body = destination
+	['## <cp path="[source]">[destination]</cp> - Copy a file or entry'],
+
+	// --- Examples: single copy, glob batch, cross-scheme
+	[
+		'Example: <cp path="src/config.js">src/config.backup.js</cp>',
+		"Simple file copy. Path = source, body = destination.",
+	],
+	[
+		'Example: <cp path="known://plan_*">known://archive_</cp>',
+		"Glob batch copy across known entries. Shows pattern operations on cp.",
+	],
+
+	// --- Constraints
+	[
+		"* Source path accepts patterns: `src/*.js`, `known://draft_*`",
+		"Pattern support. Distributes glob teaching beyond get.",
+	],
+	[
+		"* Use `preview` to check matches before bulk copy",
+		"Safety pattern consistent with get and rm preview.",
+	],
+];
+
+export default LINES.map(([text]) => text).join("\n");

package/src/plugins/engine/engine.sql

@@ -10,11 +10,4 @@ WHERE
 	AND s.model_visible = 1
 ORDER BY ke.turn, ke.refs, ke.tokens DESC;
 
--- PREP: get_promoted_token_total
-SELECT COALESCE(SUM(ke.tokens), 0) AS total
-FROM known_entries AS ke
-JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
-WHERE
-	ke.run_id = :run_id
-	AND ke.fidelity IN ('full', 'summary')
-	AND s.model_visible = 1;
+

package/src/plugins/engine/turn_context.sql

@@ -37,15 +37,10 @@ WHERE run_id = :run_id AND turn = :turn;
 -- PREP: get_turn_distribution
 SELECT
 	CASE category
-		WHEN 'file' THEN 'files'
-		WHEN 'file_symbols' THEN 'files'
-		WHEN 'file_index' THEN 'keys'
-		WHEN 'known' THEN 'known'
-		WHEN 'known_index' THEN 'keys'
-		WHEN 'unknown' THEN 'history'
-		WHEN 'result' THEN 'history'
-		WHEN 'prompt' THEN 'system'
-		WHEN 'system' THEN 'system'
+		WHEN 'data' THEN 'data'
+		WHEN 'logging' THEN 'logging'
+		WHEN 'unknown' THEN 'unknown'
+		WHEN 'prompt' THEN 'prompt'
 		ELSE 'system'
 	END AS bucket,
 	COALESCE(SUM(tokens), 0) AS tokens,

package/src/plugins/env/README.md

@@ -1,13 +1,12 @@
 # env
 
-Stores environment/context information as a pass-through entry.
+Runs an exploratory shell command and records the output.
 
 ## Registration
 
 - **Tool**: `env`
-- **Modes**: ask, act
-- **Category**: ask
-- **Handler**: Upserts the entry body as `pass` state with original attributes preserved.
+- **Category**: `logging`
+- **Handler**: Upserts the entry at status 202 (proposed) with original attributes preserved.
 
 ## Projection
 

package/src/plugins/env/env.js

@@ -1,4 +1,4 @@
-import { readFileSync } from "node:fs";
+import docs from "./envDoc.js";
 
 export default class Env {
 	#core;
@@ -9,10 +9,10 @@ export default class Env {
 		core.on("handler", this.handler.bind(this));
 		core.on("full", this.full.bind(this));
 		core.on("summary", this.summary.bind(this));
-		const docs = readFileSync(new URL("./docs.md", import.meta.url), "utf8");
-		core.filter("instructions.toolDocs", async (content) =>
-			content ? `${content}\n\n${docs}` : docs,
-		);
+		core.filter("instructions.toolDocs", async (docsMap) => {
+			docsMap.env = docs;
+			return docsMap;
+		});
 	}
 
 	async handler(entry, rummy) {

package/src/plugins/env/envDoc.js

@@ -0,0 +1,29 @@
+// Tool doc for <env>. Each entry: [text, rationale].
+// Text goes to the model. Rationale stays in source.
+// Changing ANY line requires reading ALL rationales first.
+const LINES = [
+	// --- Syntax
+	["## <env>[command]</env> - Run an exploratory shell command"],
+
+	// --- Examples: version check and git status — safe, read-only commands
+	[
+		"Example: <env>npm --version</env>",
+		"Version check. Safe, no side effects.",
+	],
+	[
+		"Example: <env>git log --oneline -5</env>",
+		"Git history. Shows env for read-only investigation.",
+	],
+
+	// --- Constraints: hard boundaries
+	[
+		'* YOU MUST NOT use <env/> to read or list files — use <get path="*" preview/> instead',
+		"Prevents cat/ls through shell. Forces file access through get for proper tracking.",
+	],
+	[
+		"* YOU MUST use <sh/> for commands with side effects",
+		"Separates exploration from action. env = observe, sh = mutate.",
+	],
+];
+
+export default LINES.map(([text]) => text).join("\n");

package/src/plugins/file/README.md

@@ -2,24 +2,21 @@
 
 Owns file-related projections and file constraint management.
 
-## Files
-
-- **file.js** — Plugin registration, projection hooks, and constraint CRUD (activate, ignore, drop).
-- **FileScanner.js** — Scans project directories for file entries.
-- **GitProvider.js** — Git integration for file discovery and status.
-- **ProjectContext.js** — Builds project-level context from scanned files.
-- **FsProvider.js** — Filesystem abstraction for file reading/writing.
-
 ## Registration
 
-- **Projections**: Registers `onProject` handlers for schemes: `file`, `known`, `skill`, `ask`, `act`, `progress`. All project the entry body directly.
+- **Schemes**: `file` (bare paths), `http`, `https` — all `category: "data"`
+- **Views**: `full` and `summary` for file scheme. Default identity views
+  for `http`/`https` (overridden by rummy.web when installed).
 - **No tool handler** — file operations are dispatched through `set`, `get`, `rm`, etc.
 
 ## File Constraints
 
-Static methods `activate`, `ignore`, and `drop` manage per-project file constraints in the database. Constraints control file visibility across all runs:
+Static methods `setConstraint` and `dropConstraint` manage per-project
+file constraints in the database. Constraints are project-level config
+(backbone), not tool dispatch. See SPEC.md §2.3.
 
-- `active` / `readonly` — always promoted into context.
+- `active` / `readonly` — promoted into context.
 - `ignore` — excluded from scans; demotes existing entries.
 
-Paths are normalized to project-relative when absolute.
+Entry promotion/demotion from constraints goes through the standard
+tool handler chain via `dispatchTool`.

package/src/plugins/file/file.js

@@ -1,34 +1,45 @@
 import { isAbsolute, relative } from "node:path";
 
+/**
+ * File plugin: projections and constraints for filesystem entries.
+ *
+ * Bare file paths (src/app.js) have scheme=NULL in the DB because
+ * schemeOf() only recognizes "://" patterns. The schemes table has
+ * a "file" entry so v_model_context can JOIN via COALESCE(scheme, 'file').
+ * This is the one exception to "every scheme has a plugin owner" —
+ * the file plugin owns the NULL scheme through the "file" registry entry.
+ */
 export default class File {
 	#core;
 
 	constructor(core) {
 		this.#core = core;
-		core.registerScheme({ category: "file" });
-		core.registerScheme({ name: "http", category: "file" });
-		core.registerScheme({ name: "https", category: "file" });
+		// "file" scheme covers bare paths (scheme IS NULL in DB)
+		core.registerScheme({ category: "data" });
+		core.registerScheme({ name: "http", category: "data" });
+		core.registerScheme({ name: "https", category: "data" });
 		core.on("full", this.full.bind(this));
-
-		// Register identity projections for schemes that just pass through body
-		for (const scheme of ["known", "skill", "ask", "act", "progress"]) {
-			core.hooks.tools.onView(scheme, (entry) => entry.body);
-		}
+		core.on("summary", this.summary.bind(this));
+		// Default identity views for http/https — rummy.web overrides these
+		core.hooks.tools.onView("http", (entry) => entry.body);
+		core.hooks.tools.onView("https", (entry) => entry.body);
 	}
 
 	full(entry) {
 		return entry.body;
 	}
 
-	static async activate(
-		db,
-		knownStore,
-		projectId,
-		pattern,
-		visibility = "active",
-	) {
+	summary() {
+		return "";
+	}
+
+	/**
+	 * Set a project-level file constraint. Backbone operation —
+	 * constraints are project config, not tool dispatch.
+	 */
+	static async setConstraint(db, projectId, pattern, visibility = "active") {
 		const path = await normalizePath(db, projectId, pattern);
-		if (!path) return { status: "ok" };
+		if (!path) return null;
 
 		await db.upsert_file_constraint.run({
 			project_id: projectId,
@@ -36,34 +47,22 @@ export default class File {
 			visibility,
 		});
 
-		const runs = await db.get_all_runs.all({ project_id: projectId });
-		if (visibility === "active") {
-			for (const run of runs) {
-				await knownStore.promoteByPattern(run.id, path, null, 0);
-			}
-		} else if (visibility === "ignore") {
-			for (const run of runs) {
-				await knownStore.demoteByPattern(run.id, path, null);
-			}
-		}
-
-		return { status: "ok" };
-	}
-
-	static async ignore(db, knownStore, projectId, pattern) {
-		return File.activate(db, knownStore, projectId, pattern, "ignore");
+		return path;
 	}
 
-	static async drop(db, projectId, pattern) {
+	/**
+	 * Remove a project-level file constraint.
+	 */
+	static async dropConstraint(db, projectId, pattern) {
 		const path = await normalizePath(db, projectId, pattern);
-		if (!path) return { status: "ok" };
+		if (!path) return null;
 
 		await db.delete_file_constraint.run({
 			project_id: projectId,
 			pattern: path,
 		});
 
-		return { status: "ok" };
+		return path;
 	}
 }
 

package/src/plugins/get/README.md

@@ -5,8 +5,7 @@ Retrieves and promotes entries by path or glob pattern.
 ## Registration
 
 - **Tool**: `get`
-- **Modes**: ask, act
-- **Category**: ask
+- **Category**: `logging`
 - **Handler**: Fetches matching entries via `getEntriesByPattern`, promotes them with `promoteByPattern`, and records the result.
 
 ## Projection
@@ -17,3 +16,4 @@ Shows `get {path}` followed by the entry body.
 
 - Pattern queries (globs or body filters) produce a summary of matched paths.
 - Exact path queries report the path and token count, or "not found".
+- Budget check: rejects with 413 if incoming tokens exceed remaining context.