@possumtech/rummy 0.2.8 → 0.3.0

package/src/llm/XaiClient.js

@@ -3,6 +3,7 @@ import msg from "../agent/messages.js";
 export default class XaiClient {
 	#baseUrl;
 	#apiKey;
+	#contextCache = new Map();
 
 	constructor(baseUrl, apiKey) {
 		this.#baseUrl = baseUrl;
@@ -107,7 +108,51 @@ export default class XaiClient {
 		);
 	}
 
-	async getContextSize(_model) {
-		return Number(process.env.RUMMY_CONTEXT_SIZE) || 131072;
+	async getContextSize(model) {
+		if (this.#contextCache.has(model)) return this.#contextCache.get(model);
+
+		if (!this.#apiKey) throw new Error(msg("error.xai_api_key_missing"));
+
+		// Query xAI models endpoint
+		const modelsUrl = this.#baseUrl.replace(/\/responses$/, "/models");
+		const res = await fetch(modelsUrl, {
+			headers: { Authorization: `Bearer ${this.#apiKey}` },
+			signal: AbortSignal.timeout(5000),
+		});
+
+		if (res.ok) {
+			const data = await res.json();
+			const models = data.data || data.models || [];
+			const entry = models.find(
+				(m) => m.id === model || `${m.id}-latest` === model,
+			);
+			if (entry?.context_length) {
+				this.#contextCache.set(model, entry.context_length);
+				return entry.context_length;
+			}
+		}
+
+		// Try /v1/language-models for richer metadata
+		const langUrl = this.#baseUrl.replace(
+			/\/responses$/,
+			`/language-models/${model}`,
+		);
+		const langRes = await fetch(langUrl, {
+			headers: { Authorization: `Bearer ${this.#apiKey}` },
+			signal: AbortSignal.timeout(5000),
+		}).catch(() => null);
+
+		if (langRes?.ok) {
+			const langData = await langRes.json();
+			if (langData?.context_length) {
+				this.#contextCache.set(model, langData.context_length);
+				return langData.context_length;
+			}
+		}
+
+		throw new Error(
+			`Cannot determine context size for xAI model "${model}". ` +
+				"Register the model with addModel(contextLength) or set context_length in the models table.",
+		);
 	}
 }

package/src/plugins/ask_user/README.md

@@ -5,9 +5,8 @@ Presents a question to the user with optional multiple-choice answers.
 ## Registration
 
 - **Tool**: `ask_user`
-- **Modes**: ask, act
-- **Category**: act
-- **Handler**: Parses options (semicolon or comma delimited) and upserts a `proposed` entry awaiting user response.
+- **Category**: `logging`
+- **Handler**: Parses options (semicolon or comma delimited) and upserts at status 202 (proposed) awaiting user response.
 
 ## Projection
 
@@ -15,4 +14,5 @@ Shows the question and answer attributes.
 
 ## Behavior
 
-Options are split by semicolons first, falling back to commas. The entry stays in `proposed` state until resolved by the client.
+Options are split by semicolons first, falling back to commas. The entry
+stays at status 202 until resolved by the client via `run/resolve`.

package/src/plugins/ask_user/ask_user.js

@@ -1,4 +1,4 @@
-import { readFileSync } from "node:fs";
+import docs from "./ask_userDoc.js";
 
 export default class AskUser {
 	#core;
@@ -9,10 +9,10 @@ export default class AskUser {
 		core.on("handler", this.handler.bind(this));
 		core.on("full", this.full.bind(this));
 		core.on("summary", this.summary.bind(this));
-		const docs = readFileSync(new URL("./docs.md", import.meta.url), "utf8");
-		core.filter("instructions.toolDocs", async (content) =>
-			content ? `${content}\n\n${docs}` : docs,
-		);
+		core.filter("instructions.toolDocs", async (docsMap) => {
+			docsMap.ask_user = docs;
+			return docsMap;
+		});
 	}
 
 	async handler(entry, rummy) {

package/src/plugins/ask_user/ask_userDoc.js

@@ -0,0 +1,29 @@
+// Tool doc for <ask_user>. Each entry: [text, rationale].
+// Text goes to the model. Rationale stays in source.
+// Changing ANY line requires reading ALL rationales first.
+const LINES = [
+	// --- Syntax: question attr + options in body
+	['## <ask_user question="[Question?]">[option1; option2; ...]</ask_user>'],
+
+	// --- Constraints FIRST: frames correct usage before examples
+	[
+		"* YOU SHOULD use for decisions, preferences, or approvals the user must make",
+		"Positive framing. Shows what ask_user IS for, not just what it isn't.",
+	],
+	[
+		"* YOU SHOULD use <get> to find information before asking the user",
+		"Gentle redirect. Encourages self-sufficiency without forbidding interaction.",
+	],
+
+	// --- Examples: genuine decision points where user input is valuable
+	[
+		'Example: <ask_user question="Which test framework?">Mocha; Jest; Node Native</ask_user>',
+		"Preference decision. Model truly cannot know this without asking.",
+	],
+	[
+		'Example: <ask_user question="Deploy to staging or production?">staging; production</ask_user>',
+		"Consequential action. Shows ask_user for high-stakes choices.",
+	],
+];
+
+export default LINES.map(([text]) => text).join("\n");

package/src/plugins/budget/BudgetGuard.js

@@ -0,0 +1,74 @@
+import { countTokens } from "../../agent/tokens.js";
+
+export class BudgetExceeded extends Error {
+	constructor(path, requested, remaining) {
+		super(
+			`Budget exceeded: ${path} needs ${requested} tokens, ${remaining} remaining`,
+		);
+		this.name = "BudgetExceeded";
+		this.status = 413;
+		this.path = path;
+		this.requested = requested;
+		this.remaining = remaining;
+	}
+}
+
+export default class BudgetGuard {
+	#ceiling;
+	#baseline;
+	#spent;
+	#tripped;
+	#tripSource;
+
+	constructor(ceiling, baseline) {
+		this.#ceiling = ceiling ?? null;
+		this.#baseline = baseline;
+		this.#spent = 0;
+		this.#tripped = false;
+		this.#tripSource = null;
+	}
+
+	get isTripped() {
+		return this.#tripped;
+	}
+
+	get tripSource() {
+		return this.#tripSource;
+	}
+
+	get remaining() {
+		if (this.#ceiling === null) return Infinity;
+		return this.#ceiling - this.#baseline - this.#spent;
+	}
+
+	get spent() {
+		return this.#spent;
+	}
+
+	check(tokens, path) {
+		if (this.#ceiling === null) return;
+		if (this.#tripped) throw new BudgetExceeded(path, tokens, 0);
+		if (tokens <= 0) return;
+		const remaining = this.remaining;
+		if (tokens > remaining) throw new BudgetExceeded(path, tokens, remaining);
+	}
+
+	charge(tokens) {
+		if (tokens > 0) this.#spent += tokens;
+	}
+
+	trip(source) {
+		this.#tripped = true;
+		this.#tripSource = source;
+	}
+
+	/**
+	 * Compute the token delta for an upsert. New entry = full cost.
+	 * Update = difference between new and old body.
+	 */
+	static delta(newBody, existingBody) {
+		const newTokens = countTokens(newBody);
+		const oldTokens = existingBody ? countTokens(existingBody) : 0;
+		return newTokens - oldTokens;
+	}
+}

package/src/plugins/budget/README.md

@@ -0,0 +1,43 @@
+# budget
+
+Context ceiling enforcement and panic mode recovery.
+
+## Files
+
+- **budget.js** — Plugin. Pre-LLM enforce, BudgetGuard activation,
+  panic prompt generation.
+- **BudgetGuard.js** — Write-layer gate. Installed on KnownStore during
+  dispatch. Checks token delta on every upsert, promote, and body update.
+
+## Registration
+
+- **Hook**: `hooks.budget.enforce` — pre-LLM ceiling check.
+- **Hook**: `hooks.budget.activate(store, contextSize, assembledTokens)` — install guard.
+- **Hook**: `hooks.budget.deactivate(store)` — remove guard.
+- **Hook**: `hooks.budget.panicPrompt({ shortfall, assembledTokens, contextSize })` — generate panic prompt.
+
+## Budget Contract
+
+`contextSize` is the ceiling. `countTokens()` is the measurement.
+Over = 413. Under = 200. No margins.
+
+## BudgetGuard
+
+Installed on KnownStore by TurnExecutor before dispatch, cleared in
+`finally`. Gates `upsert()`, `promoteByPattern()`, `updateBodyByPattern()`.
+
+Exemptions: `status >= 400` (error entries), `model_visible = 0` (audit),
+`fidelity = "archive"` (not in context).
+
+On first violation: `BudgetExceeded` thrown, guard trips, all subsequent
+writes fail. TurnExecutor catches per-tool, writes 413 result entry.
+
+## Panic Mode
+
+When a new prompt exceeds the ceiling, AgentLoop enqueues a panic loop.
+The model receives the exact shortfall and must free space using core
+tools (get, set, known, unknown, rm, mv, cp, summarize, update).
+Excluded: sh, env, search, ask_user.
+
+Strike system: 3 consecutive turns without context reduction = hard 413.
+Any reduction resets the counter. One panic attempt per drain cycle.

package/src/plugins/budget/budget.js

@@ -0,0 +1,79 @@
+import { countTokens } from "../../agent/tokens.js";
+import BudgetGuard, { BudgetExceeded } from "./BudgetGuard.js";
+
+function measureMessages(messages) {
+	return messages.reduce((sum, m) => sum + countTokens(m.content), 0);
+}
+
+export { BudgetExceeded };
+
+export default class Budget {
+	#core;
+
+	constructor(core) {
+		this.#core = core;
+		core.hooks.budget = {
+			enforce: this.enforce.bind(this),
+			activate: this.activate.bind(this),
+			deactivate: this.deactivate.bind(this),
+			panicPrompt: Budget.panicPrompt,
+			BudgetExceeded,
+		};
+	}
+
+	static panicPrompt({ assembledTokens, contextSize }) {
+		const target = Math.floor(contextSize * 0.75);
+		const mustFree = assembledTokens - target;
+		return [
+			`CONTEXT OVERFLOW: ${assembledTokens} tokens, ceiling ${contextSize}.`,
+			`YOU MUST free ${mustFree} tokens to get below ${target} (75%).`,
+			"YOU MUST NOT load or create new content. Only reduce.",
+			"",
+			"<knowns> above shows each entry with its token count.",
+			"Target the largest entries first.",
+			'<rm path="..."/> to delete entries you no longer need.',
+			'<set path="..." fidelity="summary" summary="keywords"/> to compress.',
+			'<set path="..." fidelity="archive"/> to archive out of context.',
+			"<summarize/> when done. <update/> if still working.",
+		].join("\n");
+	}
+
+	async enforce({ contextSize, messages, rows }) {
+		if (!contextSize) {
+			return { messages, rows, demoted: [], assembledTokens: 0, status: 200 };
+		}
+
+		const assembledTokens = measureMessages(messages);
+
+		console.warn(
+			`[RUMMY] Budget enforce: ${assembledTokens} tokens, ceiling ${contextSize}, ${rows.length} rows`,
+		);
+
+		if (assembledTokens > contextSize) {
+			const overflow = assembledTokens - contextSize;
+			console.warn(
+				`[RUMMY] Budget 413: ${assembledTokens} tokens > ${contextSize} ceiling (${overflow} over)`,
+			);
+			return {
+				messages,
+				rows,
+				demoted: [],
+				assembledTokens,
+				status: 413,
+				overflow,
+			};
+		}
+
+		return { messages, rows, demoted: [], assembledTokens, status: 200 };
+	}
+
+	activate(store, contextSize, assembledTokens) {
+		const guard = new BudgetGuard(contextSize, assembledTokens);
+		store.budgetGuard = guard;
+		return guard;
+	}
+
+	deactivate(store) {
+		store.budgetGuard = null;
+	}
+}

package/src/plugins/cp/README.md

@@ -5,9 +5,8 @@ Copies an entry from one path to another within the K/V store.
 ## Registration
 
 - **Tool**: `cp`
-- **Modes**: ask, act
-- **Category**: act
-- **Handler**: Reads source body, writes to destination. K/V destinations resolve immediately (`pass`); file destinations produce a `proposed` entry.
+- **Category**: `logging`
+- **Handler**: Reads source body, writes to destination. Scheme destinations resolve immediately (status 200); file destinations produce status 202 (proposed).
 
 ## Projection
 
@@ -15,4 +14,6 @@ Shows `cp {from} {to}`.
 
 ## Behavior
 
-Warns if the destination already exists and will be overwritten. Uses `KnownStore.scheme()` to determine whether the destination is a K/V path or a file path.
+Warns if the destination already exists and will be overwritten. Uses
+`KnownStore.scheme()` to determine whether the destination is a scheme
+path or a file path.

package/src/plugins/cp/cp.js

@@ -1,5 +1,5 @@
-import { readFileSync } from "node:fs";
 import KnownStore from "../../agent/KnownStore.js";
+import docs from "./cpDoc.js";
 
 export default class Cp {
 	#core;
@@ -10,15 +10,19 @@ export default class Cp {
 		core.on("handler", this.handler.bind(this));
 		core.on("full", this.full.bind(this));
 		core.on("summary", this.summary.bind(this));
-		const docs = readFileSync(new URL("./docs.md", import.meta.url), "utf8");
-		core.filter("instructions.toolDocs", async (content) =>
-			content ? `${content}\n\n${docs}` : docs,
-		);
+		core.filter("instructions.toolDocs", async (docsMap) => {
+			docsMap.cp = docs;
+			return docsMap;
+		});
 	}
 
 	async handler(entry, rummy) {
 		const { entries: store, sequence: turn, runId, loopId } = rummy;
 		const { path, to } = entry.attributes;
+		const VALID = { stored: 1, summary: 1, index: 1, full: 1 };
+		const fidelity = VALID[entry.attributes.fidelity]
+			? entry.attributes.fidelity
+			: undefined;
 
 		const source = await store.getBody(runId, path);
 		if (source === null) return;
@@ -37,7 +41,7 @@ export default class Cp {
 				loopId,
 			});
 		} else {
-			await store.upsert(runId, turn, to, source, 200, { loopId });
+			await store.upsert(runId, turn, to, source, 200, { fidelity, loopId });
 			await store.upsert(runId, turn, entry.resultPath, body, 200, {
 				attributes: { from: path, to, isMove: false, warning },
 				loopId,

package/src/plugins/cp/cpDoc.js

@@ -0,0 +1,29 @@
+// Tool doc for <cp>. Each entry: [text, rationale].
+// Text goes to the model. Rationale stays in source.
+// Changing ANY line requires reading ALL rationales first.
+const LINES = [
+	// --- Syntax: path attr = source, body = destination
+	['## <cp path="[source]">[destination]</cp> - Copy a file or entry'],
+
+	// --- Examples: single copy, glob batch, cross-scheme
+	[
+		'Example: <cp path="src/config.js">src/config.backup.js</cp>',
+		"Simple file copy. Path = source, body = destination.",
+	],
+	[
+		'Example: <cp path="known://plan_*">known://archive_</cp>',
+		"Glob batch copy across known entries. Shows pattern operations on cp.",
+	],
+
+	// --- Constraints
+	[
+		"* Source path accepts globs: `src/*.js`, `known://draft_*`",
+		"Pattern support. Distributes glob teaching beyond get.",
+	],
+	[
+		"* Use `preview` to check matches before bulk copy",
+		"Safety pattern consistent with get and rm preview.",
+	],
+];
+
+export default LINES.map(([text]) => text).join("\n");

package/src/plugins/current/README.md

@@ -1,7 +1,7 @@
 # current
 
 Renders the `<current>` section of the user message — the active loop's
-model responses, tool results, and agent warnings.
+tool results and lifecycle signals.
 
 ## Registration
 
@@ -9,6 +9,6 @@ model responses, tool results, and agent warnings.
 
 ## Behavior
 
-Filters turn_context rows where `category` is `result` or `structural`
-and `source_turn >= loopStartTurn`. Renders each entry chronologically
-with status symbols. Empty on the first turn of a loop.
+Filters turn_context rows where `category === "logging"` and
+`source_turn >= loopStartTurn`. Renders each entry chronologically
+with turn number and status. Empty on the first turn of a loop.

package/src/plugins/current/current.js

@@ -8,9 +8,7 @@ export default class Current {
 
 	async assembleCurrent(content, ctx) {
 		const entries = ctx.rows.filter(
-			(r) =>
-				(r.category === "result" || r.category === "structural") &&
-				r.source_turn >= ctx.loopStartTurn,
+			(r) => r.category === "logging" && r.source_turn >= ctx.loopStartTurn,
 		);
 		if (entries.length === 0) return content;
 
@@ -27,8 +25,13 @@ async function renderToolTag(entry, core) {
 			? JSON.parse(entry.attributes)
 			: entry.attributes;
 
-	const path = `${entry.scheme}://${attrs?.path || attrs?.file || attrs?.command || ""}`;
+	const target = attrs?.path || attrs?.file || attrs?.command || "";
+	const turn = entry.source_turn ? ` turn="${entry.source_turn}"` : "";
 	const status = entry.status ? ` status="${entry.status}"` : "";
+	const summary =
+		typeof attrs?.summary === "string"
+			? ` summary="${attrs.summary.slice(0, 80)}"`
+			: "";
 
 	let body;
 	try {
@@ -41,7 +44,7 @@ async function renderToolTag(entry, core) {
 	}
 
 	if (body) {
-		return `<tool path="${path}"${status}>${body}</tool>`;
+		return `<${entry.scheme} path="${target}"${turn}${status}${summary}>${body}</${entry.scheme}>`;
 	}
-	return `<tool path="${path}"${status}/>`;
+	return `<${entry.scheme} path="${target}"${turn}${status}${summary}/>`;
 }

package/src/plugins/engine/engine.sql

@@ -10,11 +10,4 @@ WHERE
 	AND s.model_visible = 1
 ORDER BY ke.turn, ke.refs, ke.tokens DESC;
 
--- PREP: get_promoted_token_total
-SELECT COALESCE(SUM(ke.tokens), 0) AS total
-FROM known_entries AS ke
-JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
-WHERE
-	ke.run_id = :run_id
-	AND ke.fidelity IN ('full', 'summary')
-	AND s.model_visible = 1;
+

package/src/plugins/engine/turn_context.sql

@@ -37,15 +37,10 @@ WHERE run_id = :run_id AND turn = :turn;
 -- PREP: get_turn_distribution
 SELECT
 	CASE category
-		WHEN 'file' THEN 'files'
-		WHEN 'file_symbols' THEN 'files'
-		WHEN 'file_index' THEN 'keys'
-		WHEN 'known' THEN 'known'
-		WHEN 'known_index' THEN 'keys'
-		WHEN 'unknown' THEN 'history'
-		WHEN 'result' THEN 'history'
-		WHEN 'prompt' THEN 'system'
-		WHEN 'system' THEN 'system'
+		WHEN 'data' THEN 'data'
+		WHEN 'logging' THEN 'logging'
+		WHEN 'unknown' THEN 'unknown'
+		WHEN 'prompt' THEN 'prompt'
 		ELSE 'system'
 	END AS bucket,
 	COALESCE(SUM(tokens), 0) AS tokens,

package/src/plugins/env/README.md

@@ -1,13 +1,12 @@
 # env
 
-Stores environment/context information as a pass-through entry.
+Runs an exploratory shell command and records the output.
 
 ## Registration
 
 - **Tool**: `env`
-- **Modes**: ask, act
-- **Category**: ask
-- **Handler**: Upserts the entry body as `pass` state with original attributes preserved.
+- **Category**: `logging`
+- **Handler**: Upserts the entry at status 202 (proposed) with original attributes preserved.
 
 ## Projection
 

package/src/plugins/env/env.js

@@ -1,4 +1,4 @@
-import { readFileSync } from "node:fs";
+import docs from "./envDoc.js";
 
 export default class Env {
 	#core;
@@ -9,10 +9,10 @@ export default class Env {
 		core.on("handler", this.handler.bind(this));
 		core.on("full", this.full.bind(this));
 		core.on("summary", this.summary.bind(this));
-		const docs = readFileSync(new URL("./docs.md", import.meta.url), "utf8");
-		core.filter("instructions.toolDocs", async (content) =>
-			content ? `${content}\n\n${docs}` : docs,
-		);
+		core.filter("instructions.toolDocs", async (docsMap) => {
+			docsMap.env = docs;
+			return docsMap;
+		});
 	}
 
 	async handler(entry, rummy) {

package/src/plugins/env/envDoc.js

@@ -0,0 +1,29 @@
+// Tool doc for <env>. Each entry: [text, rationale].
+// Text goes to the model. Rationale stays in source.
+// Changing ANY line requires reading ALL rationales first.
+const LINES = [
+	// --- Syntax
+	["## <env>[command]</env> - Run an exploratory shell command"],
+
+	// --- Examples: version check and git status — safe, read-only commands
+	[
+		"Example: <env>npm --version</env>",
+		"Version check. Safe, no side effects.",
+	],
+	[
+		"Example: <env>git log --oneline -5</env>",
+		"Git history. Shows env for read-only investigation.",
+	],
+
+	// --- Constraints: hard boundaries
+	[
+		'* YOU MUST NOT use <env/> to read or list files — use <get path="*" preview/> instead',
+		"Prevents cat/ls through shell. Forces file access through get for proper tracking.",
+	],
+	[
+		"* YOU MUST use <sh/> for commands with side effects",
+		"Separates exploration from action. env = observe, sh = mutate.",
+	],
+];
+
+export default LINES.map(([text]) => text).join("\n");

package/src/plugins/file/README.md

@@ -2,24 +2,21 @@
 
 Owns file-related projections and file constraint management.
 
-## Files
-
-- **file.js** — Plugin registration, projection hooks, and constraint CRUD (activate, ignore, drop).
-- **FileScanner.js** — Scans project directories for file entries.
-- **GitProvider.js** — Git integration for file discovery and status.
-- **ProjectContext.js** — Builds project-level context from scanned files.
-- **FsProvider.js** — Filesystem abstraction for file reading/writing.
-
 ## Registration
 
-- **Projections**: Registers `onProject` handlers for schemes: `file`, `known`, `skill`, `ask`, `act`, `progress`. All project the entry body directly.
+- **Schemes**: `file` (bare paths), `http`, `https` — all `category: "data"`
+- **Views**: `full` and `summary` for file scheme. Default identity views
+  for `http`/`https` (overridden by rummy.web when installed).
 - **No tool handler** — file operations are dispatched through `set`, `get`, `rm`, etc.
 
 ## File Constraints
 
-Static methods `activate`, `ignore`, and `drop` manage per-project file constraints in the database. Constraints control file visibility across all runs:
+Static methods `setConstraint` and `dropConstraint` manage per-project
+file constraints in the database. Constraints are project-level config
+(backbone), not tool dispatch. See SPEC.md §2.3.
 
-- `active` / `readonly` — always promoted into context.
+- `active` / `readonly` — promoted into context.
 - `ignore` — excluded from scans; demotes existing entries.
 
-Paths are normalized to project-relative when absolute.
+Entry promotion/demotion from constraints goes through the standard
+tool handler chain via `dispatchTool`.