@possumtech/rummy 2.0.1 → 2.1.0

package/src/agent/Entries.js

@@ -1,5 +1,6 @@
 import slugify from "../sql/functions/slugify.js";
 import { PermissionError } from "./errors.js";
+import encodeSegment from "./pathEncode.js";
 
 export default class Entries {
 	#db;
@@ -14,10 +15,7 @@ export default class Entries {
 		this.#onChanged = onChanged;
 	}
 
-	/**
-	 * Populate the scheme cache. Can be called explicitly (e.g. at boot
-	 * after initPlugins finishes) or runs lazily on first need. Idempotent.
-	 */
+	// Populate the scheme cache; idempotent, lazy on first need.
 	async loadSchemes(db) {
 		const rows = await (db || this.#db).get_all_schemes.all();
 		this.#schemes.clear();
@@ -51,9 +49,9 @@ export default class Entries {
 		try {
 			// Decode first (idempotent), then encode — but preserve slashes
 			const decoded = decodeURIComponent(rest);
-			return `${scheme}://${decoded.split("/").map(encodeURIComponent).join("/")}`;
+			return `${scheme}://${decoded.split("/").map(encodeSegment).join("/")}`;
 		} catch {
-			return `${scheme}://${rest.split("/").map(encodeURIComponent).join("/")}`;
+			return `${scheme}://${rest.split("/").map(encodeSegment).join("/")}`;
 		}
 	}
 
@@ -63,7 +61,7 @@ export default class Entries {
 	}
 
 	async dedup(runId, scheme, target, turn) {
-		const encodedTarget = encodeURIComponent(target);
+		const encodedTarget = encodeSegment(target);
 		const turnPrefix = turn ? `turn_${turn}/` : "";
 		const candidate = `${scheme}://${turnPrefix}${encodedTarget}`;
 		const existing = await this.#db.get_entry_body.get({
@@ -74,12 +72,15 @@ export default class Entries {
 		return `${candidate}_${++this.#seq}`;
 	}
 
-	// Log entries share a single namespace at log://turn_N/action/slug.
-	// The action segment is the tool/plugin name (set, get, search, update,
-	// error, etc.). Target is URL-encoded so slashes and scheme separators
-	// survive round-trips.
+	// Single namespace log://turn_N/action/slug; target URL-encoded for round-trip safety.
 	async logPath(runId, turn, action, target) {
-		const encodedTarget = encodeURIComponent(target);
+		// Cap target before encoding: the schema's CHECK(length(path) <= 2048)
+		// otherwise blows up when callers pass long error messages or other
+		// arbitrary text. encodeURIComponent expands ~3x for ASCII, more for
+		// Unicode; 150 raw chars stays comfortably under 2048 even after
+		// worst-case expansion. The full message belongs in body, not path.
+		const safeTarget = String(target).slice(0, 150);
+		const encodedTarget = encodeSegment(safeTarget);
 		const candidate = `log://turn_${turn}/${action}/${encodedTarget}`;
 		const existing = await this.#db.get_entry_body.get({
 			run_id: runId,
@@ -90,9 +91,7 @@ export default class Entries {
 	}
 
 	async slugPath(runId, scheme, content, summary) {
-		// Prefer summary, fall back to body content, then empty — slugify
-		// handles empty explicitly by returning "" and the caller generates
-		// a sequence-only path.
+		// summary > content > empty; slugify("") yields "" and we sequence-only.
 		let source = "";
 		if (summary) source = summary;
 		else if (content) source = content;
@@ -111,12 +110,7 @@ export default class Entries {
 		return `${prefix}${base}_${++this.#seq}`;
 	}
 
-	/**
-	 * Resolve a scheme's declared scope kind + writer list + category.
-	 * Unregistered or declaration-less schemes default to run-level +
-	 * model/plugin writers so ad-hoc paths (e.g. bare filenames) still
-	 * work.
-	 */
+	// Scheme's scope/writers/category; bare paths default to run + model/plugin.
 	async #schemeRules(scheme) {
 		await this.#ensureSchemes();
 		const row = scheme ? this.#schemes.get(scheme) : null;
@@ -154,17 +148,7 @@ export default class Entries {
 		return `run:${runId}`;
 	}
 
-	/**
-	 * set — create or update an entry. The semantically wide primitive.
-	 *
-	 * Modes (selected by which options are present):
-	 *   — write content:         body given, state ∈ {proposed,streaming,resolved,failed,cancelled}
-	 *   — change visibility only:  visibility given, body omitted
-	 *   — change state only:     state given, body omitted (resolve a proposal)
-	 *   — merge attributes:      attributes given, body omitted
-	 *   — append to body:        append:true (streaming)
-	 *   — pattern match:         path contains wildcards or bodyFilter set
-	 */
+	// set — create or update an entry; see PLUGINS.md primitives.
 	async set({
 		runId,
 		projectId = null,
@@ -185,14 +169,9 @@ export default class Entries {
 		if (!runId) throw new Error("set: runId is required");
 		if (!path) throw new Error("set: path is required");
 
-		// Pattern mode is explicit (pattern: true) or implicit when a
-		// body filter is supplied. The literal `*` character can appear
-		// inside legitimate exact paths (e.g. rm://foo%2F* as a result
-		// path for an rm against a pattern); we don't infer pattern mode
-		// from the path alone.
+		// Pattern mode is explicit; never inferred from `*` in path.
 		const isPattern = pattern === true || bodyFilter !== null;
 
-		// Pattern mode: update matching entries (visibility / body / both).
 		if (isPattern) {
 			if (body != null && !append) {
 				await this.#db.update_body_by_pattern.run({
@@ -279,14 +258,7 @@ export default class Entries {
 			throw new PermissionError(scheme, writer, writers);
 		}
 		const scope = this.#resolveScope(kind, runId, projectId);
-		// Log entries self-describe via `action` so consumers (renderer,
-		// client UIs, tests) can read the action without parsing the
-		// path. Only inject `action` when the caller passes attributes
-		// — a null `attributes` means "don't touch existing" and the
-		// SQL's COALESCE handles preservation on UPDATE. If we generated
-		// `{action: m[1]}` for every null-attributes log write, every
-		// body-only update to a log entry would clobber existing attrs
-		// (command, summary, demotedCount, ...).
+		// Inject `action` only when caller passes attributes; null means COALESCE preserves existing.
 		const effectiveAttributes = attributes ? { ...attributes } : null;
 		if (scheme === "log" && effectiveAttributes) {
 			const m = normalized.match(/^log:\/\/turn_\d+\/([^/]+)\//);
@@ -321,11 +293,7 @@ export default class Entries {
 		}
 	}
 
-	/**
-	 * get — promote entry(ies) to visible visibility. Default visibility is
-	 * "visible"; pass visibility explicitly for a read-with-side-effect at
-	 * a different visibility (rare).
-	 */
+	// get — promote entry(ies); see PLUGINS.md primitives.
 	async get({
 		runId,
 		turn = 0,
@@ -352,11 +320,7 @@ export default class Entries {
 		this.#emitChanged(runId, path, "promote");
 	}
 
-	/**
-	 * rm — remove entry view(s). Matches single path or pattern; optional
-	 * bodyFilter narrows pattern matches. `filesOnly` restricts to bare
-	 * file-scheme entries (scheme IS NULL).
-	 */
+	// rm — remove entry view(s); see PLUGINS.md primitives.
 	async rm({ runId, path, bodyFilter = null, filesOnly = false }) {
 		if (!runId) throw new Error("rm: runId is required");
 		if (!path) throw new Error("rm: path is required");
@@ -381,10 +345,7 @@ export default class Entries {
 		this.#emitChanged(runId, path, "remove");
 	}
 
-	/**
-	 * cp — copy an entry to a new path. Source body becomes new body;
-	 * source view unchanged.
-	 */
+	// cp — copy an entry to a new path; see PLUGINS.md primitives.
 	async cp({
 		runId,
 		turn = 0,
@@ -411,9 +372,7 @@ export default class Entries {
 		});
 	}
 
-	/**
-	 * mv — rename an entry. Equivalent to cp + rm on source.
-	 */
+	// mv — rename (cp + rm).
 	async mv({
 		runId,
 		turn = 0,
@@ -439,13 +398,7 @@ export default class Entries {
 		await this.rm({ runId, path: from });
 	}
 
-	/**
-	 * update — once-per-turn lifecycle signal from the model (or plugin
-	 * speaking on its behalf). Writes to update://<slug> with body as the
-	 * content and attributes.status carrying the model's continuation code
-	 * (102 continue, 200/204 terminal, 422 can't-answer). Returns the
-	 * slug path.
-	 */
+	// update — once-per-turn lifecycle signal; see PLUGINS.md.
 	async update({
 		runId,
 		turn = 0,
@@ -475,7 +428,12 @@ export default class Entries {
 		runId,
 		path,
 		body = null,
-		{ limit = null, offset = null, includeAuditSchemes = false } = {},
+		{
+			limit = null,
+			offset = null,
+			since = null,
+			includeAuditSchemes = false,
+		} = {},
 	) {
 		return this.#db.get_entries_by_pattern.all({
 			run_id: runId,
@@ -483,6 +441,7 @@ export default class Entries {
 			body: body ? body : null,
 			limit,
 			offset,
+			since,
 			include_audit_schemes: includeAuditSchemes ? 1 : null,
 		});
 	}
@@ -497,10 +456,7 @@ export default class Entries {
 	}
 
 	async waitForResolution(runId, path) {
-		// Check current state first — if a synchronous in-process resolver
-		// (yolo) flipped the entry to terminal during proposal.pending,
-		// the state change has already happened and no future drain will
-		// fire. Without this guard, in-process resolvers would deadlock.
+		// Pre-check: yolo's synchronous resolver may have already flipped state, no drain will fire.
 		const current = await this.getState(runId, path);
 		if (
 			current &&
@@ -559,9 +515,7 @@ export default class Entries {
 		return new Set(rows.map((r) => r.body));
 	}
 
-	/**
-	 * Unknown entries for a run, in DB order. Rows include path + body.
-	 */
+	// Unknown entries in DB order; rows include path + body.
 	async getUnknowns(runId) {
 		return this.#db.get_unknowns.all({ run_id: runId });
 	}
@@ -580,14 +534,7 @@ export default class Entries {
 		});
 	}
 
-	/**
-	 * Demote all promoted entries for a run on a given turn. Returns the
-	 * affected rows (path, tokens) so callers can summarize.
-	 *
-	 * Implemented as SELECT-then-UPDATE because SQLite's RETURNING doesn't
-	 * support the cross-table lookup needed to report content paths/tokens
-	 * from the view-layer update.
-	 */
+	// SELECT-then-UPDATE: SQLite RETURNING can't cross to the view layer.
 	async demoteTurnEntries(runId, turn) {
 		const targets = await this.#db.get_turn_demotion_targets.all({
 			run_id: runId,
@@ -597,14 +544,7 @@ export default class Entries {
 		return targets;
 	}
 
-	/**
-	 * Demote every currently-visible entry in a run. Used by budget
-	 * postDispatch as the fallback when this-turn demotion finds nothing
-	 * and the packet still overflows — left-over promotions from prior
-	 * turns the model didn't demote themselves. Returns the affected
-	 * rows (path, tokens, turn) ordered oldest promotion first so the
-	 * error body can name them.
-	 */
+	// Budget postDispatch fallback: demote every visible entry in the run.
 	async demoteRunVisibleEntries(runId) {
 		const targets = await this.#db.get_run_visible_targets.all({
 			run_id: runId,
@@ -613,17 +553,12 @@ export default class Entries {
 		return targets;
 	}
 
-	/**
-	 * Run metadata lookup. Exposed here so plugins don't reach into
-	 * core.db for run-scoped lookups.
-	 */
+	// Plugin-facing run lookup; avoids reaching into core.db.
 	async getRun(runId) {
 		return this.#db.get_run_by_id.get({ id: runId });
 	}
 
-	/**
-	 * Turn-level usage stats write (telemetry). Same rationale as getRun.
-	 */
+	// Plugin-facing turn-stats write.
 	async updateTurnStats(stats) {
 		return this.#db.update_turn_stats.run(stats);
 	}

package/src/agent/ProjectAgent.js

@@ -87,10 +87,7 @@ export default class ProjectAgent {
 		return this.#agentLoop.inject(run, message, mode, options);
 	}
 
-	// Synchronously create (or fork) a run row and return the alias.
-	// Caller is expected to follow up with a kickoff (ask/act) that
-	// operates on the returned alias. Lets RPC respond with the real
-	// alias before the long-running loop starts.
+	// Create/fork the run row synchronously; caller follows up with ask/act.
 	async ensureRun(projectId, model, run, prompt, options = {}) {
 		return this.#agentLoop.ensureRun(projectId, model, run, prompt, options);
 	}
@@ -103,11 +100,7 @@ export default class ProjectAgent {
 		this.#agentLoop.abort(runId);
 	}
 
-	/**
-	 * Abort every in-flight run and wait for them to settle. Called
-	 * from the server's close path so the Node event loop isn't held
-	 * open by detached kickoff Promises after shutdown.
-	 */
+	// Abort all in-flight runs and drain so the event loop can exit.
 	async shutdown() {
 		await this.#agentLoop.abortAll();
 	}

package/src/agent/TurnExecutor.js

@@ -1,5 +1,6 @@
 import RummyContext from "../hooks/RummyContext.js";
 import { ContextExceededError } from "../llm/errors.js";
+import { PermissionError } from "./errors.js";
 import materializeContext from "./materializeContext.js";
 import XmlParser from "./XmlParser.js";
 
@@ -44,7 +45,6 @@ export default class TurnExecutor {
 			sequence: turn,
 		});
 
-		// Build RummyContext before turn.started so plugins can write entries
 		const rummy = new RummyContext(
 			{
 				tag: "turn",
@@ -78,7 +78,6 @@ export default class TurnExecutor {
 				loopPrompt,
 			},
 		);
-		// Plugins write prompt/instructions entries
 		await this.#hooks.turn.started.emit({
 			rummy,
 			mode,
@@ -89,12 +88,9 @@ export default class TurnExecutor {
 
 		await this.#hooks.processTurn(rummy);
 
-		// Project instructions://system through the instructions tool's projection
 		const systemPrompt =
 			await this.#hooks.instructions.resolveSystemPrompt(rummy);
 
-		// Materialize turn_context: VIEW rows projected through tools
-		const demoted = [];
 		const budgetCtx = {
 			runId: currentRunId,
 			loopId: currentLoopId,
@@ -102,7 +98,6 @@ export default class TurnExecutor {
 			systemPrompt,
 			mode,
 			toolSet,
-			demoted,
 			loopIteration,
 		};
 		const initial = await materializeContext({
@@ -118,13 +113,6 @@ export default class TurnExecutor {
 			rowCount: initial.rows.length,
 		});
 
-		await this.#hooks.run.progress.emit({
-			projectId,
-			run: currentAlias,
-			turn,
-			status: "thinking",
-		});
-
 		const budgetResult = await this.#hooks.budget.enforce({
 			contextSize,
 			messages: initial.messages,
@@ -158,15 +146,19 @@ export default class TurnExecutor {
 			turn,
 		});
 
-		// Call LLM. Transient-error retry + context-exceeded detection live
-		// in LlmProvider; context-exceeded surfaces as ContextExceededError.
 		await this.#hooks.llm.request.started.emit({ model: requestedModel, turn });
 		let rawResult;
 		try {
 			rawResult = await this.#llmProvider.completion(
 				filteredMessages,
 				requestedModel,
-				{ temperature: options?.temperature, signal },
+				{
+					temperature: options?.temperature,
+					signal,
+					// Per-run stable identifier for provider-side prompt caching
+					// (xAI prompt_cache_key, OpenAI prompt_cache_key, etc.).
+					runAlias: runRow?.alias || `run_${currentRunId}`,
+				},
 			);
 		} catch (err) {
 			if (err instanceof ContextExceededError) {
@@ -201,19 +193,8 @@ export default class TurnExecutor {
 			usage: result.usage,
 		});
 		const responseMessage = result.choices?.[0]?.message;
-		// A valid completion response always carries content (possibly
-		// empty) on the message; protect against that specific case so
-		// downstream parsers see a string.
 		const content = responseMessage?.content ? responseMessage.content : "";
 
-		await this.#hooks.run.progress.emit({
-			projectId,
-			run: currentAlias,
-			turn,
-			status: "processing",
-		});
-
-		// Parse and emit — plugins handle audit storage
 		const { commands, warnings, unparsed } = XmlParser.parse(content);
 		for (const w of warnings) {
 			await this.#hooks.error.log.emit({
@@ -225,7 +206,7 @@ export default class TurnExecutor {
 				status: 422,
 			});
 		}
-		if (commands.length === 0 && !!unparsed?.trim() && warnings.length === 0) {
+		if (commands.length === 0 && unparsed?.trim() && warnings.length === 0) {
 			await this.#hooks.error.log.emit({
 				store: this.#entries,
 				runId: currentRunId,
@@ -236,10 +217,7 @@ export default class TurnExecutor {
 			});
 		}
 
-		// Merge reasoning contributions from subscribers (think plugin's
-		// <think> tag, other plugin reasoning sources). Filter starts with
-		// the API-provided reasoning_content and layers on each plugin's
-		// contribution.
+		// Layer plugin reasoning contributions onto the API-provided seed.
 		if (responseMessage) {
 			const seed = responseMessage.reasoning_content
 				? responseMessage.reasoning_content
@@ -264,7 +242,7 @@ export default class TurnExecutor {
 			userMsg: userMsg?.content,
 		});
 
-		// --- PHASE 1: RECORD ---
+		// PHASE 1: RECORD
 		const recorded = [];
 		for (const cmd of commands) {
 			const entry = await this.#record(
@@ -277,14 +255,7 @@ export default class TurnExecutor {
 			if (entry) recorded.push(entry);
 		}
 
-		// --- PHASE 2: DISPATCH ---
-		// Sequential queue. Each tool completes before the next starts.
-		// On failure: abort remaining. On proposal: notify client, await
-		// resolution, continue.
-		// Narration text outside tags is fine when the turn also emitted
-		// at least one command — "OK", "Let me check:", reasoning prefixes
-		// are natural. Parse warnings and no-tags responses already emitted
-		// errors above; dispatch crashes and failed entries emit below.
+		// PHASE 2: DISPATCH — sequential; abort-after-failure; proposals notify-and-await.
 		let abortAfter = null;
 
 		for (const entry of recorded) {
@@ -309,6 +280,21 @@ export default class TurnExecutor {
 			try {
 				await this.#hooks.tools.dispatch(entry.scheme, entry, rummy);
 			} catch (dispatchErr) {
+				// PermissionError is the model attempting a documented-forbidden
+				// write (e.g. <set path="prompt://1"> with body). Surface as a
+				// soft 403 so the model can adjust on the next turn; do not
+				// abort sibling entries — the rest of the turn was valid.
+				if (dispatchErr instanceof PermissionError) {
+					await this.#hooks.error.log.emit({
+						store: this.#entries,
+						runId: currentRunId,
+						turn,
+						loopId: currentLoopId,
+						message: dispatchErr.message,
+						status: 403,
+					});
+					continue;
+				}
 				await this.#hooks.error.log.emit({
 					store: this.#entries,
 					runId: currentRunId,
@@ -323,11 +309,9 @@ export default class TurnExecutor {
 			await this.#hooks.tool.after.emit({ entry, rummy });
 			await this.#hooks.entry.created.emit(entry);
 
-			// Plugins (e.g. set) materialize pending proposals from the
-			// recorded entry — e.g. search/replace revisions → set:// 202.
+			// Plugins materialize pending proposals (e.g. set search/replace → 202).
 			await this.#hooks.proposal.prepare.emit({ rummy, recorded: [entry] });
 
-			// Check for any proposals created by this entry's dispatch
 			const proposed = await this.#entries.getUnresolved(currentRunId);
 			for (const p of proposed) {
 				await this.#hooks.proposal.pending.emit({
@@ -338,39 +322,18 @@ export default class TurnExecutor {
 				});
 				await this.#entries.waitForResolution(currentRunId, p.path);
 				const resolved = await this.#entries.getState(currentRunId, p.path);
-				if (resolved?.status >= 400) {
-					await this.#hooks.error.log.emit({
-						store: this.#entries,
-						runId: currentRunId,
-						turn,
-						loopId: currentLoopId,
-						message: `Proposal ${p.path} rejected: status ${resolved.status}.`,
-						status: resolved.status,
-					});
-					abortAfter = entry.scheme;
-				}
+				// Failure surfaces in the proposal entry itself; abort cascade
+				// triggers the trailing-action "Aborted — preceding <X>" body.
+				if (resolved?.status >= 400) abortAfter = entry.scheme;
 			}
 
 			if (!abortAfter) {
 				const entryPath = entry.resultPath || entry.path;
 				const row = await this.#entries.getState(currentRunId, entryPath);
-				if (row?.status >= 400) {
-					await this.#hooks.error.log.emit({
-						store: this.#entries,
-						runId: currentRunId,
-						turn,
-						loopId: currentLoopId,
-						message: `Entry ${entryPath} failed: status ${row.status}.`,
-						status: row.status,
-					});
-					abortAfter = entry.scheme;
-				}
+				if (row?.status >= 400) abortAfter = entry.scheme;
 			}
 		}
 
-		// Turn Demotion: budget plugin re-materializes end-of-turn context
-		// and demotes this turn's promoted entries on overflow. Overflow
-		// emits an error (status 413) via the unified error channel.
 		await this.#hooks.budget.postDispatch({
 			contextSize,
 			ctx: budgetCtx,
@@ -409,21 +372,14 @@ export default class TurnExecutor {
 		return turnResult;
 	}
 
-	/**
-	 * Record a parsed command as a known_entries row.
-	 * Returns the recorded entry descriptor, or null if rejected/skipped.
-	 */
+	// Record a parsed command; returns the entry descriptor or rejects on bad shapes.
 	async #record(runId, loopId, turn, mode, cmd) {
 		const scheme = cmd.name;
-		// Each tool's XmlParser shape surfaces exactly one of these
-		// three fields as its addressable target. Treat absent as empty
-		// so the length/control-char validation below catches bad shapes
-		// rather than letting an undefined slip through.
 		let rawTarget = "";
 		if (cmd.path) rawTarget = cmd.path;
 		else if (cmd.command) rawTarget = cmd.command;
 		else if (cmd.question) rawTarget = cmd.question;
-		// Reject paths that are likely reasoning bleed — too long or contain non-printing chars
+		// Reject likely reasoning bleed: oversize or control chars in target.
 		if (rawTarget.length > 512 || /\p{Cc}/u.test(rawTarget)) {
 			const rejectPath = await this.#entries.logPath(
 				runId,
@@ -454,18 +410,14 @@ export default class TurnExecutor {
 		const target = rawTarget;
 		const resultPath = await this.#entries.logPath(runId, turn, scheme, target);
 
-		// Pass parsed command fields through as attributes
 		const { name: _, ...attributes } = cmd;
 		if (cmd.path) attributes.path = target;
 
-		// Same per-shape resolution as rawTarget; the three sources are
-		// mutually exclusive per tool. Empty string when none set.
 		let body = "";
 		if (cmd.body) body = cmd.body;
 		else if (cmd.command) body = cmd.command;
 		else if (cmd.question) body = cmd.question;
 
-		// Filter: plugins can validate/transform before recording
 		const filtered = await this.#hooks.entry.recording.filter(
 			{
 				scheme,
@@ -478,7 +430,17 @@ export default class TurnExecutor {
 			{ store: this.#entries, runId, turn, loopId, mode },
 		);
 		if (filtered.state === "failed" || filtered.state === "cancelled") {
-			return filtered;
+			await this.#entries.set({
+				runId,
+				turn,
+				loopId,
+				path: filtered.path,
+				body: filtered.body,
+				state: filtered.state,
+				outcome: filtered.outcome,
+				attributes: filtered.attributes,
+			});
+			return { ...filtered, resultPath: filtered.path };
 		}
 
 		return {