@possumtech/rummy 2.0.1 → 2.2.0

package/src/plugins/instructions/instructions.js

@@ -1,194 +1,86 @@
-import { existsSync, readFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import Protocol from "./protocol.js";
 
+// Base system-prompt content (header + Core XML Command Grammar). The
+// `[%TOOLS%]` placeholder is substituted with the active-toolset tag list
+// at render time. Tool-specific docs no longer live here; they're a
+// separate `assembly.system` participant at priority 100.
 const baseInstructions = readFileSync(
-	new URL("./instructions.md", import.meta.url),
+	new URL("./instructions-system.md", import.meta.url),
 	"utf8",
 );
 
-// 1XY status encoding: X=current phase, Y=next phase. Y routes through
-// phaseForStatus to select next turn's <instructions>. Phases 4–9 are
-// reserved (status codes 1X4..1X9); add new phases by dropping in
-// `instructions_10N.md`. Absent files render no <instructions> block —
-// the model runs on base instructions only. This lets you route ahead
-// of writing the prose (e.g. an upcoming "ask lite" phase 9).
-const PHASES = [4, 5, 6, 7, 8, 9];
-const phaseInstructions = Object.fromEntries(
-	PHASES.flatMap((p) => {
-		const url = new URL(`./instructions_10${p}.md`, import.meta.url);
-		return existsSync(url) ? [[p, readFileSync(url, "utf8").trim()]] : [];
-	}),
-);
-const TURN_FROM_PATH = /^log:\/\/turn_(\d+)\/update\//;
-
-function phaseForStatus(status) {
-	if (status == null) return 4;
-	if (status === 200) return 7;
-	const last = status % 10;
-	return PHASES.includes(last) ? last : 4;
-}
+// Tight, non-modal reminder rendered LATE in the user message
+// (`assembly.user` priority 165, between unknowns at 150 and budget at
+// 175) so the rules sit adjacent to the action site — recency keeps the
+// per-turn discipline warm.
+const userInstructions = readFileSync(
+	new URL("./instructions-user.md", import.meta.url),
+	"utf8",
+).trim();
 
-// Scan an already-materialized row set for the most recent update
-// emission's status. Used by the assembly.user filter so the phase
-// instructions ride with the user message (dynamic, expected to
-// change every turn) instead of the system prompt (stable, cached).
-// Validation is upstream (update.js isValidStatus + 422 error log) so
-// we trust the status and route on it directly — a whitelist here
-// silently drops advertised completion codes whose contracts drift,
-// which is worse than a noisy fallback.
-function latestUpdateStatusFromRows(rows) {
-	let bestTurn = -1;
-	let bestStatus = null;
-	for (const r of rows) {
-		const m = TURN_FROM_PATH.exec(r.path);
-		if (!m) continue;
-		const turn = Number(m[1]);
-		const attrs =
-			typeof r.attributes === "string"
-				? JSON.parse(r.attributes)
-				: r.attributes;
-		const status = attrs?.status;
-		if (status == null) continue;
-		// Rejected updates are written for the model's audit trail but are
-		// not navigation events — phase router skips them so the model
-		// stays in the stage it was already in.
-		if (attrs?.rejected) continue;
-		if (turn > bestTurn || (turn === bestTurn && status > bestStatus)) {
-			bestTurn = turn;
-			bestStatus = status;
-		}
-	}
-	return bestStatus;
-}
+const TURN_FROM_PATH = /^log:\/\/turn_(\d+)\/update\//;
 
 export default class Instructions {
 	#core;
 
 	constructor(core) {
 		this.#core = core;
-		core.on("visible", this.full.bind(this));
-		core.on("turn.started", this.onTurnStarted.bind(this));
-		core.hooks.instructions.resolveSystemPrompt =
-			this.resolveSystemPrompt.bind(this);
-		core.hooks.instructions.validateNavigation =
-			this.validateNavigation.bind(this);
 		core.hooks.instructions.findLatestSummary =
 			this.findLatestSummary.bind(this);
-		// Dynamic phase instructions live in the user message (above
-		// <prompt>) so the system message stays cache-stable across turns.
-		// Priority 250 puts us between <log> (100), <unknowns> (200),
-		// and <prompt> (300).
-		core.filter("assembly.user", this.assembleInstructions.bind(this), 250);
+		// System message: priority chain. Base header + grammar at 50,
+		// joined per-tool docs at 100, persona at 150 (in persona.js).
+		core.filter("assembly.system", this.assembleSystemBase.bind(this), 50);
+		core.filter("assembly.system", this.assembleSystemToolDocs.bind(this), 100);
+		// User message: per-turn reminder block at the front of the user
+		// packet — sets discipline before the prompt.
+		core.filter("assembly.user", this.assembleInstructions.bind(this), 30);
 		new Protocol(core);
 	}
 
-	/**
-	 * Materialize the system prompt for a run: look up the
-	 * instructions://system entry, project it through the promoted view.
-	 * TurnExecutor calls this once per turn before context assembly.
-	 */
-	async resolveSystemPrompt(rummy) {
-		const { entries: store, runId, hooks } = rummy;
-		const entries = await store.getEntriesByPattern(
-			runId,
-			"instructions://system",
-			null,
-			{ includeAuditSchemes: true },
-		);
-		// The entry is always written by onTurnStarted before this runs.
-		const entry = entries[0];
-		const attributes = await store.getAttributes(
-			runId,
-			"instructions://system",
-		);
-		return hooks.tools.view("instructions", {
-			path: "instructions://system",
-			scheme: "instructions",
-			body: entry.body,
-			attributes,
-			visibility: "visible",
-			category: "system",
-		});
+	#activeToolSet(ctx) {
+		return ctx.toolSet
+			? new Set(ctx.toolSet)
+			: new Set(this.#core.hooks.tools.names);
 	}
 
-	/**
-	 * Reject illegal stage navigation. Two checks:
-	 *
-	 *   1. Forward skip — `nextPhase > currentPhase + 1`. Models advancing
-	 *      more than one stage at a time are jumping past required work.
-	 *      Returns and continuations (nextPhase ≤ currentPhase) always pass.
-	 *
-	 *   2. Deployment with prior prompts — any status landing the model in
-	 *      Deployment (phase 7) requires zero visible PRIOR prompts. State-
-	 *      property rule covering both entry (167) and continuation (177,
-	 *      200) — once in Deployment, the model still can't claim it with
-	 *      undemoted prior prompts. The current (latest) prompt always
-	 *      stays visible since Deployment must act on it.
-	 *
-	 * On rejection the caller marks the update entry rejected (so the
-	 * phase router skips it) and emits an error log; navigation rejections
-	 * count as normal strikes.
-	 */
-	async validateNavigation(status, rummy) {
-		const currentPhase = await this.#getCurrentPhase(rummy);
-		const nextPhase = phaseForStatus(status);
-		if (nextPhase > currentPhase + 1) {
-			return { ok: false, reason: "Illegal navigation attempt" };
-		}
-		if (nextPhase === 7) {
-			const visible = await this.#countVisiblePriorPrompts(rummy);
-			if (visible > 0) {
-				return {
-					ok: false,
-					reason: `Illegal navigation attempt: ${visible} visible prior prompts`,
-				};
-			}
-		}
-		return { ok: true };
+	// assembly.system @ 50 — header + Core XML Command Grammar with the
+	// advertised tool tag list substituted into the `[%TOOLS%]` slot.
+	assembleSystemBase(content, ctx) {
+		const activeTools = this.#activeToolSet(ctx);
+		const advertised = this.#core.hooks.tools.advertisedNames.filter((n) =>
+			activeTools.has(n),
+		);
+		const tools = advertised.map((n) => `<${n}/>`).join(", ");
+		return `${content}${baseInstructions.replace("[%TOOLS%]", tools)}`;
 	}
 
-	async #getCurrentPhase(rummy) {
-		// `**` (not `*`) for the slug position — update slugs are derived
-		// from the model's update body and can contain URL-encoded `/`
-		// characters (e.g. `known%3A//foo/bar` in a "ready for deployment"
-		// summary). Single `*` doesn't cross those embedded slashes and
-		// silently misses the prior turn's update.
-		const updates = await rummy.entries.getEntriesByPattern(
-			rummy.runId,
-			"log://*/update/**",
-			null,
+	// assembly.system @ 100 — per-tool docs joined in tool-registration
+	// order. Each tool plugin contributes its block via the
+	// `instructions.toolDocs` sub-filter (registry-style: filter
+	// participants mutate a docsMap keyed by tool name).
+	async assembleSystemToolDocs(content, ctx) {
+		const activeTools = this.#activeToolSet(ctx);
+		const toolDocs = await this.#core.hooks.instructions.toolDocs.filter(
+			{},
+			{ toolSet: activeTools },
 		);
-		let bestTurn = -1;
-		let bestStatus = null;
-		for (const e of updates) {
-			const m = TURN_FROM_PATH.exec(e.path);
-			if (!m) continue;
-			const turn = Number(m[1]);
-			if (turn >= rummy.sequence) continue;
-			const attrs =
-				typeof e.attributes === "string"
-					? JSON.parse(e.attributes)
-					: e.attributes;
-			if (attrs?.rejected) continue;
-			if (attrs?.status == null) continue;
-			if (turn > bestTurn) {
-				bestTurn = turn;
-				bestStatus = attrs.status;
-			}
-		}
-		return phaseForStatus(bestStatus);
+		const docsText = this.#core.hooks.tools.names
+			.filter((n) => activeTools.has(n))
+			.filter((key) => toolDocs[key])
+			.map((key) => toolDocs[key])
+			.join("\n\n");
+		if (!docsText) return content;
+		return `${content}\n\n${docsText}`;
+	}
+
+	// assembly.user @ 165 — per-turn reminder, same body every turn.
+	assembleInstructions(content, _ctx) {
+		return `${content}<instructions>\n${userInstructions}\n</instructions>\n`;
 	}
 
-	/**
-	 * Find the latest successful Deployment summary from a log-entry list.
-	 * Matches `log://turn_N/update/...` entries with status=200 (successful
-	 * Deployment completion) and returns the most recent. Used by
-	 * AgentLoop telemetry to surface the model's latest delivery.
-	 *
-	 * Lives here, not in AgentLoop, because "what counts as a summary" is
-	 * state-machine knowledge — phase 7's success status (200) is the
-	 * definition. AgentLoop just consumes the result.
-	 */
+	// Latest terminal update (status=200) — used by cli.js to print the
+	// run's final answer. State-machine knowledge lives here, not AgentLoop.
 	findLatestSummary(logEntries) {
 		return logEntries
 			.filter((e) => {
@@ -201,90 +93,4 @@ export default class Instructions {
 			})
 			.at(-1);
 	}
-
-	async #countVisiblePriorPrompts(rummy) {
-		const prompts = await rummy.entries.getEntriesByPattern(
-			rummy.runId,
-			"prompt://*",
-			null,
-		);
-		const visible = prompts.filter((p) => p.visibility === "visible");
-		if (visible.length === 0) return 0;
-		// Exclude the current (latest) prompt — that's what Deployment acts on.
-		// Demoting it would force the model to deliver on content it hid from
-		// itself. Only PRIOR prompts are subject to demote-before-Deployment.
-		let maxNum = -1;
-		for (const p of visible) {
-			const m = /^prompt:\/\/(\d+)$/.exec(p.path);
-			if (m && Number(m[1]) > maxNum) maxNum = Number(m[1]);
-		}
-		return visible.filter((p) => {
-			const m = /^prompt:\/\/(\d+)$/.exec(p.path);
-			return !m || Number(m[1]) !== maxNum;
-		}).length;
-	}
-
-	async onTurnStarted({ rummy }) {
-		const { entries: store, sequence: turn, runId } = rummy;
-		const runRow = await store.getRun(runId);
-		const toolSet = rummy.toolSet
-			? [...rummy.toolSet]
-			: this.#core.hooks.tools.names;
-		// instructions:// is an audit scheme (writable_by: ["system"]).
-		// No per-turn phase state on this entry — keeps the system
-		// prompt cache-stable across turns. Phase selection happens at
-		// assembly.user time from the current row set.
-		await store.set({
-			runId,
-			turn,
-			path: "instructions://system",
-			body: "",
-			state: "resolved",
-			writer: "system",
-			attributes: {
-				// runRow.persona is a nullable TEXT column; absent row is
-				// a system bug — let the null propagate if runRow exists.
-				persona: runRow.persona,
-				toolSet,
-			},
-		});
-	}
-
-	async full(entry) {
-		const attrs = entry.attributes;
-		const activeTools = attrs.toolSet
-			? new Set(attrs.toolSet)
-			: new Set(this.#core.hooks.tools.names);
-		const toolDocs = await this.#core.hooks.instructions.toolDocs.filter(
-			{},
-			{ toolSet: activeTools },
-		);
-		// Hidden tools are excluded at the registry level (see ToolRegistry).
-		const sorted = this.#core.hooks.tools.advertisedNames.filter((n) =>
-			activeTools.has(n),
-		);
-		const tools = sorted.map((n) => `<${n}/>`).join(", ");
-		const docsText = sorted
-			.filter((key) => toolDocs[key])
-			.map((key) => toolDocs[key])
-			.join("\n\n");
-		let prompt = baseInstructions
-			.replace("[%TOOLS%]", tools)
-			.replace("[%TOOLDOCS%]", docsText);
-		if (attrs.persona) prompt += `\n\n## Persona\n\n${attrs.persona}`;
-		return prompt;
-	}
-
-	// Renders the current phase's instructions as an <instructions>
-	// block in the user message. Runs at priority 250 — after <log>
-	// and <unknowns>, immediately before <prompt>. System prompt stays
-	// static so prompt caching keeps its prefix intact across turns.
-	// A routed phase without an instructions_10N.md file emits nothing —
-	// the model proceeds on base instructions alone.
-	assembleInstructions(content, ctx) {
-		const status = latestUpdateStatusFromRows(ctx.rows);
-		const step = phaseInstructions[phaseForStatus(status)];
-		if (!step) return content;
-		return `${content}<instructions>\n${step}\n</instructions>\n`;
-	}
 }

package/src/plugins/known/README.md

@@ -1,13 +1,17 @@
 # known {#known_plugin}
 
-Writes knowledge entries into the store at full visibility.
+Writes knowledge entries into the store at full visibility, and renders
+the project's data surface as the bifurcated `<summary>` /
+`<visible>` blocks at the top of the user message.
 
 ## Registration
 
 - **Tool**: `known`
 - **Category**: `data`
 - **Handler**: Upserts the entry body at the target path with status 200.
-- **Filter**: `assembly.system` at priority 100 — renders `<knowns>` section.
+- **Filters**:
+  - `assembly.user` priority 50 — renders `<summary>`.
+  - `assembly.user` priority 75 — renders `<visible>`.
 
 ## Projection
 
@@ -15,7 +19,22 @@ Shows `# known {path}` followed by the entry body.
 
 ## Assembly
 
-Filters turn_context rows where `category === "data"`. Renders all
-data entries (files, knowledge, skills, URLs) into the `<knowns>` section
-of the system message. Third-party plugins that register with
-`category: "data"` automatically appear here.
+Filters `ctx.rows` where `category === "data"`. Two separate blocks
+emit at the top of the user message in this order:
+
+- `<summary>` — each data entry whose visibility is `visible` or
+  `summarized`, rendered under its scheme tag with the plugin's
+  summary projection as body (truncated knowns, code symbols,
+  page abstracts — whatever the plugin's `summary()` hook produces).
+  Archived entries — including prompts — are filtered out uniformly
+  (no carve-out).
+- `<visible>` — each data entry whose visibility is `visible`,
+  rendered with the plugin's visible projection (full body) as the
+  tag body. A visible entry appears in *both* blocks: summary
+  projection up top, full body below.
+
+This split lets `<summary>` stay cache-stable across promote/demote
+operations — only `<visible>` mutates when the model promotes a
+summary or demotes a visible entry. Third-party plugins that register
+with `category: "data"` automatically appear in both blocks under
+their scheme tag.

package/src/plugins/known/known.js

@@ -1,5 +1,6 @@
 import { stateToStatus } from "../../agent/httpStatus.js";
 import { countTokens } from "../../agent/tokens.js";
+import { renderEntry, SUMMARY_MAX_CHARS } from "../helpers.js";
 
 const MAX_ENTRY_TOKENS = Number(process.env.RUMMY_MAX_ENTRY_TOKENS);
 
@@ -8,14 +9,15 @@ export default class Known {
 
 	constructor(core) {
 		this.#core = core;
+		core.ensureTool();
 		core.registerScheme({ category: "data" });
 		core.on("handler", this.handler.bind(this));
 		core.on("visible", this.full.bind(this));
 		core.on("summarized", this.summary.bind(this));
-		core.filter("assembly.system", this.assembleContext.bind(this), 100);
-		// <known> is internal — written via <set path="known://...">. Hidden
-		// from all model-facing tool lists. Handler still dispatches if the
-		// model emits <known> directly out of habit.
+		core.filter("assembly.system", this.assembleSummarized.bind(this), 200);
+		core.filter("assembly.system", this.assembleVisible.bind(this), 250);
+		// Hidden from the advertised tool list — written via <set path="known://...">.
+		// The known:// scheme lifecycle is taught in instructions-user.md.
 		core.markHidden();
 	}
 
@@ -23,23 +25,20 @@ export default class Known {
 		const { entries: store, sequence: turn, runId, loopId } = rummy;
 		if (!entry.body) return;
 
-		// Size gate
 		const entryTokens = countTokens(entry.body);
 		if (entryTokens > MAX_ENTRY_TOKENS) {
-			const rejectPath = await store.slugPath(runId, "known", entry.body);
 			await store.set({
 				runId,
 				turn,
-				path: rejectPath,
+				loopId,
+				path: entry.resultPath,
 				body: `Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
 				state: "failed",
 				outcome: `overflow:${entryTokens}`,
-				loopId,
 			});
 			return;
 		}
 
-		// Resolve path: explicit or auto-generated slug
 		let knownPath = entry.attributes?.path;
 		if (knownPath && !knownPath.includes("://")) {
 			knownPath = `known://${knownPath}`;
@@ -49,13 +48,11 @@ export default class Known {
 				runId,
 				"known",
 				entry.body,
-				entry.attributes?.summary,
+				entry.attributes?.tags,
 			);
 		}
 
-		// Dedup: if path exists, update rather than duplicate. An empty
-		// new body means "preserve the existing entry's body" (e.g. the
-		// model is updating attributes only).
+		// Dedup: existing path → update; empty body preserves existing body.
 		const existing = await store.getEntriesByPattern(runId, knownPath, null);
 		if (existing.length > 0) {
 			const nextBody = entry.body === "" ? existing[0].body : entry.body;
@@ -86,32 +83,43 @@ export default class Known {
 		return entry.body;
 	}
 
-	// Summarized knowns keep the first 500 characters so the model
-	// doesn't lose the plot when budget auto-demotion kicks in on its
-	// own work. Anything larger gets capped so a pathologically big
-	// known doesn't saturate the packet at summarized visibility
-	// either. Matches the pattern on `<prompt>` summarized view.
+	// Summarized: first SUMMARY_MAX_CHARS of the body. The model already
+	// knows summarized data is approximate (taught in instructions), so
+	// we don't owe it a "[truncated]" marker that would push the body
+	// past the contract floor.
 	summary(entry) {
 		if (!entry.body) return "";
-		if (entry.body.length <= 500) return entry.body;
-		return `${entry.body.slice(0, 500)}\n[truncated — promote to see the full body]`;
+		return entry.body.slice(0, SUMMARY_MAX_CHARS);
+	}
+
+	// Identity-keyed summary lines: every data entry the run is tracking
+	// at visibility=visible or visibility=summarized.
+	async assembleSummarized(content, ctx) {
+		const entries = ctx.rows.filter(
+			(r) =>
+				r.category === "data" &&
+				(r.visibility === "visible" || r.visibility === "summarized"),
+		);
+		if (entries.length === 0) return content;
+		const lines = entries.map((e) =>
+			renderContextTag(e, e.sBody != null ? e.sBody : e.body),
+		);
+		return `${content}<summary>\n${lines.join("\n")}\n</summary>\n`;
 	}
 
-	async assembleContext(content, ctx) {
-		const entries = ctx.rows.filter((r) => r.category === "data");
+	async assembleVisible(content, ctx) {
+		const entries = ctx.rows.filter(
+			(r) => r.category === "data" && r.visibility === "visible",
+		);
 		if (entries.length === 0) return content;
-		const demotedSet = new Set(ctx.demoted);
-		const lines = entries.map((e) => renderContextTag(e, demotedSet));
-		return `${content}\n\n<context>\n${lines.join("\n")}\n</context>`;
+		const lines = entries.map((e) =>
+			renderContextTag(e, e.vBody != null ? e.vBody : e.body),
+		);
+		return `${content}<visible>\n${lines.join("\n")}\n</visible>\n`;
 	}
 }
 
-function renderContextTag(entry, demotedSet) {
-	// schemeOf() returns NULL / "" for bare file paths; translate for the tag.
-	const tag = entry.scheme ? entry.scheme : "file";
-	const turn = entry.source_turn ? ` turn="${entry.source_turn}"` : "";
-	const tokens = entry.aTokens != null ? ` tokens="${entry.aTokens}"` : "";
-	const lines = entry.vLines != null ? ` lines="${entry.vLines}"` : "";
+function renderContextTag(entry, projectedBody) {
 	const attrs =
 		typeof entry.attributes === "string"
 			? JSON.parse(entry.attributes)
@@ -122,28 +130,16 @@ function renderContextTag(entry, demotedSet) {
 			: entry.state
 				? stateToStatus(entry.state, entry.outcome)
 				: null;
-	const status =
-		statusValue != null && statusValue !== 200
-			? ` status="${statusValue}"`
-			: "";
-	const stateAttr =
-		entry.state && entry.state !== "resolved" ? ` state="${entry.state}"` : "";
-	const outcomeAttr = entry.outcome ? ` outcome="${entry.outcome}"` : "";
-	const visibility = entry.visibility
-		? ` visibility="${entry.visibility}"`
-		: "";
-	const flag = demotedSet?.has(entry.path) ? " demoted" : "";
-	// Always render summary attribute on knowns — empty value hints the model
-	// it forgot to add searchable keywords.
-	const summaryText =
-		typeof attrs?.summary === "string"
-			? attrs.summary.replace(/"/g, "'").slice(0, 80)
-			: "";
-	const summary = ` summary="${summaryText}"`;
-
-	const attrStr = `${turn}${status}${stateAttr}${outcomeAttr}${summary}${visibility}${tokens}${lines}${flag}`;
-	if (entry.body) {
-		return `<${tag} path="${entry.path}"${attrStr}>${entry.body}</${tag}>`;
+	const meta = {};
+	if (entry.source_turn) meta.turn = entry.source_turn;
+	if (statusValue != null && statusValue !== 200) meta.status = statusValue;
+	if (entry.state && entry.state !== "resolved") meta.state = entry.state;
+	if (entry.outcome) meta.outcome = entry.outcome;
+	if (typeof attrs?.tags === "string") {
+		meta.tags = attrs.tags.slice(0, 80);
 	}
-	return `<${tag} path="${entry.path}"${attrStr}/>`;
+	if (entry.visibility === "archived") meta.visibility = "archived";
+	if (entry.aTokens != null) meta.tokens = entry.aTokens;
+	if (entry.vLines != null) meta.lines = entry.vLines;
+	return renderEntry(entry.path, meta, projectedBody);
 }

package/src/plugins/log/README.md

@@ -29,7 +29,7 @@ size. Resolution:
   own body tokens.
 - `sh` and `env` own multiple streaming channels (`sh://turn_N/{slug}_N`)
   — no single target to point at. `tokens=` is omitted; the channels
-  render their own tokens in `<context>`.
+  render their own tokens in `<visible>`.
 
 ## Behavior
 
@@ -43,6 +43,6 @@ Log entries (`log://turn_N/{action}/{slug}`) are audit records —
 summary, exit status, references to where the data lives — and never
 carry the payload itself. Payload for streaming actions lives under the
 producer's own scheme (`sh://`, `env://`, future `search://`, etc.) at
-`category=data`, and is rendered inside `<context>` by the known
+`category=data`, and is rendered inside `<visible>` by the known
 plugin. Scheme determines category; data and logging never share a
 scheme. See [scheme_category_split](#scheme_category_split).