@possumtech/rummy 0.4.0 → 2.0.0

package/src/plugins/instructions/instructions.js

@@ -1,28 +1,128 @@
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
+import Protocol from "./protocol.js";
 
-const preamble = readFileSync(
-	new URL("./preamble.md", import.meta.url),
+const baseInstructions = readFileSync(
+	new URL("./instructions.md", import.meta.url),
 	"utf8",
 );
 
+// 1XY status encoding: X=current phase, Y=next phase. Y routes through
+// phaseForStatus to select next turn's <instructions>. Phases 4–9 are
+// reserved (status codes 1X4..1X9); add new phases by dropping in
+// `instructions_10N.md`. Absent files render no <instructions> block —
+// the model runs on base instructions only. This lets you route ahead
+// of writing the prose (e.g. an upcoming "ask lite" phase 9).
+const PHASES = [4, 5, 6, 7, 8, 9];
+const phaseInstructions = Object.fromEntries(
+	PHASES.flatMap((p) => {
+		const url = new URL(`./instructions_10${p}.md`, import.meta.url);
+		return existsSync(url) ? [[p, readFileSync(url, "utf8").trim()]] : [];
+	}),
+);
+const TURN_FROM_PATH = /^log:\/\/turn_(\d+)\/update\//;
+
+function phaseForStatus(status) {
+	if (status == null) return 4;
+	if (status === 200) return 8;
+	const last = status % 10;
+	return PHASES.includes(last) ? last : 4;
+}
+
+// Scan an already-materialized row set for the most recent update
+// emission's status. Used by the assembly.user filter so the phase
+// instructions ride with the user message (dynamic, expected to
+// change every turn) instead of the system prompt (stable, cached).
+// Validation is upstream (update.js isValidStatus + 422 error log) so
+// we trust the status and route on it directly — a whitelist here
+// silently drops advertised completion codes whose contracts drift,
+// which is worse than a noisy fallback.
+function latestUpdateStatusFromRows(rows) {
+	let bestTurn = -1;
+	let bestStatus = null;
+	for (const r of rows) {
+		const m = TURN_FROM_PATH.exec(r.path);
+		if (!m) continue;
+		const turn = Number(m[1]);
+		const attrs =
+			typeof r.attributes === "string"
+				? JSON.parse(r.attributes)
+				: r.attributes;
+		const status = attrs?.status;
+		if (status == null) continue;
+		if (turn > bestTurn || (turn === bestTurn && status > bestStatus)) {
+			bestTurn = turn;
+			bestStatus = status;
+		}
+	}
+	return bestStatus;
+}
+
 export default class Instructions {
 	#core;
 
 	constructor(core) {
 		this.#core = core;
-		core.on("full", this.full.bind(this));
+		core.on("visible", this.full.bind(this));
 		core.on("turn.started", this.onTurnStarted.bind(this));
+		core.hooks.instructions.resolveSystemPrompt =
+			this.resolveSystemPrompt.bind(this);
+		// Dynamic phase instructions live in the user message (above
+		// <prompt>) so the system message stays cache-stable across turns.
+		// Priority 250 puts us between <log> (100), <unknowns> (200),
+		// and <prompt> (300).
+		core.filter("assembly.user", this.assembleInstructions.bind(this), 250);
+		new Protocol(core);
+	}
+
+	/**
+	 * Materialize the system prompt for a run: look up the
+	 * instructions://system entry, project it through the promoted view.
+	 * TurnExecutor calls this once per turn before context assembly.
+	 */
+	async resolveSystemPrompt(rummy) {
+		const { entries: store, runId, hooks } = rummy;
+		const entries = await store.getEntriesByPattern(
+			runId,
+			"instructions://system",
+			null,
+		);
+		// The entry is always written by onTurnStarted before this runs.
+		const entry = entries[0];
+		const attributes = await store.getAttributes(
+			runId,
+			"instructions://system",
+		);
+		return hooks.tools.view("instructions", {
+			path: "instructions://system",
+			scheme: "instructions",
+			body: entry.body,
+			attributes,
+			visibility: "visible",
+			category: "system",
+		});
 	}
 
 	async onTurnStarted({ rummy }) {
 		const { entries: store, sequence: turn, runId } = rummy;
-		const runRow = await rummy.db.get_run_by_id.get({ id: runId });
+		const runRow = await store.getRun(runId);
 		const toolSet = rummy.toolSet
 			? [...rummy.toolSet]
 			: this.#core.hooks.tools.names;
-		await store.upsert(runId, turn, "instructions://system", "", 200, {
+		// instructions:// is an audit scheme (writable_by: ["system"]).
+		// No per-turn phase state on this entry — keeps the system
+		// prompt cache-stable across turns. Phase selection happens at
+		// assembly.user time from the current row set.
+		await store.set({
+			runId,
+			turn,
+			path: "instructions://system",
+			body: "",
+			state: "resolved",
+			writer: "system",
 			attributes: {
-				persona: runRow?.persona || null,
+				// runRow.persona is a nullable TEXT column; absent row is
+				// a system bug — let the null propagate if runRow exists.
+				persona: runRow.persona,
 				toolSet,
 			},
 		});
@@ -33,22 +133,36 @@ export default class Instructions {
 		const activeTools = attrs.toolSet
 			? new Set(attrs.toolSet)
 			: new Set(this.#core.hooks.tools.names);
-		const sorted = this.#core.hooks.tools.names.filter((n) =>
-			activeTools.has(n),
-		);
-		const tools = sorted.join(", ");
 		const toolDocs = await this.#core.hooks.instructions.toolDocs.filter(
 			{},
 			{ toolSet: activeTools },
 		);
+		// Hidden tools are excluded at the registry level (see ToolRegistry).
+		const sorted = this.#core.hooks.tools.advertisedNames.filter((n) =>
+			activeTools.has(n),
+		);
+		const tools = sorted.map((n) => `<${n}/>`).join(", ");
 		const docsText = sorted
 			.filter((key) => toolDocs[key])
 			.map((key) => toolDocs[key])
 			.join("\n\n");
-		let prompt = preamble
+		let prompt = baseInstructions
 			.replace("[%TOOLS%]", tools)
 			.replace("[%TOOLDOCS%]", docsText);
 		if (attrs.persona) prompt += `\n\n## Persona\n\n${attrs.persona}`;
 		return prompt;
 	}
+
+	// Renders the current phase's instructions as an <instructions>
+	// block in the user message. Runs at priority 250 — after <log>
+	// and <unknowns>, immediately before <prompt>. System prompt stays
+	// static so prompt caching keeps its prefix intact across turns.
+	// A routed phase without an instructions_10N.md file emits nothing —
+	// the model proceeds on base instructions alone.
+	assembleInstructions(content, ctx) {
+		const status = latestUpdateStatusFromRows(ctx.rows);
+		const step = phaseInstructions[phaseForStatus(status)];
+		if (!step) return content;
+		return `${content}<instructions>\n${step}\n</instructions>\n`;
+	}
 }

package/src/plugins/instructions/instructions.md

@@ -0,0 +1,25 @@
+XML Commands Available: [%TOOLS%]
+
+# FCRM Engine
+
+You are a Folksonomic Context Relevance Maximization (FCRM) engine with a **Primary Directive** of Context Relevance Maximization.
+* Definition Stage: Register everything unknown about the prompt request.
+* Discovery Stage: Discover, Distill, and Demote source entries to resolve unknowns into knowns.
+* Deployment Stage: Act on the prompt.
+
+Warning: YOU MUST NOT allow the `tokens="N"` sum of irrelevant source entries or log events to exceed `tokensFree` budget.
+
+Tip: The `tokens="N"` shows how much context memory is consumed if "visible". Entries only consume tokens when at "visible" visibility.
+Tip: The "summarized" and "archived" entries and log events use no context memory (`tokensFree="N"`).
+Tip: You can use <get path="..." preview/> to preview the potential `tokens="N"` budget impact of bulk operations.
+Tip: You can use <get path="..." line="X" limit="Y"/> to read subsets of entries that would exceed your `tokensFree` budget.
+Tip: Log items are demotable just like context entries. Demote their visibility to "summarized" or "archived" as needed.
+Tip: Entries and log events that have been archived are fully hidden (no memory used, no summary), but can be retrieved later by path.
+
+# Commands
+
+Warning: YOU MUST NOT use shell commands for project file operations. Project files are entries that require XML Command operations.
+Example: <set path="src/file.txt">new file content</set>
+Example: <get path="src/*.txt" preview/>
+
+[%TOOLDOCS%]

package/src/plugins/instructions/instructions_104.md

@@ -0,0 +1,7 @@
+Definition Stage: YOU MUST ONLY define relevant information you do not know in this stage.
+
+YOU MUST create topical, taxonomized, and tagged unknown:// entries for missing information you need to discover.
+Example: <set path="unknown://countries/france/capital" summary="countries,france,capital,geography,trivia">What is the capital of France?</set>
+
+Definition Stage Continuation: <update status="144">identifying more unknowns</update>
+Definition Stage Completion: <update status="145">unknowns identified</update>

package/src/plugins/instructions/instructions_105.md

@@ -0,0 +1,46 @@
+# Discovery Stage
+
+YOU MUST ONLY perform discovery actions (Discover -> Distill -> Demote) during the Discovery Stage.
+YOU MUST discover and get source entries with information relevant to the unknown:// entries.
+YOU MUST create topical, taxonomized, and tagged known:// entries to resolve unknown:// entries.
+YOU MUST include at least 1 link to a relevant unknown and at least 1 link to a relevant source entry in every known:// entry.
+YOU MUST demote source entries to "summarized" after extracting and decomposing their relevant information into known:// entries.
+YOU MUST demote the unknown:// entries to "summarized" after they are referenced or resolved by known:// entries.
+YOU MUST demote all irrelevant source entries and log events to maximize FCRM.
+Tip: Source entry "summarized" information is not reliable. Only place "visible" source entry information in known:// entries.
+Tip: A "relevant" source entry that has been successfully distilled into known:// entries is no longer relevant.
+Tip: Discover, Distill, and Demote per source entry, not globally, to maximize FCRM.
+
+## Discovery Lifecycle: Promoting a source entry, creating a known entry, demoting the source entry, then archiving the resolved unknown
+
+### Discover
+
+<set path="trivia/capitals.csv" visibility="visible"/>
+
+### Distill
+<set path="known://countries/france/capital" summary="countries,france,capital,geography,trivia">
+    # Capital of France
+    The capital of France is Paris.
+
+    { ... }
+
+    # References
+    [unknown resolving](unknown://countries/france/capital)
+    [source entry](trivia/capitals.csv)
+</set>
+
+### Demote
+
+<set path="trivia/capitals.csv" visibility="summarized"/>
+<set path="unknown://countries/france/capital" visibility="summarized"/>
+<set path="unknown://countries/poland/capital" summary="REJECTED: Irrelevant" visibility="summarized"/>
+<set path="https://en.wikipedia.org/wiki/Paris,_Texas" summary="REJECTED: Wrong Paris" visibility="summarized"/>
+<set path="log://turn_1/set/*" visibility="archived"/>
+<set path="log://turn_1/get/trivia/*" visibility="archived"/>
+<set path="log://turn_2/get/capital%20of%20france" visibility="archived"/>
+
+## Turn Termination (CHOOSE ONLY ONE):
+
+Definition Stage Return: <update status="154">returning to definition stage</update>
+Discovery Stage Continuation: <update status="155">referencing and resolving more unknowns</update>
+Discovery Stage Completion: <update status="158">all unknowns (if any) referenced or resolved by known entries</update>

package/src/plugins/instructions/instructions_108.md

@@ -0,0 +1,8 @@
+YOU MUST act on the prompt.
+
+Turn Termination (CHOOSE ONLY ONE):
+
+Definition Stage Return: <update status="184">returning to definition stage</update>
+Discovery Stage Return: <update status="185">returning to discovery stage</update>
+Deployment Stage Continuation: <update status="188">performing more actions</update>
+Deployment Stage Completion: <update status="200">summary of actions performed, or direct answer</update>

package/src/plugins/instructions/protocol.js

@@ -0,0 +1,12 @@
+export default class Protocol {
+	#core;
+
+	constructor(core) {
+		this.#core = core;
+		core.filter("entry.recording", this.#onRecording.bind(this), 1);
+	}
+
+	async #onRecording(entry, _ctx) {
+		return entry;
+	}
+}

package/src/plugins/known/README.md

@@ -1,6 +1,6 @@
-# known
+# known {#known_plugin}
 
-Writes knowledge entries into the store at full fidelity.
+Writes knowledge entries into the store at full visibility.
 
 ## Registration
 

package/src/plugins/known/known.js

@@ -1,7 +1,7 @@
+import { stateToStatus } from "../../agent/httpStatus.js";
 import { countTokens } from "../../agent/tokens.js";
-import docs from "./knownDoc.js";
 
-const MAX_ENTRY_TOKENS = Number(process.env.RUMMY_MAX_ENTRY_TOKENS) || 512;
+const MAX_ENTRY_TOKENS = Number(process.env.RUMMY_MAX_ENTRY_TOKENS);
 
 export default class Known {
 	#core;
@@ -10,13 +10,13 @@ export default class Known {
 		this.#core = core;
 		core.registerScheme({ category: "data" });
 		core.on("handler", this.handler.bind(this));
-		core.on("full", this.full.bind(this));
-		core.on("summary", this.summary.bind(this));
-		core.filter("assembly.system", this.assembleKnown.bind(this), 100);
-		core.filter("instructions.toolDocs", async (docsMap) => {
-			docsMap.known = docs;
-			return docsMap;
-		});
+		core.on("visible", this.full.bind(this));
+		core.on("summarized", this.summary.bind(this));
+		core.filter("assembly.system", this.assembleContext.bind(this), 100);
+		// <known> is internal — written via <set path="known://...">. Hidden
+		// from all model-facing tool lists. Handler still dispatches if the
+		// model emits <known> directly out of habit.
+		core.markHidden();
 	}
 
 	async handler(entry, rummy) {
@@ -27,19 +27,20 @@ export default class Known {
 		const entryTokens = countTokens(entry.body);
 		if (entryTokens > MAX_ENTRY_TOKENS) {
 			const rejectPath = await store.slugPath(runId, "known", entry.body);
-			await store.upsert(
+			await store.set({
 				runId,
 				turn,
-				rejectPath,
-				`Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
-				413,
-				{ loopId },
-			);
+				path: rejectPath,
+				body: `Entry too large (${entryTokens} tokens, max ${MAX_ENTRY_TOKENS}). Sort the information, ideas, or plans carefully into multiple entries.`,
+				state: "failed",
+				outcome: `overflow:${entryTokens}`,
+				loopId,
+			});
 			return;
 		}
 
 		// Resolve path: explicit or auto-generated slug
-		let knownPath = entry.attributes?.path || null;
+		let knownPath = entry.attributes?.path;
 		if (knownPath && !knownPath.includes("://")) {
 			knownPath = `known://${knownPath}`;
 		}
@@ -52,65 +53,96 @@ export default class Known {
 			);
 		}
 
-		// Dedup: if path exists, update rather than duplicate
+		// Dedup: if path exists, update rather than duplicate. An empty
+		// new body means "preserve the existing entry's body" (e.g. the
+		// model is updating attributes only).
 		const existing = await store.getEntriesByPattern(runId, knownPath, null);
 		if (existing.length > 0) {
-			await store.upsert(
+			const nextBody = entry.body === "" ? existing[0].body : entry.body;
+			await store.set({
 				runId,
 				turn,
-				existing[0].path,
-				entry.body || existing[0].body,
-				200,
-				{ attributes: entry.attributes, loopId },
-			);
+				path: existing[0].path,
+				body: nextBody,
+				state: "resolved",
+				attributes: entry.attributes,
+				loopId,
+			});
 			return;
 		}
 
-		await store.upsert(runId, turn, knownPath, entry.body, 200, {
+		await store.set({
+			runId,
+			turn,
+			path: knownPath,
+			body: entry.body,
+			state: "resolved",
 			attributes: entry.attributes,
 			loopId,
 		});
 	}
 
 	full(entry) {
-		return `# known ${entry.path}\n${entry.body}`;
+		return entry.body;
 	}
 
+	// Summarized knowns keep the first 500 characters so the model
+	// doesn't lose the plot when budget auto-demotion kicks in on its
+	// own work. Anything larger gets capped so a pathologically big
+	// known doesn't saturate the packet at summarized visibility
+	// either. Matches the pattern on `<prompt>` summarized view.
 	summary(entry) {
-		return this.full(entry);
+		if (!entry.body) return "";
+		if (entry.body.length <= 500) return entry.body;
+		return `${entry.body.slice(0, 500)}\n[truncated — promote to see the full body]`;
 	}
 
-	async assembleKnown(content, ctx) {
+	async assembleContext(content, ctx) {
 		const entries = ctx.rows.filter((r) => r.category === "data");
 		if (entries.length === 0) return content;
-
-		// Rows arrive pre-sorted by SQL: summary → full, then by recency
-		const demotedSet = new Set(ctx.demoted || []);
-		const lines = entries.map((e) => renderKnownTag(e, demotedSet));
-		return `${content}\n\n<knowns>\n${lines.join("\n")}\n</knowns>`;
+		const demotedSet = new Set(ctx.demoted);
+		const lines = entries.map((e) => renderContextTag(e, demotedSet));
+		return `${content}\n\n<context>\n${lines.join("\n")}\n</context>`;
 	}
 }
 
-function renderKnownTag(entry, demotedSet) {
-	const tag = entry.scheme || "file";
+function renderContextTag(entry, demotedSet) {
+	// schemeOf() returns NULL / "" for bare file paths; translate for the tag.
+	const tag = entry.scheme ? entry.scheme : "file";
 	const turn = entry.source_turn ? ` turn="${entry.source_turn}"` : "";
-	const tokens = entry.tokens ? ` tokens="${entry.tokens}"` : "";
-	const status = entry.status ? ` status="${entry.status}"` : "";
-	const fidelity = entry.fidelity ? ` fidelity="${entry.fidelity}"` : "";
-	const flag = demotedSet?.has(entry.path) ? " demoted" : "";
-
+	const tokens = entry.aTokens != null ? ` tokens="${entry.aTokens}"` : "";
 	const attrs =
 		typeof entry.attributes === "string"
 			? JSON.parse(entry.attributes)
 			: entry.attributes;
-	const summary =
+	const statusValue =
+		attrs?.status != null
+			? attrs.status
+			: entry.state
+				? stateToStatus(entry.state, entry.outcome)
+				: null;
+	const status =
+		statusValue != null && statusValue !== 200
+			? ` status="${statusValue}"`
+			: "";
+	const stateAttr =
+		entry.state && entry.state !== "resolved" ? ` state="${entry.state}"` : "";
+	const outcomeAttr = entry.outcome ? ` outcome="${entry.outcome}"` : "";
+	const visibility = entry.visibility
+		? ` visibility="${entry.visibility}"`
+		: "";
+	const flag = demotedSet?.has(entry.path) ? " demoted" : "";
+	// Always render summary attribute on knowns — empty value hints the model
+	// it forgot to add searchable keywords.
+	const summaryText =
 		typeof attrs?.summary === "string"
-			? ` summary="${attrs.summary.replace(/"/g, "'").slice(0, 80)}"`
+			? attrs.summary.replace(/"/g, "'").slice(0, 80)
 			: "";
+	const summary = ` summary="${summaryText}"`;
 
-	if (entry.fidelity === "archive") return "";
-	if (entry.fidelity === "summary") {
-		return `<${tag} path="${entry.path}"${turn}${status}${summary}${fidelity}${tokens}${flag}/>`;
+	const attrStr = `${turn}${status}${stateAttr}${outcomeAttr}${summary}${visibility}${tokens}${flag}`;
+	if (entry.body) {
+		return `<${tag} path="${entry.path}"${attrStr}>${entry.body}</${tag}>`;
 	}
-	return `<${tag} path="${entry.path}"${turn}${status}${summary}${fidelity}${tokens}${flag}>${entry.body}</${tag}>`;
+	return `<${tag} path="${entry.path}"${attrStr}/>`;
 }

package/src/plugins/known/knownDoc.js

@@ -1,30 +1,3 @@
-// Tool doc for <known/>. Each entry: [text, rationale].
-// Text goes to the model. Rationale stays in source.
-// Changing ANY line requires reading ALL rationales first.
-const LINES = [
-	[
-		'## <known path="known://topic/subtopic" summary="keyword,keyword,keyword">[specific facts, decisions, or plans]</known> - Sort and save what you learn for later recall',
-	],
-	[
-		'Example: <known summary="hedberg,comedian,death,2005">Mitch Hedberg died on March 30, 2005</known>',
-		"Summary-first pattern: comma-separated keywords, path auto-generated.",
-	],
-	[
-		'Example: <known path="known://people/rumsfeld" summary="defense,secretary,born,1932">Donald Rumsfeld was born in 1932 and served as Secretary of Defense</known>',
-		"Explicit path form: slashed path=category/key, summary=keywords.",
-	],
-	[
-		'* Recall with <get path="known://people/*">keyword</get>',
-		"Cross-tool lifecycle: glob by category, filter by keyword.",
-	],
-	[
-		"* YOU SHOULD write `summary` keywords, you can search for them later",
-		"Motivates summary writing through self-interest.",
-	],
-	[
-		"* YOU MUST sort and save all new facts, decisions, and plans in their own <known> entries",
-		"Critical behavioral constraint. 'new' prevents re-saving known facts.",
-	],
-];
+import { loadDoc } from "../helpers.js";
 
-export default LINES.map(([text]) => text).join("\n");
+export default loadDoc(import.meta.url, "knownDoc.md");

package/src/plugins/known/knownDoc.md

@@ -0,0 +1,8 @@
+## <set path="known://topic/subtopic" summary="keyword,keyword,keyword">[specific facts, decisions, or plans]</set> - Sort and save what you learn for later recall
+<!-- Use <set> to write known entries (not <known>). Matches instructions examples. -->
+
+Example: <set path="known://people/rumsfeld" summary="defense,secretary,born,1932">Donald Rumsfeld was born in 1932 and served as Secretary of Defense</set>
+<!-- Explicit path form: slashed path=category/key, summary=keywords. -->
+
+* Recall with <get path="known://people/*">keyword</get>
+<!-- Cross-tool lifecycle: pattern by category, filter by keyword. -->

package/src/plugins/log/README.md

@@ -0,0 +1,48 @@
+# log {#log_plugin}
+
+Assembles the `<log>` block in the user message: every
+`category="logging"` entry across the entire run, rendered as XML tool
+tags in v_model_context sort order.
+
+## Registration
+
+- **Filter**: `assembly.user` (priority 100) — contributes the `<log>`
+  block to the user packet.
+
+## Rendering
+
+Each logging entry renders with its scheme as the tag name (`<get>`,
+`<set>`, `<search>`, `<rm>`, `<cp>`, `<mv>`, `<sh>`, `<env>`,
+`<update>`, `<ask_user>`, `<error>`, `<budget>`). Attributes:
+`path`, `turn`, `status`, `state`, `outcome`, `summary`, `visibility`,
+`tokens`.
+
+**`tokens=` invariant.** The value is always the full-visibility cost
+of the thing the tag represents — never the log entry's own stub body
+size. Resolution:
+
+- If the log entry has `attrs.path` referencing a data entry (`get`,
+  `set`, `mv`, `cp`): `tokens=` is that target's tokens. Promotes the
+  audit record into a cost-accurate signal the model can plan against.
+- If the action's log body itself IS the cost-bearing content
+  (`search`, `update`, `error`, `ask_user`): `tokens=` is the entry's
+  own body tokens.
+- `sh` and `env` own multiple streaming channels (`sh://turn_N/{slug}_N`)
+  — no single target to point at. `tokens=` is omitted; the channels
+  render their own tokens in `<context>`.
+
+## Behavior
+
+No loop-boundary split. The `turn` attribute on every entry carries
+when it happened; the model derives loop membership from the data if
+it matters. One chronological log from turn 1 to now.
+
+## Scheme invariant
+
+Log entries (`log://turn_N/{action}/{slug}`) are audit records —
+summary, exit status, references to where the data lives — and never
+carry the payload itself. Payload for streaming actions lives under the
+producer's own scheme (`sh://`, `env://`, future `search://`, etc.) at
+`category=data`, and is rendered inside `<context>` by the known
+plugin. Scheme determines category; data and logging never share a
+scheme. See [scheme_category_split](#scheme_category_split).