@possumtech/rummy 2.1.0 → 2.2.1

package/src/plugins/hedberg/hedberg.js

@@ -1,68 +1,25 @@
-import { parseEditContent } from "./edits.js";
-import HeuristicMatcher, { generatePatch } from "./matcher.js";
-import { parseJsonEdit } from "./normalize.js";
-import { hedmatch, hedsearch } from "./patterns.js";
-import { parseSed } from "./sed.js";
-
-// Stochastic→deterministic boundary; exposes pattern/edit utilities on core.hedberg. SPEC #hedberg.
-export default class Hedberg {
+// Hedberg plugin shim. The library lives at `src/lib/hedberg/`; this
+// shim's only job is to expose the same surface as `core.hooks.hedberg`
+// for external plugins (e.g. rummy.repo's FileScanner) that can't
+// reach into rummy/main's internals via direct import.
+//
+// Internal plugins should import from `src/lib/hedberg/` directly —
+// the hook namespace is for plugins shipped in separate packages.
+
+import Hedberg from "../../lib/hedberg/hedberg.js";
+import { generatePatch } from "../../lib/hedberg/matcher.js";
+import { hedmatch, hedsearch } from "../../lib/hedberg/patterns.js";
+
+export default class HedbergPlugin {
 	#core;
 
 	constructor(core) {
 		this.#core = core;
-
 		core.hooks.hedberg = {
 			match: hedmatch,
 			search: hedsearch,
 			replace: Hedberg.replace,
-			parseSed,
-			parseEdits: parseEditContent,
-			parseJsonEdit,
 			generatePatch,
 		};
 	}
-
-	// Order: sed regex → literal → heuristic fuzzy.
-	static replace(body, search, replacement, { sed = false, flags = "" } = {}) {
-		let patch = null;
-		let warning = null;
-		let error = null;
-		const searchText = search;
-		const replaceText = replacement;
-
-		if (sed) {
-			try {
-				const re = new RegExp(
-					searchText,
-					flags.includes("g") ? flags : `${flags}g`,
-				);
-				// Strip regex-meta escapes in replacement; String.replace only interprets `$`, not `\`.
-				const unescaped = replaceText.replace(/\\([[\](){}.*+?^$|\\])/g, "$1");
-				patch = body.replace(re, unescaped);
-				if (patch === body) patch = null;
-			} catch {
-				// Invalid regex — fall through to literal/heuristic interpretation
-			}
-		}
-
-		if (!patch && body.includes(searchText)) {
-			patch = body.replaceAll(searchText, replaceText);
-		}
-
-		if (!patch) {
-			const matched = HeuristicMatcher.matchAndPatch(
-				"",
-				body,
-				searchText,
-				replaceText,
-			);
-			patch = matched.newContent;
-			warning = matched.warning;
-			error = matched.error;
-		}
-
-		return { patch, searchText, replaceText, warning, error };
-	}
 }
-
-export { generatePatch };

package/src/plugins/helpers.js

@@ -2,6 +2,47 @@ import { readFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 
+// Hard system ceiling on the size of any summarized projection. Single
+// source of truth — every plugin's `summarized` view must produce output
+// ≤ this many characters; materializeContext's defensive cap fires when
+// a plugin breaks the contract. Change this number, everything downstream
+// stays consistent (no "450 here, 480 there, 500 over yonder" drift).
+export const SUMMARY_MAX_CHARS = 500;
+
+// Render a single entry as a heredoc-fenced block. Replaces the prior
+// per-plugin XML tag rendering (`<known path="..." ...>body</known>`).
+//
+// Why heredoc, not XML: the model emits XML for actions (`<set>`,
+// `<get>`, `<sh>`). When entries were ALSO XML, the inner tag could
+// look indistinguishable from a tool call — a file containing a
+// `<set>` example, an env streamed-stdout containing tool-shaped text,
+// or the model's own `<known>` body containing accidental tag-shaped
+// content could leak into the model's emit pattern. Heredoc fences
+// have zero training prior in tool-emission position, so they're
+// structurally separate from the action grammar.
+//
+// Format: `{json-meta} <<:::{path}\n{body}\n:::{path}`. The path is the
+// terminator — collision requires the body to literally contain its own
+// URI on a line by itself, which is vanishingly rare without active
+// malice. JSON metadata is sorted-key stringified for prefix-cache
+// stability (different field order = different bytes = cache miss).
+export function renderEntry(path, metadata, body) {
+	const meta = canonicalJson(metadata);
+	if (!body) {
+		return `${meta} <<:::${path}\n:::${path}`;
+	}
+	const trailingNewline = body.endsWith("\n") ? "" : "\n";
+	return `${meta} <<:::${path}\n${body}${trailingNewline}:::${path}`;
+}
+
+// JSON.stringify with sorted top-level keys for byte-stable output.
+function canonicalJson(obj) {
+	const keys = Object.keys(obj).sort();
+	const sorted = {};
+	for (const k of keys) sorted[k] = obj[k];
+	return JSON.stringify(sorted);
+}
+
 // Read sibling tooldoc .md; strips HTML comments (rationale stays out of the model packet).
 export function loadDoc(metaUrl, name) {
 	const dir = dirname(fileURLToPath(metaUrl));
@@ -18,11 +59,19 @@ export function logPathToDataBase(logPath) {
 	return `${m[2]}://turn_${m[1]}/${m[3]}`;
 }
 
-// env/sh stdout/stderr summary projection: header with line range + last
-// TAIL_LINES of body. The header tells the model exactly which slice is
-// shown so it can issue <get line="N" limit="M"/> for the rest without
-// re-running the command.
-export function streamSummary(label, entry, TAIL_LINES = 12) {
+// env/sh stdout/stderr summary projection: line-tail with a final hard
+// truncation to SUMMARY_MAX_CHARS. The line cap is the natural unit
+// when the program emits text; the final size cap is the contract floor
+// (see materializeContext) and protects against few-newline output like
+// terminal-control programs (cmatrix, htop) emitting one giant ANSI line.
+//
+// Output stays as a flat string (not a renderEntry block) because the
+// caller (log.assembleLog) wraps each log entry in renderEntry with its
+// own metadata; this is the BODY of that block. Effectively double
+// fencing — `<<:::log://turn_3/sh/foo` outer, then this header inside —
+// but that's correct: the outer fence labels "this is sh activity at
+// turn 3", and the body inside is the slice of stdout the model sees.
+export function streamSummary(label, entry, MAX_LINES = 20) {
 	if (!entry.body) return "";
 	const { body, attributes } = entry;
 	const command = attributes.command;
@@ -32,13 +81,18 @@ export function streamSummary(label, entry, TAIL_LINES = 12) {
 		? body.slice(0, -1).split("\n")
 		: body.split("\n");
 	const total = lines.length;
-	if (total <= TAIL_LINES) {
-		return `# ${label} ${command} (${channel}, ${total}L)\n${body}`;
-	}
-	const startLine = total - TAIL_LINES + 1;
-	const tail =
-		lines.slice(-TAIL_LINES).join("\n") + (trailingNewline ? "\n" : "");
-	return `# ${label} ${command} (${channel}, tail L${startLine}-${total}/${total}; <get line="1" limit="N"/> for head)\n${tail}`;
+	const lineTail =
+		total <= MAX_LINES
+			? body
+			: lines.slice(-MAX_LINES).join("\n") + (trailingNewline ? "\n" : "");
+
+	const header =
+		total <= MAX_LINES
+			? `# ${label} ${command} (${channel}, ${total}L)`
+			: `# ${label} ${command} (${channel}, lines ${total - MAX_LINES + 1} through ${total} of ${total}; <get line="1" limit="N"/> for head)`;
+
+	const out = `${header}\n${lineTail}`;
+	return out.length > SUMMARY_MAX_CHARS ? out.slice(0, SUMMARY_MAX_CHARS) : out;
 }
 
 // Pattern-result log entry shared by get/set/store/rm.

package/src/plugins/index.js

@@ -3,10 +3,9 @@ import { existsSync } from "node:fs";
 import { readdir, stat } from "node:fs/promises";
 import { basename, isAbsolute, join } from "node:path";
 import { pathToFileURL } from "node:url";
-import config from "../agent/config.js";
 import PluginContext from "../hooks/PluginContext.js";
 
-const { PLUGINS_LOAD_TIMEOUT } = config;
+const PLUGINS_LOAD_TIMEOUT = Number(process.env.RUMMY_PLUGINS_LOAD_TIMEOUT);
 
 let globalPrefix;
 function getGlobalPrefix() {

package/src/plugins/instructions/README.md

@@ -1,59 +1,56 @@
 # instructions {#instructions_plugin}
 
 Projects the model-facing instructions into the assembled packet.
-Cleanly split into a stable system-side base and a dynamic user-side
-phase directive so prompt caching holds across turns within a run.
+Split into a stable system-side base and a per-turn user-side
+imperative reminder so prompt caching holds across turns within a
+run.
 
 ## Registration
 
-- **View**: `full` — renders the `instructions.md` base (identity +
-  `[%TOOLS%]` + `[%TOOLDOCS%]` + optional persona) for the
-  `instructions://system` entry. Stable across turns.
-- **Event**: `turn.started` — writes `instructions://system` entry
-  with `{ persona, toolSet }` attributes.
-- **Filter**: `instructions.toolDocs` — gathers docs from all tool
-  plugins into a docsMap.
-- **Filter**: `assembly.user` (priority 250) — renders the current
-  phase's `instructions_10N.md` as `<instructions>` immediately
-  before `<prompt>`. Phase selected from the latest `<update status>`
-  emission in this turn's row set.
+- **Filter**: `assembly.system` (priority 50) — renders the system-
+  prompt header + Core XML Command Grammar from
+  `instructions-system.md` with `[%TOOLS%]` substituted to the
+  active-toolset tag list.
+- **Filter**: `assembly.system` (priority 100) — renders the joined
+  per-tool docs. Each tool plugin contributes its block via the
+  `instructions.toolDocs` sub-filter (registry-style: filter
+  participants mutate a docsMap keyed by tool name). Render order
+  follows tool-registration order.
+- **Filter**: `assembly.user` (priority 165) — renders
+  `instructions-user.md` as `<instructions>` late in the user
+  message, between `<unknowns>` (150) and `<budget>` (175). The
+  user message is a sandwich: `<prompt>` (30) leads for cache
+  stability, dynamic state fills the middle, then rules and
+  budget close out so the action site sees them with recency.
+- **Filter**: `instructions.toolDocs` — sub-filter the toolDocs
+  participant calls. Tool plugins (and skill) extend this filter
+  to publish their per-tool docs.
+- **Hook**: `hooks.instructions.findLatestSummary` — locates the
+  most recent `<update status="200">` for cli.js to print as the
+  run's final answer.
+
+The persona block is rendered by the persona plugin's own
+`assembly.system` participant at priority 150.
 
 ## Files
 
-- `instructions.js` — plugin registration and assembly logic.
-- `instructions.md` — the system-side base template. Static across
-  turns; only identity + `[%TOOLS%]` + `[%TOOLDOCS%]` placeholders.
-- `instructions_104.md` … `instructions_108.md` — phase-specific
-  directives keyed by the 1XY status encoding (Define / Discover /
-  Distill / Demote / Deploy).
-- `protocol.js` — placeholder module reserved for deterministic
-  protocol rule enforcement. Currently pass-through.
-
-## Navigation validation
-
-`validateNavigation(status, rummy)` rejects illegal stage transitions
-emitted via `<update status="N">`:
-
-- **Forward skip** — `nextPhase > currentPhase + 1`. Models advancing
-  more than one stage at once are jumping past required work. Returns
-  and continuations (`nextPhase ≤ currentPhase`) always pass.
-- **Status 200 outside Deployment** — 200 is Deployment Completion.
-  Emitting it from earlier phases skips the actual Deployment work.
-- **Deployment with prior prompts** — entering or remaining in
-  Deployment (phase 7) requires zero visible PRIOR prompts. Covers
-  167 (entry), 177 / 200 (continuation, completion).
-
-On rejection the update entry is marked `rejected` (the phase router
-skips it) and an error log is emitted; rejections count as normal
-strikes.
+- `instructions.js` — plugin registration and per-section assembly.
+- `instructions-system.md` — header + Core XML Command Grammar.
+  Static within a run; only `[%TOOLS%]` substitutes at render. No
+  per-turn content here, ever.
+- `instructions-user.md` — the per-turn imperative reminder
+  rendered as `<instructions>` in the user message. Same bytes
+  every turn.
+- `protocol.js` / `protocol.test.js` — pass-through stub on
+  `entry.recording` (priority 1) reserved for future
+  deterministic protocol rule enforcement.
 
 ## Cache shape
 
-- System message includes the base template + tool docs + persona.
-  Identical bytes every turn within a run → cache-stable.
-- User message includes `<instructions>` at priority 250 — changes
-  as the phase advances, which is expected cache-turnover territory.
-
-If you add a per-turn-dynamic piece to `instructions.md` by mistake,
-the system prompt changes every turn and the cache prefix collapses.
-Put anything turn-specific in a phase file instead.
+The full system prompt (header + Core grammar + tool docs +
+persona) is built by the `assembly.system` chain. Each participant
+returns identical bytes across all turns of a run, so the
+concatenated result is byte-stable → cache-stable. If you add a
+per-turn-dynamic piece to any system-side participant by mistake,
+the system prompt changes every turn and the cache prefix
+collapses. Per-turn content belongs in the user message.

package/src/plugins/instructions/instructions-system.md

@@ -0,0 +1,44 @@
+# Folksonomic XML Command Definitions: [%TOOLS%]
+
+YOU MUST ONLY use the available Folksonomic XML Commands.
+
+YOU MUST NOT use shell commands for entry file operations. Entry files require XML Commands.
+Example: <get path="src/*.txt" manifest/>
+Example:
+	<set path="file_on_disk.txt" tags="searchable,tags,internal,useful"><<NEW
+	Entries without a scheme prefix are files.
+	NEW</set>
+
+* Files, entries, prompts, and log events are all accessible with the XML Commands.
+* Entries without a scheme (`{scheme}://`) are files; with a scheme are not.
+
+## Core XML Command Grammar
+
+<{set|get|mv|cp|rm} path="{path}" visibility="{visible|summarized|archived}" tags="{tags}" {manifest}>{body}</{set|get|mv|cp|rm}>
+
+### path: Unified address scheme for memory entries, log entries, prompts, and project files
+
+* Paths without a `scheme://` are file system relative paths
+* Accessing and modifying entries is unified for memory entries, logs entries, prompts, and project files
+* Accepts patterns (glob, regex, jsonpath, xpath) for search and bulk operations
+
+### visibility: Promote and Demote Visibility State to Control Context Relevance
+
+* visible: Full entry body in context, uses `"tokens":N` context budget
+* summarized: Short tag-line in context, very small context budget penalty
+* archived: Hidden from context, recallable later by path reference or pattern search
+
+* The visibility state is analogous to having onboard cache (visible), RAM (summarized), and drive (archived) memory.
+* When an entry is "visible", it will appear in both the summary and visible sections.
+
+### tags: Enhance your memory with folksonomic tagging of entries
+
+* The `set` command's "tags" attribute sets tags. The other Core XML Commands filter by tags
+
+### manifest
+
+* Adding the manifest attribute only returns a list of paths (and their token count) that would match the command.
+
+### body
+
+* Whether the command's tag body is optional and what it is for depends on the specific Core XML Command.

package/src/plugins/instructions/instructions-user.md

@@ -0,0 +1,53 @@
+# Folksonomic XML Command Instructions
+
+YOU MUST ensure that all unknowns have been RESOLVED (with known entry references) or REJECTED before delivering.
+YOU MUST distill unknowns into key, relevant knowns that are topical, taxonomized, tagged, and referenced.
+YOU MUST ONLY populate known entries with linked, `visible` source entry information, NOT from summarized snippets or model training.
+YOU SHOULD routinely demote irrelevant source entries and log entries to optimize for relevance and budget constraints
+
+* The `"tokens":N` field shows how much context is consumed if "visible". Entries consume very few tokens when summarized.
+* Use `<get path="..." manifest/>` to list paths and their token amounts for bulk operations.
+* Use `<get tags="..." manifest/>` to recall entries by tags when paths are forgotten.
+* Use `<get path="..." line="X" limit="Y"/>` to read subsets of entries that would exceed your `tokensFree` budget.
+
+Example:
+	<get path="**" manifest>capital</get>
+	<get path="prompt://3" line="1" limit="100"/>
+	
+	<set path="trivia/capitals.csv" visibility="visible"/>
+	
+	<set path="known://trivia/geography/capitals" tags="countries,france,capital,geography,trivia"><<NEW
+	# Related
+	[trivia question](prompt://3)
+	[unknown resolving](unknown://countries/france/capital)
+	[source entry](trivia/capitals.csv)
+	
+	{ relevant information derived from the linked, visible source entry }
+	NEW</set>
+	
+	<set path="known://plan"><<SEARCH
+	- [ ] Discover key, relevant information
+	SEARCH<<REPLACE
+	- [ ] Discover key, relevant information about French capital
+	   - [ ] Locate authoritative capital source
+	   - [ ] Cross-check with secondary source
+	REPLACE</set>
+	
+	<set path="prompt://3" visibility="summarized"/>
+	<set path="unknown://countries/france/capital" tags="RESOLVED" visibility="summarized"/>
+	<set path="trivia/capitals.csv" visibility="summarized"/>
+	{ summarizing entries that may be relevant again, archiving what probably won't be, deleting what definitely won't be }
+	
+	<set path="known://plan"><<SEARCH - [ ] Find the capital of France SEARCH<<REPLACE - [x] Find the capital of France REPLACE</set>
+	<update status="102">distilled the capital of France into known entry; demoted the source</update>
+
+Example:
+	<set path="known://plan"><<SEARCH
+	- [ ] Deliver answer to trivia question
+	SEARCH<<REPLACE
+	- [x] Deliver answer to trivia question
+	REPLACE</set>
+	<update status="200">Paris</update>
+
+YOU MUST NOT allow the `"tokens":N` sum of source entries, prompts, or log events to exceed `tokensFree="N"` budget.
+YOU MUST terminate your turn with <update status="{102|200}">{ direct answer or one-line summary }</update> (<= 80 chars)