@possumtech/rummy 2.2.1 → 2.3.2

package/src/agent/XmlParser.js

@@ -3,17 +3,8 @@ import {
 	parseMarkerBody,
 } from "../lib/hedberg/marker.js";
 
-// Edit-marker body opacity. When `#findBodyEnd` is scanning a `<set>`
-// body and hits an opener, jump past the matching closer so tag-shaped
-// content inside the marker (`</set>`, `<get/>`, etc.) doesn't trigger
-// structural recovery.
-//
-// Two opener shapes are recognized for opacity:
-//   - `<<IDENT` — current edit syntax (parsed by marker.js).
-//   - `<<:::IDENT` — packet-rendering shape (engine emits via
-//     plugins/helpers.js). A model copy-pasting the packet shape into
-//     a `<set>` body should still get clean opacity even though
-//     marker.js routes such bodies to plain-body REPLACE.
+// Edit-marker body opacity inside `<set>`. Two opener shapes recognized:
+// `<<IDENT` (edit syntax) and `<<:::IDENT` (packet-rendering shape).
 function skipBareMarker(s, pos) {
 	const m = s.slice(pos).match(/^<<([A-Z][A-Za-z0-9_]*)/);
 	if (!m) return null;
@@ -53,15 +44,10 @@ export const ALL_TOOLS = new Set([
 	"think",
 ]);
 
-// Per-tool resolution: missing canonical attribute is filled silently from the body.
+// Per-tool resolution: missing canonical attribute is filled from the body.
 function resolveCommand(name, a, rawBody) {
-	// Generic heredoc affordance: any non-`<set>` plugin's body may be
-	// wrapped in a single `<<IDENT...IDENT` heredoc to opaquely contain
-	// multi-line scripts, tag-shaped prose, or content with special
-	// characters. Plugins consume the unwrapped inner body verbatim;
-	// the IDENT is exposed as `heredocIdent` on the command for plugins
-	// that want to act on the label. `<set>` is exempt because it does
-	// its own multi-op heredoc parsing via `parseMarkerBody`.
+	// Non-`<set>` plugins accept a single `<<IDENT...IDENT` heredoc wrapper
+	// for opaque multi-line content; `<set>` does its own marker parsing.
 	if (name !== "set") {
 		const heredoc = extractSingleHeredoc(rawBody);
 		if (heredoc) {
@@ -72,25 +58,15 @@ function resolveCommand(name, a, rawBody) {
 	const trimmed = rawBody.trim();
 
 	if (name === "set") {
-		// `search`/`replace` as attributes is no longer in the grammar;
-		// strip them so they can't sneak past via the attribute spread.
 		const { search: _s, replace: _r, ...rest } = a;
 		a = rest;
 
-		// Self-close / no-body: visibility/metadata op.
 		if (!trimmed) return { name, ...a, body: a.body || "" };
 
-		// Edit syntax (SPEC.md "Edit Syntax"): walks the body for
-		// `<<:::IDENT...:::IDENT` markers and returns an ordered op
-		// list. No markers → plain body, treated as full-replace.
-		// Non-keyword IDENTs (path-flavored, identifier-flavored)
-		// route to REPLACE so the model gets a working write whatever
-		// IDENT it picks.
 		const { ops, error } = parseMarkerBody(rawBody);
 		if (error) return { name, ...a, error };
 		if (ops) return { name, ...a, operations: ops };
 
-		// No markers — plain body, full-replace.
 		return { name, ...a, body: trimmed };
 	}
 
@@ -100,9 +76,7 @@ function resolveCommand(name, a, rawBody) {
 		return { name, ...a, body, status };
 	}
 
-	// Body shorthand fallback: when the attribute is unset (undefined),
-	// fall back to the trimmed body. Empty-string attrs are preserved
-	// as-is — handlers validate. `||` would conflate the two cases.
+	// Distinguish unset attr (falls back to body) from empty-string attr.
 	const fromBody = trimmed === "" ? null : trimmed;
 
 	if (name === "get" || name === "rm") {
@@ -137,43 +111,10 @@ const NAME_CHAR = /[a-zA-Z0-9_]/;
 const ATTR_KEY_CHAR = /[a-zA-Z0-9_:-]/;
 const WS = /\s/;
 
-// Tokenizer for rummy's closed set of tool tags. Body opacity for closed
-// bodies; tail recovery for unclosed bodies.
-//
-// Design contract:
-//   - Tool tags (<get>, <set>, <sh>, ...) are the only syntactic special tags.
-//     Any other "<...>" sequence in OUTER text is treated as literal text.
-//   - Inside a tool tag's body, content is OPAQUE: only the matching
-//     `</tagname>` close (depth-counted for same-name nesting) ends the
-//     body. Mismatched closes of OTHER tag names — `</env>`, `</mv>`,
-//     `</foo>` inside a `<set>` body — are body content, not structural
-//     signals.
-//   - Backtick spans (`...`) and triple-backtick fences (```...```)
-//     suppress tag recognition AT THE OUTER LEVEL ONLY (between tool
-//     calls). Documentation prose with backticked tag examples doesn't
-//     get parsed as commands. Inside tool bodies backticks are content;
-//     bodies that need opacity for tag-like content use the edit-syntax
-//     marker family (see SPEC.md "Edit Syntax"), which has no
-//     false-positive failure modes (unlike inside-body backtick
-//     tracking, which would suppress closing tags on bodies with stray
-//     unbalanced backticks).
-//   - Edit-syntax marker opacity (set only): `<<:::IDENT...:::IDENT`
-//     spans inside a `<set>` body are skipped during tag detection so
-//     content with `</set>` literals or marker-shaped text stays as
-//     body. Multiple markers per body supported; see marker.js.
-//   - Same-name nesting (`<set>...<set/>...</set>`) is depth-counted so
-//     nested examples don't prematurely close the outer. Same-name
-//     nesting also disables tail recovery — the model's intent is clearly
-//     opaque body content.
-//   - Unclosed openers (no matching close, no same-name nesting) try
-//     tail recovery: scan the captured body for the leftmost position
-//     whose suffix tokenizes cleanly into ≥1 well-formed tool calls
-//     with zero leftover text. If found, end the unclosed body there
-//     and let the trailing tags parse as proper siblings. The warning
-//     surfaces "Unclosed <name> — recovered N trailing tool call(s)"
-//     so the model can see what happened. If recovery finds nothing,
-//     capture body to EOF and emit "Unclosed <name> — content captured
-//     anyway".
+// Tokenizer for rummy's closed set of tool tags. See SPEC.md "XML Parser"
+// for the full design contract; in short: opaque tool bodies, outer-text
+// backtick suppression, edit-marker opacity inside `<set>`, depth-counted
+// same-name nesting, tail recovery for unclosed openers.
 export default class XmlParser {
 	static MAX_COMMANDS = Number(process.env.RUMMY_MAX_COMMANDS);
 
@@ -198,8 +139,7 @@ export default class XmlParser {
 				break;
 			}
 
-			// Triple-backtick fence toggles take precedence over single backtick
-			// because ``` overlaps `.
+			// Triple takes precedence over single because ``` overlaps `.
 			if (s[i] === "`" && s[i + 1] === "`" && s[i + 2] === "`") {
 				inTripleFence = !inTripleFence;
 				text.push("```");
@@ -227,9 +167,15 @@ export default class XmlParser {
 			}
 
 			const { name, attrs, selfClose, end: openerEnd } = opener;
+			const openerStart = i;
 
 			if (selfClose) {
-				commands.push(resolveCommand(name, attrs, ""));
+				const source = s.slice(openerStart, openerEnd);
+				commands.push({
+					...resolveCommand(name, attrs, ""),
+					source,
+					inner: "",
+				});
 				i = openerEnd;
 				continue;
 			}
@@ -245,10 +191,14 @@ export default class XmlParser {
 					warnings.push(`Unclosed <${name}> tag — content captured anyway`);
 				}
 			}
-			commands.push(resolveCommand(name, attrs, body));
+			const source = s.slice(openerStart, result.afterClose);
+			const inner = body.replace(/^\n+/, "").replace(/\n+$/, "");
+			commands.push({
+				...resolveCommand(name, attrs, body),
+				source,
+				inner,
+			});
 			i = result.afterClose;
-
-			// Body terminated; reset outer-text fence tracking.
 			inSingleBacktick = false;
 			inTripleFence = false;
 		}
@@ -266,8 +216,7 @@ export default class XmlParser {
 		};
 	}
 
-	// Returns { name, attrs, selfClose, end } if `s[pos..]` opens a known tool,
-	// else null. `end` is the index after the closing `>` (or `/>`).
+	// Returns { name, attrs, selfClose, end } or null. `end` is post-`>`/`/>`.
 	static #matchOpener(s, pos) {
 		if (s[pos] !== "<") return null;
 		let i = pos + 1;
@@ -277,7 +226,6 @@ export default class XmlParser {
 		const name = s.slice(nameStart, i).toLowerCase();
 		if (!ALL_TOOLS.has(name)) return null;
 
-		// Char after the name must end the name token cleanly.
 		if (i < s.length && !WS.test(s[i]) && s[i] !== "/" && s[i] !== ">") {
 			return null;
 		}
@@ -322,7 +270,6 @@ export default class XmlParser {
 			i++;
 		}
 
-		// Hit EOF without closing — not a parseable opener.
 		return null;
 	}
 
@@ -367,33 +314,12 @@ export default class XmlParser {
 		return attrs;
 	}
 
-	// Scans body content from `fromPos` until the matching `</name>` closer,
-	// counting depth so same-name nested examples don't prematurely close.
-	// Returns { bodyEnd, afterClose, unclosed }.
-	//
-	// Strict body opacity: only `</name>` (matching the open) and same-name
-	// nested opens affect parsing. Mismatched closes of OTHER tag names are
-	// body content, period.
-	//
-	// Backtick fences (`…`, ```…```) inside the body suppress all tag
-	// recognition — a markdown table cell containing `<set>` examples
-	// stays as content, not interpreted as a nested tag. This matches
-	// the outer-level convention and is the load-bearing reason a model
-	// can write documentation about rummy commands inside a deliverable
-	// body without breaking parsing.
-	//
-	// If the matching close never arrives, emit "Unclosed" so the model
-	// sees a clear failure and corrects on the next turn.
+	// Returns { bodyEnd, afterClose, unclosed }. Same-name nesting is depth-counted.
 	static #findBodyEnd(s, name, fromPos) {
 		let depth = 1;
 		let sameNameNested = false;
 		let i = fromPos;
 		while (i < s.length) {
-			// Edit-syntax marker opacity: marker spans (bare `<<IDENT` or
-			// packet-shaped `<<:::IDENT`) are opaque — tag detection
-			// skips them so inner `</set>` and other tag-shaped content
-			// stays as body. Multiple markers per `<set>` body are
-			// supported; check on every iteration.
 			if (
 				name === "set" &&
 				(s.startsWith("<<:::", i) ||
@@ -436,17 +362,8 @@ export default class XmlParser {
 			}
 			i++;
 		}
-		// Unclosed: try tail recovery, but only if the body never
-		// nested a same-name opener. Same-name nesting is the model
-		// deliberately using opaque body for examples (`<set>` writing
-		// docs about `<set>`); we trust the body content as authored.
-		// No nesting means a plain botched `</set>` — recovery is safe.
-		// If the body's tail is a clean sequence of one or more
-		// well-formed tool calls (zero leftover text), end the body
-		// at the start of that tail and let the outer tokenizer parse
-		// those calls as proper siblings. Closes the silent-swallow
-		// gap when a model botches `</set>` after SEARCH/REPLACE and
-		// emits trailing `<sh>` / `<update>`.
+		// Unclosed → tail recovery, unless same-name nesting (treated as
+		// authored opaque body content with intentional tag examples).
 		if (sameNameNested) {
 			return { bodyEnd: s.length, afterClose: s.length, unclosed: true };
 		}
@@ -462,11 +379,7 @@ export default class XmlParser {
 		return { bodyEnd: s.length, afterClose: s.length, unclosed: true };
 	}
 
-	// Scan body content for the leftmost position whose suffix tokenizes
-	// cleanly into ≥1 commands with no leftover non-whitespace text.
-	// Returns { tailStart, commandCount } or null. Only considers opener
-	// positions; treats the suffix as outer-level so backtick fences and
-	// tag recognition match the parent tokenizer's behavior.
+	// Find leftmost suffix that tokenizes cleanly to ≥1 commands; null if none.
 	static #findTailRecovery(s, fromPos) {
 		let best = null;
 		let i = fromPos;

package/src/agent/errors.js

@@ -1,27 +1,13 @@
-// Outcomes that record a failure but don't strike — findings the model
-// adapts to, not contract violations. `not_found` (model acted on an
-// entry that doesn't exist) and `conflict` (SEARCH text didn't match
-// current body) are recoverable: read the new state, try again.
-// `unparsed` (free text outside any tool tag — comments, "thinking
-// out loud" between tool calls) is non-actionable but not malicious;
-// the empty-turn failure mode is already caught by update plugin's
-// 422 "Missing update", so striking unparsed too is duplicative.
-// Hard outcomes (validation, permission, exit:N) DO strike. Shared
-// between error.js's verdict accumulator (recordedFailed gate) and
-// Entries' auto-failure hook (passes soft=true so error.log.emit
-// skips turn errors increment when the outcome is soft).
+// Recoverable outcomes — recorded but no strike.
 export const SOFT_FAILURE_OUTCOMES = new Set([
 	"not_found",
 	"conflict",
 	"unparsed",
 ]);
 
-// Writer tier excluded from scheme.writable_by; see SPEC writer_tiers.
+// SPEC writer_tiers.
 export class PermissionError extends Error {
 	constructor(scheme, writer, allowed) {
-		// Paths without `://` have a null scheme. Report null verbatim
-		// rather than substituting a plausible-sounding "file" — there is
-		// no scheme called "file" and the error must reflect actual state.
 		const schemeLabel = scheme === null ? "(none)" : scheme;
 		super(
 			`403: writer "${writer}" not permitted for scheme "${schemeLabel}" (allowed: ${allowed.join(", ")})`,
@@ -33,12 +19,7 @@ export class PermissionError extends Error {
 	}
 }
 
-// Body length exceeded the entries.body CHECK constraint (RUMMY_ENTRY_SIZE_MAX
-// at create-time). Surfaced as a 413 strike. The cap value lives only in the
-// schema — JS does not duplicate it, because the database persists across
-// rummy invocations and the env var that built the schema may differ from
-// the env var seen by the running instance. Reporting body size is enough
-// for the model to adapt; operators can read the cap from the schema.
+// 413 strike: body exceeded entries.body CHECK (RUMMY_ENTRY_SIZE_MAX).
 export class EntryOverflowError extends Error {
 	constructor(path, size) {
 		super(

package/src/agent/materializeContext.js

@@ -2,16 +2,9 @@ import { SUMMARY_MAX_CHARS } from "../plugins/helpers.js";
 import ContextAssembler from "./ContextAssembler.js";
 import { countLines, countTokens } from "./tokens.js";
 
-// Defensive cap: model-written summary projections (knowns, unknowns,
-// log actions, etc.) must produce ≤ SUMMARY_MAX_CHARS — the contract
-// floor for terse model-authored summaries. File-scheme entries are
-// exempt: their summarized projection is a structural derivative
-// (rummy.repo's symbol map), bounded by the file's actual complexity,
-// not by writer discipline. Truncating symbol data at 500 chars
-// destroys its utility. Files either render blank (no symbols
-// extracted) or render their full symbol map.
-
 // Rebuild turn_context from v_model_context and assemble messages.
+// File-scheme is exempt from SUMMARY_MAX_CHARS (its summary is a structural
+// symbol map, not writer-bounded prose).
 export default async function materializeContext({
 	db,
 	hooks,
@@ -27,12 +20,11 @@ export default async function materializeContext({
 }) {
 	await db.clear_turn_context.run({ run_id: runId, turn });
 	const viewRows = await db.get_model_context.all({ run_id: runId });
-	// Per-entry token accounting; merged back after the turn_context roundtrip.
 	const tokenAccounting = new Map();
 	for (const row of viewRows) {
 		const scheme = row.scheme ? row.scheme : "file";
 		const attrs = row.attributes ? JSON.parse(row.attributes) : null;
-		// Dispatch log entries to their action plugin's view via path segment.
+		// Log entries dispatch to their action plugin's view via path segment.
 		let projectionKey = scheme;
 		if (scheme === "log") {
 			const m = row.path.match(/^log:\/\/turn_\d+\/([^/]+)\//);

package/src/hooks/Hooks.js

@@ -2,7 +2,6 @@ import HookRegistry from "./HookRegistry.js";
 import RpcRegistry from "./RpcRegistry.js";
 import ToolRegistry from "./ToolRegistry.js";
 
-// Strictly-typed hook surface; replaces the previous Proxy magic.
 export default function createHooks(debug = false) {
 	const registry = new HookRegistry(debug);
 	const tools = new ToolRegistry();
@@ -20,13 +19,10 @@ export default function createHooks(debug = false) {
 	});
 
 	return {
-		// Core Turn Pipeline
 		onTurn: registry.onTurn.bind(registry),
 		processTurn: registry.processTurn.bind(registry),
 
-		// Explicit Hook Schema
 		boot: {
-			// Post-init, pre-accept-connections; one-shot post-init actions subscribe here.
 			completed: createEvent("boot.completed"),
 		},
 		project: {
@@ -48,13 +44,6 @@ export default function createHooks(debug = false) {
 			step: {
 				completed: createEvent("run.step.completed"),
 			},
-			// Fire-and-forget wake: any plugin that wants to deliver a new
-			// prompt onto a (possibly dormant) run emits with
-			// {runAlias, body, mode}. AgentLoop subscribes and runs inject —
-			// writes prompt://<nextTurn>, enqueues a loop, ensures the
-			// drainer is up. This is the "streaming child closed after the
-			// loop ended" rendezvous: the producer doesn't care whether the
-			// run is alive or asleep, just that the prompt reaches it.
 			wake: createEvent("run.wake"),
 		},
 		loop: {
@@ -63,28 +52,12 @@ export default function createHooks(debug = false) {
 		},
 		turn: {
 			started: createEvent("turn.started"),
-			// Pre-LLM packet shaping. Filter chain: subscribers receive
-			// `{ messages, rows, contextSize, lastPromptTokens,
-			// assembledTokens, ok, overflow }` and return a transformed
-			// packet. Budget plugin participates here to enforce ceilings
-			// (may demote, may set ok=false on overflow). Other plugins
-			// could trim, re-order, or annotate — same surface.
 			beforeDispatch: createFilter("turn.beforeDispatch"),
 			response: createEvent("turn.response"),
-			// Post-dispatch event. Fired after the per-entry dispatch
-			// loop, before turn.completed. Budget subscribes here for
-			// post-dispatch demotion / 413 overflow detection.
 			dispatched: createEvent("turn.dispatched"),
 			completed: createEvent("turn.completed"),
-			// Verdict filter chain: each subscriber receives the current
-			// verdict object and returns a (possibly modified) one.
-			// Initial value is { continue: true }; final value drives the
-			// loop's continue/abandon decision. Multi-plugin: strike streak,
-			// cycle detect, stagnation pressure, future voters all
-			// participate via this surface.
 			verdict: createFilter("turn.verdict"),
 		},
-		// SPEC #resolution covers the proposal hook chain.
 		proposal: {
 			prepare: createEvent("proposal.prepare"),
 			pending: createEvent("proposal.pending"),
@@ -115,9 +88,7 @@ export default function createHooks(debug = false) {
 			},
 			messages: createFilter("llm.messages"),
 			response: createFilter("llm.response"),
-			// Plugins contribute reasoning text into reasoning_content; fires between parse and turn.response.
 			reasoning: createFilter("llm.reasoning"),
-			// Provider entries: { name, matches, completion, getContextSize }.
 			providers: [],
 		},
 		file: {},

package/src/hooks/PluginContext.js

@@ -1,3 +1,14 @@
+import { projectEmission, summarizeEmission } from "../plugins/helpers.js";
+
+// Shared projection helpers for plugins (including external ones).
+// Action log entries tab-indent body via emission; summaries cap at
+// SUMMARY_MAX_CHARS via summarize. External plugins use these via
+// `core.projection` to avoid drift across the action-log paradigm.
+const PROJECTION = Object.freeze({
+	emission: projectEmission,
+	summarize: summarizeEmission,
+});
+
 // Plugin-only registration interface; tool verbs live on RummyContext. PLUGINS.md.
 export default class PluginContext {
 	#name;
@@ -8,6 +19,10 @@ export default class PluginContext {
 		this.#hooks = hooks;
 	}
 
+	get projection() {
+		return PROJECTION;
+	}
+
 	get name() {
 		return this.#name;
 	}

package/src/lib/hedberg/hedberg.js

@@ -1,9 +1,7 @@
 import HeuristicMatcher, { generatePatch } from "./matcher.js";
 import { hedmatch, hedsearch } from "./patterns.js";
 
-// Stochastic→deterministic boundary; exposes pattern utilities on
-// core.hedberg. SPEC #hedberg. Edit-shape parsing lives in marker.js
-// and is invoked from XmlParser at <set> resolution time.
+// SPEC #hedberg. Edit-shape parsing lives in marker.js.
 export default class Hedberg {
 	#core;
 
@@ -18,17 +16,9 @@ export default class Hedberg {
 		};
 	}
 
-	// Order: literal substitution → heuristic fuzzy.
-	//
-	// sed=true semantically means "literal substring substitution with
-	// regex-style escape friendliness." The model writes `\[`, `\.`,
-	// `\|`, etc. out of muscle memory from real sed, but we don't
-	// compile a regex — native String.replaceAll does the substitution.
-	// We strip the regex-meta backslashes from search and replacement
-	// so the model's escaped chars match their literal counterparts in
-	// body. This sidesteps a class of "regex-meta in content" failures
-	// and the parser-edge-case surface that compiling user input as
-	// regex drags in.
+	// Literal substitution first, heuristic fuzzy fallback. `sed=true` strips
+	// regex-meta backslashes for muscle-memory escape friendliness; we never
+	// actually compile a regex.
 	static replace(body, search, replacement, { sed = false } = {}) {
 		let patch = null;
 		let warning = null;

package/src/lib/hedberg/marker.js

@@ -1,49 +1,14 @@
-// Edit-syntax marker parser. Recognizes bash-heredoc-shaped
-// `<<IDENT...IDENT` body markers inside `<set>` content and routes
-// by IDENT prefix to one of six operations: NEW, PREPEND, APPEND,
-// REPLACE, DELETE, SEARCH. Non-keyword IDENTs (e.g. `<<DOC`, `<<EOF`)
-// route to REPLACE — the content between markers becomes the full
-// new body.
-//
-// Grammar:
-//   - Opener: `<<IDENT` where IDENT matches `[A-Z][A-Za-z0-9_]*`.
-//     Boundary: preceded by start-of-body, whitespace, or `>` (so
-//     `vec<<SEARCH` mid-token does not false-trigger).
-//   - Closer: bare IDENT (matching opener exactly) with non-word
-//     boundaries — preceded by whitespace/start, followed by
-//     whitespace, `<`, `>`, or end.
-//   - SEARCH must be immediately followed by REPLACE; the pair maps
-//     to one search_replace op. Lone SEARCH is a parse error.
-//   - Trailing alphanumeric suffix on the IDENT is opaque to routing
-//     (`<<SEARCH1` and `<<SEARCH` both route to SEARCH). Suffix
-//     exists so nested markers can disambiguate, same convention as
-//     bash heredoc `<<EOF1` vs `<<EOF`. When a body literally
-//     contains the bare keyword (`SEARCH` in prose or code), the
-//     model picks a suffix so the inner literal does not prematurely
-//     close the outer marker.
-//
-// The bare `<<IDENT` shape is visibly distinct from the engine's
-// packet-rendering shape `<<:::IDENT` (see plugins/helpers.js). Edit
-// syntax is bare-only: a body with `<<:::IDENT` does NOT match this
-// parser and falls through to plain-body REPLACE with the markers
-// preserved as literal content. Keep the two grammars distinct so
-// model emissions and engine renderings can never be confused.
-//
-// Returns:
-//   { ops: null,    error: null }   — no markers found, treat body as plain.
-//   { ops: [{...}], error: null }   — well-formed marker(s).
-//   { ops: null,    error: "..." }  — parse failure (lone SEARCH, unclosed).
+// Edit-syntax marker parser for `<set>` bodies. Grammar in SPEC.md "Edit Syntax".
+// Returns { ops, error } — `ops: null` on either no-markers or parse failure.
 
 const KEYWORD_RE =
 	/^(NEW|PREPEND|APPEND|REPLACE|DELETE|SEARCH)([A-Za-z0-9_]*)$/;
 
-// Opener: `<<IDENT` preceded by start-of-input, whitespace, or `>`.
 const OPENER_RE = /(?<=^|[\s>])<<([A-Z][A-Za-z0-9_]*)/;
 
 function operationFromIdent(ident) {
 	const m = ident.match(KEYWORD_RE);
 	if (m) return m[1].toLowerCase();
-	// Non-keyword IDENT — treat as REPLACE.
 	return "replace";
 }
 
@@ -60,10 +25,7 @@ function findOpener(body, startIdx) {
 
 function findCloser(body, startIdx, ident) {
 	const escIdent = ident.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-	// Closer: bare IDENT with non-word boundaries — preceded by
-	// whitespace or start-of-input, followed by whitespace, `<`, `>`,
-	// or end. The trailing `<` lets the SEARCH closer adjoin an
-	// immediately-following `<<REPLACE` opener (`SEARCH<<REPLACE`).
+	// Trailing `<` lets `SEARCH<<REPLACE` adjoin without intermediate whitespace.
 	const re = new RegExp(`(?<=^|\\s)${escIdent}(?=[\\s<>]|$)`);
 	const slice = body.slice(startIdx);
 	const match = slice.match(re);
@@ -81,21 +43,7 @@ function trimMarkerNewlines(content) {
 	return result;
 }
 
-// Detect a body that is exactly one heredoc wrapping its entire content.
-// Returns `{ ident, content }` if `body` is `<<IDENT\n...\nIDENT` (with
-// optional surrounding whitespace), otherwise `null`. Used by non-`<set>`
-// plugins to let models opaquely wrap multi-line scripts, tag-shaped
-// prose, or content with special characters — without requiring escaping
-// or string-quoting at the model layer. The plugin sees the unwrapped
-// inner content as its body; the IDENT is attached to the command as
-// `heredocIdent` for plugins that want to act on the label.
-//
-// Reuses the same `findOpener`/`findCloser` helpers as `parseMarkerBody`,
-// so the grammar (boundary rules, IDENT shape, suffix nesting) stays
-// single-sourced. Difference is just the validation: this function
-// requires the heredoc to span the body exactly (opener at start,
-// closer at end), where `parseMarkerBody` accepts multiple markers in
-// sequence.
+// Returns { ident, content } if `body` is exactly one heredoc; null otherwise.
 export function extractSingleHeredoc(body) {
 	if (!body) return null;
 	const trimmed = body.trim();
@@ -114,7 +62,6 @@ export function extractSingleHeredoc(body) {
 }
 
 export function parseMarkerBody(body) {
-	// Cheap rejection — most `<set>` bodies don't contain markers.
 	if (!/<<[A-Z]/.test(body)) return { ops: null, error: null };
 
 	const raw = [];
@@ -125,7 +72,17 @@ export function parseMarkerBody(body) {
 		const op = operationFromIdent(opener.ident);
 		const closer = findCloser(body, opener.openerEnd, opener.ident);
 		if (!closer) {
-			return { ops: null, error: `unclosed <<${opener.ident}` };
+			// Tail-close recovery: last opener with no closer and no further
+			// opener absorbs body to EOF. SEARCH stays strict (needs REPLACE).
+			if (op === "search") {
+				return { ops: null, error: `unclosed <<${opener.ident}` };
+			}
+			const tail = body.slice(opener.openerEnd);
+			if (findOpener(tail, 0)) {
+				return { ops: null, error: `unclosed <<${opener.ident}` };
+			}
+			raw.push({ op, content: trimMarkerNewlines(tail) });
+			break;
 		}
 		const content = trimMarkerNewlines(
 			body.slice(opener.openerEnd, closer.closerStart),
@@ -135,7 +92,6 @@ export function parseMarkerBody(body) {
 	}
 	if (raw.length === 0) return { ops: null, error: null };
 
-	// Pair adjacent SEARCH+REPLACE into one search_replace op.
 	const ops = [];
 	for (let j = 0; j < raw.length; j++) {
 		const cur = raw[j];

package/src/llm/LlmProvider.js

@@ -10,19 +10,11 @@ const LLM_DEADLINE = Number(process.env.RUMMY_LLM_DEADLINE);
 const LLM_MAX_BACKOFF = Number(process.env.RUMMY_LLM_MAX_BACKOFF);
 
 const TOKEN_DIVISOR = Number(process.env.RUMMY_TOKEN_DIVISOR);
-// Floor on derived max_tokens. If prompt eats almost the entire context,
-// we still ask for at least this many output tokens so the model has
-// room to emit a usable terminal `<update>`.
+// Floor so a near-full prompt still leaves room for a closing `<update>`.
 const MAX_TOKENS_FLOOR = 1024;
-// Fraction of the model's context the request may consume (prompt +
-// max_tokens combined). The remaining 1−X absorbs tokenizer drift
-// between our chars/RUMMY_TOKEN_DIVISOR estimate and the provider's
-// BPE-based count plus message-envelope overhead.
+// 1−X headroom absorbs BPE/estimator drift and envelope overhead.
 const BUDGET_CEILING = Number(process.env.RUMMY_BUDGET_CEILING);
 
-// Per-category retry policies. Gateway/server are bounded short because
-// upstream-down won't recover by waiting; warmup/rate_limit get the full
-// LLM deadline because they're recoverable wait states with knowable bounds.
 const POLICIES = Object.freeze({
 	gateway: { deadlineMs: 30_000, baseDelayMs: 500, maxDelayMs: 5_000 },
 	warmup: {
@@ -38,7 +30,6 @@ const POLICIES = Object.freeze({
 	server: { deadlineMs: 60_000, baseDelayMs: 1000, maxDelayMs: 10_000 },
 });
 
-// Dispatches to hooks.llm.providers; per-category transient retry; ContextExceededError surface.
 export default class LlmProvider {
 	#db;
 	#hooks;
@@ -67,27 +58,23 @@ export default class LlmProvider {
 				? Number.parseFloat(process.env.RUMMY_TEMPERATURE)
 				: undefined);
 
-		// Derive max_tokens from the model's context window minus the
-		// estimated prompt footprint. Without this, providers fall back
-		// to conservative defaults (a few thousand) and the model's
-		// response truncates mid-`<set>` body before reaching `<update>`,
-		// surfacing as a misleading "no <update>" verdict.
+		// max_tokens = effectiveContext − promptEstimate. lastPromptTokens
+		// is ground truth when available (turn 1 falls back to chars/divisor).
 		const contextLength = await this.getContextSize(model);
-		const promptEstimate = messages.reduce(
-			(sum, m) => sum + Math.ceil(m.content.length / TOKEN_DIVISOR),
-			0,
-		);
+		const promptEstimate =
+			options.lastPromptTokens > 0
+				? options.lastPromptTokens
+				: messages.reduce(
+						(sum, m) => sum + Math.ceil(m.content.length / TOKEN_DIVISOR),
+						0,
+					);
 		const effectiveContext = Math.floor(contextLength * BUDGET_CEILING);
 		let maxTokens = Math.max(
 			MAX_TOKENS_FLOOR,
 			effectiveContext - promptEstimate,
 		);
-		// Per-model output ceiling. Models advertise huge context windows
-		// but actual max OUTPUT tokens is far smaller. Sending max_tokens
-		// above the model's real output cap pushes the request into
-		// undefined-behavior territory and can correlate with mid-emission
-		// EOT sampling. Set `RUMMY_OUTPUT_CAP_<alias>` per model where
-		// the published output ceiling is known.
+		// Per-model output cap (`RUMMY_OUTPUT_CAP_<alias>`) — output ceilings
+		// are typically far smaller than advertised context windows.
 		const outputCapEnv = process.env[`RUMMY_OUTPUT_CAP_${model}`];
 		if (outputCapEnv) {
 			const cap = Number.parseInt(outputCapEnv, 10);