@possumtech/rummy 0.2.1

package/src/plugins/hedberg/patterns.js

@@ -0,0 +1,452 @@
+import { JSDOM } from "jsdom";
+
+export const deterministic = true;
+
+const cache = new Map();
+
+// --- Detection ---
+
+const XPATH_AXES = new Set([
+	"child",
+	"descendant",
+	"descendant-or-self",
+	"parent",
+	"ancestor",
+	"ancestor-or-self",
+	"following",
+	"following-sibling",
+	"preceding",
+	"preceding-sibling",
+	"self",
+	"attribute",
+	"namespace",
+]);
+
+const XPATH_FUNCTIONS =
+	/\b(position|last|contains|starts-with|not|text|count|sum|name|local-name|string-length|normalize-space|concat|substring|translate|boolean|number|string|true|false|ceiling|floor|round|id|lang|comment|processing-instruction)\s*\(/;
+
+function isPlausibleJsonPath(pattern) {
+	if (pattern.startsWith("$.")) {
+		const after = pattern[2];
+		if (!after) return false;
+		if (/[a-zA-Z_*@.[[]/.test(after)) return !pattern.includes("/");
+		return false;
+	}
+	if (pattern.startsWith("$[")) return !pattern.includes("/");
+	if (pattern === "$..") return true;
+	return false;
+}
+
+function hasXPathAxis(pattern) {
+	const matches = pattern.matchAll(/(\w[\w-]*)(?=::)/g);
+	for (const m of matches) {
+		if (XPATH_AXES.has(m[1])) return true;
+	}
+	return false;
+}
+
+function hasXPathPredicate(pattern) {
+	const brackets = pattern.matchAll(/\[([^\]]+)\]/g);
+	for (const m of brackets) {
+		const content = m[1];
+		if (content.startsWith("@")) return true;
+		if (/^\d+$/.test(content.trim())) return true;
+		if (XPATH_FUNCTIONS.test(content)) return true;
+	}
+	return false;
+}
+
+function parseSed(pattern) {
+	// s/search/replace/ or s/search/replace/flags
+	if (!pattern.startsWith("s/")) return null;
+	const parts = [];
+	let i = 2;
+	let current = "";
+	while (i < pattern.length) {
+		if (pattern[i] === "/" && pattern[i - 1] !== "\\") {
+			parts.push(current);
+			current = "";
+		} else {
+			current += pattern[i];
+		}
+		i++;
+	}
+	parts.push(current);
+	if (parts.length < 2) return null;
+	const search = parts[0];
+	const replace = parts[1];
+	const flags = parts[2] || "";
+	return { search, replace, flags };
+}
+
+function detect(pattern) {
+	// Sed-style: s/search/replace/flags
+	if (pattern.startsWith("s/")) {
+		const parsed = parseSed(pattern);
+		if (parsed) return "sed";
+	}
+
+	// Explicit regex: /pattern/ or /pattern/flags
+	if (
+		pattern.startsWith("/") &&
+		pattern.length > 2 &&
+		/\/[gimsuy]*$/.test(pattern.slice(1))
+	) {
+		const lastSlash = pattern.lastIndexOf("/");
+		if (lastSlash > 0) return "regex";
+	}
+
+	// JSONPath
+	if (
+		(pattern.startsWith("$.") ||
+			pattern.startsWith("$[") ||
+			pattern.startsWith("$..")) &&
+		isPlausibleJsonPath(pattern)
+	) {
+		return "jsonpath";
+	}
+
+	// XPath
+	if (
+		pattern.startsWith("//") &&
+		pattern.length > 2 &&
+		/[a-zA-Z_*@.]/.test(pattern[2])
+	) {
+		return "xpath";
+	}
+
+	if (pattern.startsWith("/")) {
+		if (hasXPathAxis(pattern)) return "xpath";
+		if (hasXPathPredicate(pattern)) return "xpath";
+	}
+
+	// Glob: ** or unescaped * ? not after .
+	if (pattern.includes("**")) return "glob";
+	if (/(?:^|[^.\\])[*?]/.test(pattern)) return "glob";
+
+	// Everything else is literal
+	return "literal";
+}
+
+// --- Compilation ---
+
+function globToRegex(glob) {
+	let result = "";
+	for (let i = 0; i < glob.length; i++) {
+		const c = glob[i];
+		if (c === "*") result += ".*";
+		else if (c === "?") result += ".";
+		else if (c === "[") {
+			const close = glob.indexOf("]", i + 1);
+			if (close === -1) {
+				result += "\\[";
+				continue;
+			}
+			result += glob.slice(i, close + 1);
+			i = close;
+		} else if (/[.+^${}()|\\]/.test(c)) {
+			result += `\\${c}`;
+		} else result += c;
+	}
+	return result;
+}
+
+function parseRegex(pattern) {
+	const lastSlash = pattern.lastIndexOf("/");
+	const body = pattern.slice(1, lastSlash);
+	const flags = pattern.slice(lastSlash + 1);
+	return { body, flags };
+}
+
+function parseJsonPath(path) {
+	const segments = [];
+	let i = path.startsWith("$") ? 1 : 0;
+
+	while (i < path.length) {
+		if (path[i] === ".") {
+			if (path[i + 1] === ".") {
+				segments.push({ type: "recursive" });
+				i += 2;
+				const start = i;
+				while (i < path.length && path[i] !== "." && path[i] !== "[") i++;
+				const key = path.slice(start, i);
+				if (key === "*") segments.push({ type: "wildcard" });
+				else if (key) segments.push({ type: "key", value: key });
+			} else {
+				i++;
+				const start = i;
+				while (i < path.length && path[i] !== "." && path[i] !== "[") i++;
+				const key = path.slice(start, i);
+				if (key === "*") segments.push({ type: "wildcard" });
+				else if (key) segments.push({ type: "key", value: key });
+			}
+		} else if (path[i] === "[") {
+			i++;
+			if (path[i] === "*") {
+				segments.push({ type: "wildcard" });
+				i += 2;
+			} else if (path[i] === "'" || path[i] === '"') {
+				const quote = path[i];
+				i++;
+				const start = i;
+				while (i < path.length && path[i] !== quote) i++;
+				segments.push({ type: "key", value: path.slice(start, i) });
+				i += 2;
+			} else {
+				const start = i;
+				while (i < path.length && path[i] !== "]") i++;
+				segments.push({
+					type: "index",
+					value: parseInt(path.slice(start, i), 10),
+				});
+				i++;
+			}
+		} else {
+			i++;
+		}
+	}
+	return segments;
+}
+
+function compile(pattern) {
+	const type = detect(pattern);
+	switch (type) {
+		case "literal":
+			return { type, pattern };
+		case "glob":
+			return {
+				type,
+				anchoredRe: new RegExp(`^${globToRegex(pattern)}$`),
+				searchRe: new RegExp(globToRegex(pattern)),
+			};
+		case "regex": {
+			const { body, flags } = parseRegex(pattern);
+			return {
+				type,
+				re: new RegExp(body, flags || undefined),
+				reGlobal: new RegExp(body, flags.includes("g") ? flags : `${flags}g`),
+			};
+		}
+		case "sed": {
+			const parsed = parseSed(pattern);
+			const isRegex = parsed.flags.length > 0;
+			return {
+				type,
+				search: parsed.search,
+				replace: parsed.replace,
+				flags: parsed.flags,
+				isRegex,
+				re: isRegex ? new RegExp(parsed.search, parsed.flags) : null,
+			};
+		}
+		case "xpath":
+			return { type, expr: pattern };
+		case "jsonpath":
+			return { type, segments: parseJsonPath(pattern) };
+	}
+}
+
+// --- XPath evaluation ---
+
+function evalXpath(expr, string) {
+	try {
+		const dom = new JSDOM(string, { contentType: "text/xml" });
+		const doc = dom.window.document;
+		const result = doc.evaluate(expr, doc, null, 0, null);
+		const node = result.iterateNext();
+		if (!node) return null;
+		return { match: node.textContent, node };
+	} catch {
+		return null;
+	}
+}
+
+// --- JSONPath evaluation ---
+
+function evalJsonPath(segments, string) {
+	let current;
+	try {
+		current = [JSON.parse(string)];
+	} catch {
+		return null;
+	}
+
+	for (const seg of segments) {
+		const next = [];
+		for (const node of current) {
+			if (node === null || node === undefined) continue;
+			switch (seg.type) {
+				case "key":
+					if (typeof node === "object" && seg.value in node) {
+						next.push(node[seg.value]);
+					}
+					break;
+				case "index":
+					if (Array.isArray(node) && seg.value < node.length) {
+						next.push(node[seg.value]);
+					}
+					break;
+				case "wildcard":
+					if (Array.isArray(node)) next.push(...node);
+					else if (typeof node === "object") next.push(...Object.values(node));
+					break;
+				case "recursive":
+					next.push(node);
+					collectDescendants(node, next);
+					break;
+			}
+		}
+		if (next.length === 0) return null;
+		current = next;
+	}
+	return current.length > 0 ? current : null;
+}
+
+function collectDescendants(node, out) {
+	if (node === null || typeof node !== "object") return;
+	const values = Array.isArray(node) ? node : Object.values(node);
+	for (const v of values) {
+		if (v !== null && typeof v === "object") {
+			out.push(v);
+			collectDescendants(v, out);
+		}
+	}
+}
+
+// --- Public API ---
+
+/**
+ * hedmatch — does the pattern match the ENTIRE string?
+ * For path matching, WHERE clauses, full-string comparison.
+ */
+export function hedmatch(pattern, string) {
+	if (string === null) return false;
+
+	let compiled = cache.get(pattern);
+	if (!compiled) {
+		compiled = compile(pattern);
+		cache.set(pattern, compiled);
+	}
+
+	switch (compiled.type) {
+		case "literal":
+			return string === compiled.pattern;
+		case "glob":
+			return compiled.anchoredRe.test(string);
+		case "regex":
+			return compiled.re.test(string);
+		case "sed":
+			return compiled.isRegex
+				? compiled.re.test(string)
+				: string.includes(compiled.search);
+		case "xpath":
+			return evalXpath(compiled.expr, string) !== null;
+		case "jsonpath":
+			return evalJsonPath(compiled.segments, string) !== null;
+	}
+	return false;
+}
+
+/**
+ * hedsearch — find the pattern anywhere IN the string.
+ * For substring search, content filtering, "does this text contain...".
+ * Returns { found, match, index } or { found: false }.
+ */
+export function hedsearch(pattern, string) {
+	if (string === null) return { found: false };
+
+	let compiled = cache.get(pattern);
+	if (!compiled) {
+		compiled = compile(pattern);
+		cache.set(pattern, compiled);
+	}
+
+	switch (compiled.type) {
+		case "literal": {
+			const idx = string.indexOf(compiled.pattern);
+			if (idx === -1) return { found: false };
+			return { found: true, match: compiled.pattern, index: idx };
+		}
+		case "glob": {
+			const m = compiled.searchRe.exec(string);
+			if (!m) return { found: false };
+			return { found: true, match: m[0], index: m.index };
+		}
+		case "regex": {
+			const m = compiled.re.exec(string);
+			if (!m) return { found: false };
+			return { found: true, match: m[0], index: m.index };
+		}
+		case "sed": {
+			if (compiled.isRegex) {
+				compiled.re.lastIndex = 0;
+				const m = compiled.re.exec(string);
+				if (!m) return { found: false };
+				return { found: true, match: m[0], index: m.index };
+			}
+			const idx = string.indexOf(compiled.search);
+			if (idx === -1) return { found: false };
+			return { found: true, match: compiled.search, index: idx };
+		}
+		case "xpath": {
+			const result = evalXpath(compiled.expr, string);
+			if (!result) return { found: false };
+			return { found: true, match: result.match, index: 0 };
+		}
+		case "jsonpath": {
+			const result = evalJsonPath(compiled.segments, string);
+			if (!result) return { found: false };
+			return { found: true, match: result[0], index: 0 };
+		}
+	}
+	return { found: false };
+}
+
+/**
+ * hedreplace — find pattern in string, replace with replacement.
+ * Returns the new string, or null if pattern not found.
+ */
+export function hedreplace(pattern, replacement, string) {
+	if (string === null) return null;
+
+	let compiled = cache.get(pattern);
+	if (!compiled) {
+		compiled = compile(pattern);
+		cache.set(pattern, compiled);
+	}
+
+	switch (compiled.type) {
+		case "literal": {
+			if (!string.includes(compiled.pattern)) return null;
+			return string.replaceAll(compiled.pattern, replacement);
+		}
+		case "glob": {
+			if (!compiled.searchRe.test(string)) return null;
+			return string.replace(compiled.searchRe, replacement);
+		}
+		case "regex": {
+			if (!compiled.re.test(string)) return null;
+			compiled.re.lastIndex = 0;
+			compiled.reGlobal.lastIndex = 0;
+			return string.replace(compiled.reGlobal, replacement);
+		}
+		case "sed": {
+			// For sed, replacement is embedded in the pattern. Ignore the argument.
+			if (compiled.isRegex) {
+				compiled.re.lastIndex = 0;
+				if (!compiled.re.test(string)) return null;
+				compiled.re.lastIndex = 0;
+				return string.replace(compiled.re, compiled.replace);
+			}
+			if (!string.includes(compiled.search)) return null;
+			return string.replaceAll(compiled.search, compiled.replace);
+		}
+		case "xpath":
+		case "jsonpath":
+			return null;
+	}
+	return null;
+}
+
+// SQL functions are in separate files (hedmatch.js, hedsearch.js)
+// that import from this library. Filename = SQL function name.

package/src/plugins/hedberg/sed.js

@@ -0,0 +1,48 @@
+/**
+ * Sed syntax parsing. Handles s/search/replace/flags with:
+ * - Escaped delimiters (\\/)
+ * - Chained commands (s/a/b/ s/c/d/)
+ * - Flag extraction (g, i, m, s, v)
+ */
+
+function splitSed(str) {
+	const parts = [];
+	let current = "";
+	for (let i = 0; i < str.length; i++) {
+		if (str[i] === "\\" && i + 1 < str.length) {
+			current += str[i] + str[i + 1];
+			i++;
+		} else if (str[i] === "/") {
+			parts.push(current);
+			current = "";
+		} else {
+			current += str[i];
+		}
+	}
+	parts.push(current);
+	return parts;
+}
+
+export function parseSed(input) {
+	if (!input.startsWith("s/")) return null;
+
+	const blocks = [];
+	let remaining = input;
+	while (remaining.startsWith("s/")) {
+		const parts = splitSed(remaining.slice(2));
+		if (parts.length < 2) break;
+		const flags = (parts[2] || "").match(/^[gimsv]*/)?.[0] || "";
+		blocks.push({
+			search: parts[0].replaceAll("\\/", "/"),
+			replace: parts[1].replaceAll("\\/", "/"),
+			flags,
+			sed: true,
+		});
+		const rest = parts.slice(2).join("/");
+		const next = rest.indexOf("s/");
+		remaining = next >= 0 ? rest.slice(next) : "";
+	}
+
+	if (blocks.length === 0) return null;
+	return blocks;
+}

package/src/plugins/helpers.js

@@ -0,0 +1,22 @@
+/**
+ * Shared helper for pattern-based tool results.
+ * Used by get, set, store, and rm tools.
+ */
+export async function storePatternResult(
+	store,
+	runId,
+	turn,
+	scheme,
+	path,
+	bodyFilter,
+	matches,
+	preview = false,
+) {
+	const slug = await store.slugPath(runId, scheme, path);
+	const filter = bodyFilter ? ` body="${bodyFilter}"` : "";
+	const total = matches.reduce((s, m) => s + m.tokens_full, 0);
+	const listing = matches.map((m) => `${m.path} (${m.tokens_full})`).join("\n");
+	const prefix = preview ? "PREVIEW " : "";
+	const body = `${prefix}${scheme} path="${path}"${filter}: ${matches.length} matched (${total} tokens)\n${listing}`;
+	await store.upsert(runId, turn, slug, body, "pattern");
+}

package/src/plugins/index.js

@@ -0,0 +1,180 @@
+import { existsSync } from "node:fs";
+import { readdir, stat } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { pathToFileURL } from "node:url";
+import PluginContext from "../hooks/PluginContext.js";
+
+const instances = new Map();
+
+/**
+ * Dynamically loads and registers plugins from provided directories
+ * and RUMMY_PLUGIN_* env vars.
+ */
+export async function registerPlugins(dirs = [], hooks) {
+	const uniqueDirs = [...new Set(dirs.map((d) => join(d)))];
+
+	for (const dir of uniqueDirs) {
+		await scanDir(dir, hooks, true);
+	}
+
+	await loadEnvPlugins(hooks);
+}
+
+const AUDIT_SCHEMES = [
+	"instructions",
+	"system",
+	"prompt",
+	"ask",
+	"act",
+	"progress",
+	"reasoning",
+	"model",
+	"error",
+	"user",
+	"assistant",
+	"content",
+];
+
+/**
+ * After DB is ready, inject db and store into all PluginContext instances,
+ * upsert declared schemes, and bootstrap audit schemes.
+ */
+export async function initPlugins(db, store, hooks) {
+	for (const name of AUDIT_SCHEMES) {
+		const scheme = {
+			name,
+			fidelity: ["ask", "act", "progress"].includes(name) ? "full" : "null",
+			model_visible: ["ask", "act", "progress"].includes(name) ? 1 : 0,
+			valid_states: JSON.stringify(["info"]),
+			category: "audit",
+		};
+		await db.upsert_scheme.run(scheme);
+	}
+
+	for (const ctx of instances.values()) {
+		ctx.db = db;
+		ctx.entries = store;
+		for (const scheme of ctx.schemes) {
+			await db.upsert_scheme.run(scheme);
+		}
+	}
+
+	// Register default schemes for tools that plugins ensured but didn't registerScheme for
+	if (hooks) {
+		const registered = new Set();
+		for (const ctx of instances.values()) {
+			for (const s of ctx.schemes) registered.add(s.name);
+		}
+		for (const name of AUDIT_SCHEMES) registered.add(name);
+
+		for (const toolName of hooks.tools.names) {
+			if (registered.has(toolName)) continue;
+			await db.upsert_scheme.run({
+				name: toolName,
+				fidelity: "full",
+				model_visible: 1,
+				valid_states: JSON.stringify([
+					"full",
+					"proposed",
+					"pass",
+					"rejected",
+					"error",
+					"info",
+				]),
+				category: "result",
+			});
+		}
+	}
+}
+
+async function loadEnvPlugins(hooks) {
+	for (const [key, value] of Object.entries(process.env)) {
+		if (!key.startsWith("RUMMY_PLUGIN_") || !value) continue;
+		const name = key.replace("RUMMY_PLUGIN_", "").toLowerCase();
+		try {
+			const { default: Plugin } = await import(value);
+			if (typeof Plugin?.register === "function") {
+				await Plugin.register(hooks);
+			} else if (typeof Plugin === "function") {
+				const ctx = new PluginContext(name, hooks);
+				new Plugin(ctx);
+				instances.set(name, ctx);
+			}
+			console.log(`[RUMMY] Plugin ${name}: ${value}`);
+		} catch (err) {
+			console.warn(`[RUMMY] Plugin ${name} (${value}): ${err.message}`);
+		}
+	}
+}
+
+async function scanDir(dir, hooks, isRoot = false) {
+	if (!existsSync(dir)) return;
+
+	let dirStats;
+	try {
+		dirStats = await stat(dir);
+	} catch (_err) {
+		return;
+	}
+
+	if (!dirStats.isDirectory()) {
+		if (process.env.RUMMY_DEBUG === "true") {
+			console.error(
+				`[RUMMY] Cannot scan plugin directory (not a directory): ${dir}`,
+			);
+		}
+		return;
+	}
+
+	let entries;
+	try {
+		entries = await readdir(dir);
+	} catch (err) {
+		if (process.env.RUMMY_DEBUG === "true") {
+			console.error(`[RUMMY] Failed to read directory ${dir}:`, err.message);
+		}
+		return;
+	}
+
+	for (const name of entries) {
+		if (name.endsWith(".test.js")) continue;
+
+		const fullPath = join(dir, name);
+		let stats;
+		try {
+			stats = await stat(fullPath);
+		} catch (_err) {
+			continue;
+		}
+
+		if (stats.isFile() && name.endsWith(".js")) {
+			if (name === "index.js" || name === `${basename(dir)}.js`) {
+				await loadPlugin(fullPath, hooks);
+			} else if (isRoot && name !== "index.js") {
+				await loadPlugin(fullPath, hooks);
+			}
+		} else if (stats.isDirectory()) {
+			await scanDir(fullPath, hooks, false);
+		}
+	}
+}
+
+async function loadPlugin(filePath, hooks) {
+	try {
+		const url = pathToFileURL(filePath).href;
+		const { default: Plugin } = await import(url);
+
+		if (typeof Plugin?.register === "function") {
+			await Plugin.register(hooks);
+		} else if (typeof Plugin === "function") {
+			const name = basename(filePath, ".js");
+			const ctx = new PluginContext(name, hooks);
+			const _instance = new Plugin(ctx);
+			instances.set(name, ctx);
+		}
+	} catch (err) {
+		if (process.env.RUMMY_DEBUG === "true") {
+			console.error(`[RUMMY] Plugin load failed at ${filePath}:`, err);
+		}
+	}
+}

package/src/plugins/instructions/README.md

@@ -0,0 +1,11 @@
+# instructions
+
+Projects the system prompt instructions into model context.
+
+## Registration
+
+- **Projection**: `onProject("instructions", ...)` — no tool handler.
+
+## Behavior
+
+Replaces the `[%TOOLS%]` placeholder in the prompt body with the `tools` attribute. Appends tool descriptions and persona text when present in attributes.