@possumtech/rummy 2.0.0 → 2.1.0

package/migrations/001_initial_schema.sql

@@ -218,7 +218,8 @@ CREATE TABLE IF NOT EXISTS turn_context (
 		state IN ('proposed', 'streaming', 'resolved', 'failed', 'cancelled')
 	)
 	, outcome TEXT
-	, visibility TEXT NOT NULL CHECK (visibility IN ('visible', 'summarized'))
+	-- 'archived' permitted; see prompt plugin README for the exception.
+	, visibility TEXT NOT NULL CHECK (visibility IN ('visible', 'summarized', 'archived'))
 	, body TEXT NOT NULL DEFAULT ''
 	, attributes JSON NOT NULL DEFAULT '{}' CHECK (json_valid(attributes))
 	, category TEXT NOT NULL DEFAULT 'logging'

package/package.json

@@ -1,12 +1,13 @@
 {
 	"name": "@possumtech/rummy",
-	"version": "2.0.0",
+	"version": "2.1.0",
 	"description": "Relational Unknowns Memory Management Yoke",
 	"keywords": [
 		"llm"
 	],
 	"bin": {
-		"rummy": "./bin/rummy.js"
+		"rummy": "./bin/rummy.js",
+		"rummy-cli": "./src/plugins/cli/bin.js"
 	},
 	"publishConfig": {
 		"access": "public"
@@ -41,16 +42,19 @@
 		"test:intg": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test-force-exit --test $(find test/integration -name '*.test.js')",
 		"test:e2e": "mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test-force-exit --test-reporter=spec --test $(find test/e2e -name '*.test.js') 2>&1 | tee /tmp/rummy_test_diag/e2e_$(date +%Y%m%dT%H%M%S).log",
 		"test:live": "mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test-force-exit --test-reporter=spec --test $(find test/live -name '*.test.js') 2>&1 | tee /tmp/rummy_test_diag/live_$(date +%Y%m%dT%H%M%S).log",
-		"test:clean": "rm -rf test/lme/results test/mab/results test/tmp /tmp/rummy_test_diag /tmp/rummy_test_*.db /tmp/rummy_test_*.db-shm /tmp/rummy_test_*.db-wal && echo 'Test artifacts cleaned.'",
-		"test:mab:get": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/mab/download.js",
-		"test:mab": "bash -c 'mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/mab/runner.js \"$@\" 2>&1 | tee /tmp/rummy_test_diag/mab_$(date +%Y%m%dT%H%M%S).log' --",
-		"test:grok": "bash -c 'mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --env-file-if-exists=.env.grok test/mab/runner.js \"$@\" 2>&1 | tee /tmp/rummy_test_diag/mab_grok_$(date +%Y%m%dT%H%M%S).log' --",
-		"test:mab:taxonomy": "bash -c 'mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/mab/runner.js --split Conflict_Resolution --row 0 --no-questions 2>&1 | tee /tmp/rummy_test_diag/taxonomy_$(date +%Y%m%dT%H%M%S).log' --",
-		"test:grok:taxonomy": "bash -c 'mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --env-file-if-exists=.env.grok test/mab/runner.js --split Conflict_Resolution --row 0 --no-questions 2>&1 | tee /tmp/rummy_test_diag/taxonomy_grok_$(date +%Y%m%dT%H%M%S).log' --",
+		"test:clean": "rm -rf test/lme/results test/swe/results test/swe/repos test/tmp /tmp/rummy_test_diag /tmp/rummy_test_*.db /tmp/rummy_test_*.db-shm /tmp/rummy_test_*.db-wal && echo 'Test artifacts cleaned.'",
 		"test:lme:get": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/lme/download.js",
 		"test:lme": "bash -c 'mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/lme/runner.js \"$@\" 2>&1 | tee /tmp/rummy_test_diag/lme_$(date +%Y%m%dT%H%M%S).log' --",
-		"test:mab:clean": "rm -rf test/mab/results/*/",
+		"test:swe:setup": "bash test/swe/setup.sh",
+		"test:swe:get": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/swe/download.js",
+		"test:swe": "bash -c 'mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/swe/runner.js \"$@\" 2>&1 | tee /tmp/rummy_test_diag/swe_$(date +%Y%m%dT%H%M%S).log' --",
+		"test:swe:eval": "bash -c 'cd test/swe && source .venv/bin/activate && python evaluate.py \"$@\"' --",
+		"test:swe:baseline": "bash -c 'cd test/swe && source .venv/bin/activate && python baseline.py \"$@\"' --",
 		"test:lme:clean": "rm -rf test/lme/results/*/",
+		"test:swe:clean": "rm -rf test/swe/results/*/ test/swe/repos/",
+		"test:tbench:setup": "bash -c 'set -a; source .env.tbench; set +a; bash test/tbench/setup.sh'",
+		"test:tbench": "bash -c 'mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.tbench test/tbench/runner.js \"$@\" 2>&1 | tee /tmp/rummy_test_diag/tbench_$(date +%Y%m%dT%H%M%S).log' --",
+		"test:tbench:clean": "rm -rf test/tbench/results/*/",
 		"test:clear": "rm -rf /tmp/rummy_test_diag /tmp/rummy_test_*.db /tmp/rummy_test_*.db-shm /tmp/rummy_test_*.db-wal /tmp/rummy-stories-*",
 		"test:demo": "node --env-file-if-exists=.env.example --env-file-if-exists=.env bin/demo.js",
 		"test:spec": "node test/spec-coverage.js"

package/scriptify/ask_run.js

@@ -0,0 +1,77 @@
+/**
+ * Inject a follow-up question into an existing LME run and print the answer.
+ *
+ * Usage: node scriptify/ask_run.js <db_path> <run_alias> "your question"
+ *
+ * Reuses the run's full ingested context so the model answers with all
+ * its accumulated knowledge. Used as a debugging tool to interrogate
+ * the model's reasoning after a benchmark completes.
+ */
+import TestDb from "../test/helpers/TestDb.js";
+import TestServer from "../test/helpers/TestServer.js";
+import RpcClient from "../test/helpers/RpcClient.js";
+
+const [, , dbPath, alias, ...questionParts] = process.argv;
+const question = questionParts.join(" ");
+
+if (!dbPath || !alias || !question) {
+	console.error(
+		'Usage: node scriptify/ask_run.js <db_path> <run_alias> "your question"',
+	);
+	process.exit(1);
+}
+
+const tdb = await TestDb.createAt(dbPath);
+const tserver = await TestServer.start(tdb);
+const client = new RpcClient(tserver.url);
+await client.connect();
+await client.call("rummy/hello", {
+	name: "ask_run",
+	projectRoot: "/tmp/rummy-lme",
+});
+
+console.log(`Asking ${alias}: ${question}\n`);
+
+const TERMINAL = [200, 204, 413, 422, 499, 500];
+const startRes = await client.call("set", {
+	path: `run://${alias}`,
+	body: question,
+	attributes: {
+		model: "grok",
+		mode: "ask",
+		noRepo: true,
+		noInteraction: true,
+		noWeb: true,
+		noProposals: true,
+	},
+});
+
+const deadline = Date.now() + 600_000;
+while (Date.now() < deadline) {
+	const row = await tdb.db.get_run_by_alias.get({ alias });
+	if (TERMINAL.includes(row.status)) break;
+	await new Promise((r) => setTimeout(r, 500));
+}
+
+const runRow = await tdb.db.get_run_by_alias.get({ alias });
+const entries = await tdb.db.get_known_entries.all({ run_id: runRow.id });
+const reasoning = entries
+	.filter((e) => e.scheme === "reasoning")
+	.toSorted((a, b) => b.turn - a.turn)[0];
+const assistant = entries
+	.filter((e) => e.scheme === "assistant")
+	.toSorted((a, b) => b.turn - a.turn)[0];
+
+if (reasoning) {
+	console.log("=== REASONING ===");
+	console.log(reasoning.body);
+	console.log("");
+}
+if (assistant) {
+	console.log("=== ANSWER ===");
+	console.log(assistant.body);
+}
+
+await client.close();
+await tserver.stop();
+await tdb.cleanup();

package/scriptify/cache_probe.js

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+// Probe llama-server cache behavior. Send variations of the same request
+// and inspect cached_tokens in the response usage block to determine
+// whether caching is token-prefix or message-hash level.
+
+const URL = "http://127.0.0.1:11435/v1/chat/completions";
+const MODEL = "gemma-4-26B-A4B-it-UD-Q3_K_XL.gguf";
+
+async function probe(label, system, user) {
+	const body = {
+		model: MODEL,
+		messages: [
+			{ role: "system", content: system },
+			{ role: "user", content: user },
+		],
+		think: true,
+		temperature: 0.5,
+	};
+	const res = await fetch(URL, {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify(body),
+	});
+	const data = await res.json();
+	const u = data.usage || {};
+	const cached =
+		u.prompt_tokens_details?.cached_tokens ??
+		u.cached_tokens ??
+		0;
+	console.log(
+		`[${label}] prompt_tokens=${u.prompt_tokens ?? "?"} cached_tokens=${cached} system_chars=${system.length} user_chars=${user.length}`,
+	);
+}
+
+const STATIC_SYSTEM_BASE = `You are a helpful assistant.
+
+Tools available:
+- foo: does foo
+- bar: does bar
+- baz: does baz
+
+Always be concise.`;
+
+const ADDITION_A = "\n\n<context>\n<known path=\"k1\">first known fact</known>\n</context>";
+const ADDITION_B = "\n\n<context>\n<known path=\"k1\">first known fact</known>\n<known path=\"k2\">second known fact</known>\n</context>";
+const ADDITION_C = "\n\n<context>\n<known path=\"k2\">second known fact</known>\n<known path=\"k1\">first known fact</known>\n</context>";
+
+const USER_A = "Hello.";
+
+console.log("=== Run 1: baseline (cold, then immediate repeat) ===");
+await probe("1a baseline cold", STATIC_SYSTEM_BASE, USER_A);
+await probe("1b same-as-1a    ", STATIC_SYSTEM_BASE, USER_A);
+
+console.log("\n=== Run 2: same base, then base + appended context (prefix unchanged) ===");
+await probe("2a base only     ", STATIC_SYSTEM_BASE, USER_A);
+await probe("2b base + 1 entry", STATIC_SYSTEM_BASE + ADDITION_A, USER_A);
+await probe("2c base + 2 entries", STATIC_SYSTEM_BASE + ADDITION_B, USER_A);
+
+console.log("\n=== Run 3: prefix change (entries reordered, same body) ===");
+await probe("3a base + 2 entries (k1,k2)", STATIC_SYSTEM_BASE + ADDITION_B, USER_A);
+await probe("3b base + 2 entries (k2,k1) reordered", STATIC_SYSTEM_BASE + ADDITION_C, USER_A);
+
+console.log("\n=== Run 4: small mid-prefix change ===");
+const MIDDIFF = STATIC_SYSTEM_BASE.replace("baz", "qux");
+await probe("4a stable base   ", STATIC_SYSTEM_BASE, USER_A);
+await probe("4b changed baz→qux", MIDDIFF, USER_A);

package/scriptify/cache_probe_grok.js

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+// Same probe as cache_probe.js but against OpenRouter's grok endpoint.
+// If cached_tokens behaves sanely (incremental matches preserve prefix),
+// then llama-server's behavior was the local anomaly.
+
+const URL = `${process.env.OPENROUTER_BASE_URL || "https://openrouter.ai/api/v1"}/chat/completions`;
+const MODEL = "x-ai/grok-4.1-fast";
+
+if (!process.env.OPENROUTER_API_KEY) {
+	console.error("OPENROUTER_API_KEY required");
+	process.exit(1);
+}
+
+async function probe(label, system, user) {
+	const body = {
+		model: MODEL,
+		messages: [
+			{ role: "system", content: system },
+			{ role: "user", content: user },
+		],
+		include_reasoning: true,
+		temperature: 0.5,
+	};
+	const res = await fetch(URL, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			Authorization: `Bearer ${process.env.OPENROUTER_API_KEY}`,
+		},
+		body: JSON.stringify(body),
+	});
+	const data = await res.json();
+	const u = data.usage || {};
+	const cached =
+		u.prompt_tokens_details?.cached_tokens ??
+		u.cached_tokens ??
+		u.cache_read_input_tokens ??
+		0;
+	console.log(
+		`[${label}] prompt_tokens=${u.prompt_tokens ?? "?"} cached_tokens=${cached} system_chars=${system.length}`,
+	);
+}
+
+const STATIC_SYSTEM_BASE = `You are a helpful assistant.
+
+Tools available:
+- foo: does foo
+- bar: does bar
+- baz: does baz
+
+Always be concise.`;
+
+const ADDITION_A = "\n\n<context>\n<known path=\"k1\">first known fact</known>\n</context>";
+const ADDITION_B = "\n\n<context>\n<known path=\"k1\">first known fact</known>\n<known path=\"k2\">second known fact</known>\n</context>";
+const ADDITION_C = "\n\n<context>\n<known path=\"k2\">second known fact</known>\n<known path=\"k1\">first known fact</known>\n</context>";
+
+const USER = "Hello.";
+
+console.log("=== Run 1: baseline (cold, then immediate repeat) ===");
+await probe("1a baseline cold", STATIC_SYSTEM_BASE, USER);
+await probe("1b same-as-1a   ", STATIC_SYSTEM_BASE, USER);
+
+console.log("\n=== Run 2: appended context (prefix unchanged) ===");
+await probe("2a base + 1     ", STATIC_SYSTEM_BASE + ADDITION_A, USER);
+await probe("2b base + 2     ", STATIC_SYSTEM_BASE + ADDITION_B, USER);
+
+console.log("\n=== Run 3: reordered (entries shuffled) ===");
+await probe("3a (k1,k2)      ", STATIC_SYSTEM_BASE + ADDITION_B, USER);
+await probe("3b (k2,k1)      ", STATIC_SYSTEM_BASE + ADDITION_C, USER);
+
+console.log("\n=== Run 4: mid-prefix character change ===");
+const MIDDIFF = STATIC_SYSTEM_BASE.replace("baz", "qux");
+await probe("4a stable base  ", STATIC_SYSTEM_BASE, USER);
+await probe("4b baz→qux      ", MIDDIFF, USER);

package/service.js

@@ -43,11 +43,7 @@ if (!rummyHome) {
 		}
 		for (const path of [homeExample, homeEnv]) {
 			if (!existsSync(path)) continue;
-			try {
-				process.loadEnvFile(path);
-			} catch (err) {
-				console.warn(`[RUMMY] Failed to load ${path}: ${err.message}`);
-			}
+			process.loadEnvFile(path);
 		}
 	}
 }
@@ -136,11 +132,21 @@ async function main() {
 		}
 	}
 
-	// 6b. Database Hygiene
+	// 6b. Database Hygiene — opt-in via RUMMY_RETENTION_DAYS.
 	const { statSync } = await import("node:fs");
-	try {
+	const retentionRaw = process.env.RUMMY_RETENTION_DAYS;
+	if (retentionRaw == null || retentionRaw === "") {
+		const dbSizeMB = (statSync(dbPath).size / 1024 / 1024).toFixed(2);
+		console.log(`[RUMMY] DB size: ${dbSizeMB}MB`);
+	} else {
+		const retentionDays = Number.parseInt(retentionRaw, 10);
+		if (!Number.isInteger(retentionDays) || retentionDays < 0) {
+			throw new Error(
+				`Invalid RUMMY_RETENTION_DAYS=${JSON.stringify(retentionRaw)} ` +
+					"(expected non-negative integer)",
+			);
+		}
 		const dbSizeBefore = statSync(dbPath).size;
-		const retentionDays = Number.parseInt(process.env.RUMMY_RETENTION_DAYS, 10);
 		await db.purge_old_runs.run({ retention_days: retentionDays });
 		const dbSizeAfter = statSync(dbPath).size;
 		const dbSizeMB = (dbSizeAfter / 1024 / 1024).toFixed(2);
@@ -153,8 +159,6 @@ async function main() {
 		if (dbSizeAfter > 100 * 1024 * 1024) {
 			console.warn(`[RUMMY] WARNING: Database exceeds 100MB. Consider manual cleanup.`);
 		}
-	} catch (err) {
-		console.warn(`[RUMMY] Hygiene skipped: ${err.message}`);
 	}
 
 	// 6b. Abort stuck runs (can't be running if the server just started)
@@ -164,8 +168,15 @@ async function main() {
 		console.log(`[RUMMY] Recovered ${aborted.changes} stuck run(s)`);
 	}
 
+	// 6c. Boot complete — DB open, plugins inited, models loaded,
+	// hygiene done. Plugins that need a one-shot post-boot action
+	// (e.g. the cli plugin firing a programmatic run) subscribe to
+	// this event. Fires BEFORE SocketServer so RPC clients can't
+	// race a one-shot run still being set up.
+	await hooks.boot.completed.emit({ db, hooks });
+
 	// 7. Start RPC Server
-	const port = Number.parseInt(process.env.PORT);
+	const port = Number.parseInt(process.env.RUMMY_PORT);
 	const server = new SocketServer(db, { port, hooks });
 
 	server.on("error", (err) => {