@possumtech/rummy 0.2.1

package/migrations/001_initial_schema.sql

@@ -0,0 +1,226 @@
+-- INIT: enable_mmap
+PRAGMA mmap_size = $mmap_size;
+
+-- INIT: initial_schema
+
+-- Scheme registry: single source of truth for all scheme metadata.
+-- fidelity: 'full' = always visible, 'turn' = visible when turn>0, 'null' = never visible.
+-- valid_states: JSON array of allowed state values for this scheme.
+CREATE TABLE IF NOT EXISTS schemes (
+	name TEXT PRIMARY KEY
+	, fidelity TEXT NOT NULL CHECK (fidelity IN ('full', 'turn', 'null'))
+	, model_visible BOOLEAN NOT NULL DEFAULT 1
+	, valid_states TEXT NOT NULL
+	, category TEXT
+);
+
+-- Schemes are registered by plugins at startup via core.registerScheme().
+-- Audit schemes are bootstrapped here since they have no plugin owner.
+
+-- Projects: top-level organizational unit.
+CREATE TABLE IF NOT EXISTS projects (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, name TEXT UNIQUE NOT NULL
+	, project_root TEXT
+	, config_path TEXT
+	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Models: available LLM configurations.
+-- Populated from RUMMY_MODEL_* env vars at startup.
+CREATE TABLE IF NOT EXISTS models (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, alias TEXT UNIQUE NOT NULL
+	, actual TEXT NOT NULL
+	, context_length INTEGER
+	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Runs: execution units belonging to a project.
+-- Each run has its own config (temperature, persona, context_limit).
+CREATE TABLE IF NOT EXISTS runs (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, project_id INTEGER NOT NULL REFERENCES projects (id) ON DELETE CASCADE
+	, parent_run_id INTEGER REFERENCES runs (id) ON DELETE SET NULL
+	, model TEXT
+	, status TEXT NOT NULL DEFAULT 'queued' CHECK (
+		status IN ('queued', 'running', 'proposed', 'completed', 'failed', 'aborted')
+	)
+	, alias TEXT NOT NULL UNIQUE
+	, temperature REAL CHECK (
+		temperature IS NULL OR (temperature >= 0 AND temperature <= 2)
+	)
+	, persona TEXT
+	, context_limit INTEGER CHECK (context_limit IS NULL OR context_limit >= 1024)
+	, next_turn INTEGER NOT NULL DEFAULT 1 CHECK (next_turn >= 1)
+	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_runs_alias ON runs (alias);
+CREATE INDEX IF NOT EXISTS idx_runs_project ON runs (project_id);
+
+-- Turns: usage stats and sequencing (operational, not model-facing)
+CREATE TABLE IF NOT EXISTS turns (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE
+	, sequence INTEGER NOT NULL CHECK (sequence >= 1)
+	, prompt_tokens INTEGER NOT NULL DEFAULT 0 CHECK (prompt_tokens >= 0)
+	, cached_tokens INTEGER NOT NULL DEFAULT 0 CHECK (cached_tokens >= 0)
+	, completion_tokens INTEGER NOT NULL DEFAULT 0 CHECK (completion_tokens >= 0)
+	, reasoning_tokens INTEGER NOT NULL DEFAULT 0 CHECK (reasoning_tokens >= 0)
+	, total_tokens INTEGER NOT NULL DEFAULT 0 CHECK (total_tokens >= 0)
+	, cost REAL NOT NULL DEFAULT 0 CHECK (cost >= 0)
+	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS idx_turns_run_seq ON turns (run_id, sequence);
+
+-- File constraints: client-set visibility rules, project-scoped.
+-- Persists across runs. Orthogonal to fidelity.
+CREATE TABLE IF NOT EXISTS file_constraints (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, project_id INTEGER NOT NULL REFERENCES projects (id) ON DELETE CASCADE
+	, pattern TEXT NOT NULL
+	, visibility TEXT NOT NULL CHECK (visibility IN ('active', 'readonly', 'ignore'))
+	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+	, UNIQUE (project_id, pattern)
+);
+CREATE INDEX IF NOT EXISTS idx_file_constraints_project
+ON file_constraints (project_id);
+
+-- Known K/V Store: the unified state machine.
+-- Files, knowledge, tool results, audit — everything is a keyed entry.
+-- scheme: derived from path via schemeOf(). Generated column.
+-- State validated by trigger against schemes.valid_states.
+CREATE TABLE IF NOT EXISTS known_entries (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE
+	, turn INTEGER NOT NULL DEFAULT 0 CHECK (turn >= 0)
+	, path TEXT NOT NULL
+	, body TEXT NOT NULL DEFAULT ''
+	, scheme TEXT GENERATED ALWAYS AS (schemeOf(path)) STORED
+	, state TEXT NOT NULL
+	, hash TEXT
+	, attributes JSON NOT NULL DEFAULT '{}' CHECK (json_valid(attributes))
+	, tokens INTEGER NOT NULL DEFAULT 0 CHECK (tokens >= 0)
+	, tokens_full INTEGER NOT NULL DEFAULT 0 CHECK (tokens_full >= 0)
+	, refs INTEGER NOT NULL DEFAULT 0 CHECK (refs >= 0)
+	, write_count INTEGER NOT NULL DEFAULT 1 CHECK (write_count >= 1)
+	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+	, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_known_entries_run_path
+ON known_entries (run_id, path);
+CREATE INDEX IF NOT EXISTS idx_known_entries_scheme_state
+ON known_entries (run_id, scheme, state);
+CREATE INDEX IF NOT EXISTS idx_known_entries_turn
+ON known_entries (run_id, turn);
+
+-- Validate state against schemes.valid_states on insert.
+CREATE TRIGGER IF NOT EXISTS trg_known_entry_state_insert
+BEFORE INSERT ON known_entries
+FOR EACH ROW
+BEGIN
+	SELECT RAISE(ABORT, 'invalid state for scheme')
+	WHERE NOT EXISTS (
+		SELECT 1
+		FROM schemes AS s, json_each(s.valid_states) AS j
+		WHERE
+			s.name = COALESCE(schemeOf(NEW.path), 'file')
+			AND j.value = NEW.state
+	);
+END;
+
+-- Validate state against schemes.valid_states on update.
+CREATE TRIGGER IF NOT EXISTS trg_known_entry_state_update
+BEFORE UPDATE OF state ON known_entries
+FOR EACH ROW
+WHEN OLD.state != NEW.state
+BEGIN
+	SELECT RAISE(ABORT, 'invalid state for scheme')
+	WHERE NOT EXISTS (
+		SELECT 1
+		FROM schemes AS s, json_each(s.valid_states) AS j
+		WHERE
+			s.name = COALESCE(schemeOf(NEW.path), 'file')
+			AND j.value = NEW.state
+	);
+END;
+
+-- UNRESOLVED VIEW: all entries awaiting user action
+CREATE VIEW IF NOT EXISTS v_unresolved AS
+SELECT
+	run_id
+	, path
+	, body
+	, attributes
+	, turn
+FROM known_entries
+WHERE state = 'proposed';
+
+-- Turn context: materialized snapshot of what the model sees each turn.
+-- known_entries is the warehouse. turn_context is the shipment.
+CREATE TABLE IF NOT EXISTS turn_context (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE
+	, turn INTEGER NOT NULL CHECK (turn >= 1)
+	, ordinal INTEGER NOT NULL CHECK (ordinal >= 0)
+	, path TEXT NOT NULL
+	, scheme TEXT GENERATED ALWAYS AS (schemeOf(path)) STORED
+	, fidelity TEXT NOT NULL CHECK (fidelity IN ('full', 'summary', 'index'))
+	, state TEXT NOT NULL DEFAULT 'full'
+	, body TEXT NOT NULL DEFAULT ''
+	, tokens INTEGER NOT NULL DEFAULT 0 CHECK (tokens >= 0)
+	, attributes JSON NOT NULL DEFAULT '{}' CHECK (json_valid(attributes))
+	, category TEXT NOT NULL DEFAULT 'result'
+	, source_turn INTEGER DEFAULT 0
+);
+CREATE INDEX IF NOT EXISTS idx_turn_context_run_turn
+ON turn_context (run_id, turn);
+
+-- Enforce valid run state transitions.
+-- completed → running: continuation (new turn on finished run)
+CREATE TRIGGER IF NOT EXISTS trg_run_state_transition
+BEFORE UPDATE OF status ON runs
+FOR EACH ROW
+WHEN OLD.status != NEW.status
+BEGIN
+	SELECT RAISE(ABORT, 'invalid run state transition')
+	WHERE NOT (
+		(OLD.status = 'queued' AND NEW.status IN ('running', 'aborted'))
+		OR (OLD.status = 'running' AND NEW.status IN ('proposed', 'completed', 'failed', 'aborted'))
+		OR (OLD.status = 'proposed' AND NEW.status IN ('running', 'completed', 'aborted'))
+		OR (OLD.status = 'completed' AND NEW.status IN ('running', 'aborted'))
+		OR (OLD.status = 'failed' AND NEW.status IN ('running', 'aborted'))
+		OR (OLD.status = 'aborted' AND NEW.status IN ('running'))
+	);
+END;
+
+-- Prompt queue. All prompts flow through here. Worker consumes FIFO per run.
+CREATE TABLE IF NOT EXISTS prompt_queue (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE
+	, mode TEXT NOT NULL CHECK (mode IN ('ask', 'act'))
+	, model TEXT
+	, prompt TEXT NOT NULL
+	, config JSON
+	, status TEXT NOT NULL DEFAULT 'pending'
+	CHECK (status IN ('pending', 'active', 'completed', 'aborted'))
+	, result JSON
+	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_prompt_queue_run ON prompt_queue (run_id, status);
+
+-- RPC audit log. Every call recorded unconditionally.
+CREATE TABLE IF NOT EXISTS rpc_log (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, project_id INTEGER REFERENCES projects (id) ON DELETE CASCADE
+	, method TEXT NOT NULL
+	, rpc_id INTEGER
+	, params JSON
+	, result JSON
+	, error TEXT
+	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS idx_rpc_log_project ON rpc_log (project_id);
+CREATE INDEX IF NOT EXISTS idx_rpc_log_method ON rpc_log (method);

package/package.json

@@ -0,0 +1,54 @@
+{
+	"name": "@possumtech/rummy",
+	"version": "0.2.1",
+	"description": "Relational Unknowns Memory Management Yoke",
+	"keywords": [
+		"llm"
+	],
+	"publishConfig": {
+		"access": "public"
+	},
+	"homepage": "https://github.com/possumtech/rummy#readme",
+	"bugs": {
+		"url": "https://github.com/possumtech/rummy/issues"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/possumtech/rummy.git"
+	},
+	"license": "MIT",
+	"author": "@wikitopian",
+	"engines": {
+		"node": ">=25",
+		"npm": ">=11"
+	},
+	"type": "module",
+	"main": "service.js",
+	"scripts": {
+		"start": "node --env-file-if-exists=.env.example --env-file-if-exists=.env service.js",
+		"dev": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.dev --watch service.js",
+		"debug": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.debug --inspect service.js",
+		"lint": "biome check src test migrations --fix --unsafe && npm run lint:sql",
+		"lint:sql": ". .venv/bin/activate && sqlfluff lint .",
+		"fix:sql": ". .venv/bin/activate && sqlfluff fix . --force",
+		"test": "npm run lint && npm run test:unit && npm run test:intg",
+		"test:all": "npm run lint && npm run test:unit && npm run test:intg && npm run test:e2e",
+		"test:unit": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --experimental-test-coverage --test-coverage-lines=70 --test-coverage-branches=70 --test-coverage-functions=70 --test-concurrency=1 --test $(find src -name '*.test.js')",
+		"test:intg": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test $(find test/integration -name '*.test.js')",
+		"test:e2e": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test-force-exit --test-reporter=spec --test $(find test/e2e -name '*.test.js')",
+		"test:live": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test-force-exit --test-reporter=spec --test $(find test/live -name '*.test.js')"
+	},
+	"devDependencies": {
+		"@biomejs/biome": "^2.4.6"
+	},
+	"dependencies": {
+		"@possumtech/sqlrite": "^3.1.0",
+		"htmlparser2": "^12.0.0",
+		"tiktoken": "^1.0.22",
+		"ws": "^8.19.0"
+	},
+	"optionalDependencies": {
+		"@possumtech/rummy.repo": "^0.0.4",
+		"@possumtech/rummy.web": "^0.0.7"
+	}
+}

package/service.js

@@ -0,0 +1,143 @@
+import { mkdirSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawnSync } from "node:child_process";
+
+// Helper to expand ~ in paths since node --env-file doesn't do it
+// 0. Pre-flight Check: Environment and Dependencies
+const rummyHome = process.env.RUMMY_HOME;
+
+if (!rummyHome) {
+	console.error("RUMMY Configuration Error: RUMMY_HOME is not defined in environment.");
+	process.exit(1);
+}
+
+// Check for optional system dependencies
+const gitCheck = spawnSync("git", ["--version"]);
+if (gitCheck.error || gitCheck.status !== 0) {
+	console.warn("[RUMMY] WARNING: 'git' not found. File tracking will use manual activation only.");
+}
+
+let SqlRite, SocketServer, registerPlugins, createHooks, RpcRegistry;
+try {
+	SqlRite = (await import("@possumtech/sqlrite")).default;
+	SocketServer = (await import("./src/server/SocketServer.js")).default;
+	const pluginIndex = await import("./src/plugins/index.js");
+	registerPlugins = pluginIndex.registerPlugins;
+	var initPlugins = pluginIndex.initPlugins;
+	createHooks = (await import("./src/hooks/Hooks.js")).default;
+	RpcRegistry = (await import("./src/server/RpcRegistry.js")).default;
+} catch (err) {
+	if (err.code === "ERR_MODULE_NOT_FOUND") {
+		console.error("RUMMY Dependency Error: node_modules not found or incomplete.");
+		console.error("Please run: npm install");
+		process.exit(1);
+	}
+	throw err;
+}
+
+async function main() {
+	// 1. Initialize Hooks (Agnostic Engine)
+	const debug = process.env.RUMMY_DEBUG === "true";
+	const hooks = createHooks(debug);
+	hooks.rpc.registry = new RpcRegistry();
+
+	// 2. Resolve Directories
+	const userPluginsDir = join(rummyHome, "plugins");
+	const pluginsDir = fileURLToPath(new URL("./src/plugins", import.meta.url));
+
+	// 3. Ensure Directory Structure
+	mkdirSync(userPluginsDir, { recursive: true });
+
+	// 4. Register Plugins
+	await registerPlugins([pluginsDir, userPluginsDir], hooks);
+
+	// 5. Bootstrap Persistence
+	const dbPath = process.env.RUMMY_DB_PATH || join(rummyHome, "rummy.db");
+	const functionsDir = fileURLToPath(new URL("./src/sql/functions", import.meta.url));
+	const sqlFunctions = readdirSync(functionsDir)
+		.filter((f) => f.endsWith(".js") && !f.endsWith(".test.js"))
+		.map((f) => join(functionsDir, f));
+
+	const mmapMb = Number.parseInt(process.env.RUMMY_MMAP_MB || "0", 10);
+	const db = await SqlRite.open({
+		path: dbPath,
+		dir: ["migrations", "src"],
+		functions: sqlFunctions,
+		params: {
+			mmap_size: mmapMb * 1024 * 1024,
+		},
+	});
+
+	// 6. Initialize plugins (inject DB, register schemes)
+	await initPlugins(db, null, hooks);
+
+	// 7. Bootstrap models from env vars
+	{
+		const modelAliases = [];
+		for (const key of Object.keys(process.env)) {
+			if (!key.startsWith("RUMMY_MODEL_")) continue;
+			const alias = key.replace("RUMMY_MODEL_", "");
+			const actual = process.env[key];
+			await db.upsert_model.get({
+				alias,
+				actual,
+				context_length: null,
+			});
+			modelAliases.push(alias);
+		}
+		if (modelAliases.length > 0) {
+			console.log(`[RUMMY] Models: ${modelAliases.join(", ")}`);
+		}
+	}
+
+	// 6b. Database Hygiene
+	const { statSync } = await import("node:fs");
+	try {
+		const dbSizeBefore = statSync(dbPath).size;
+		const retentionDays = Number.parseInt(process.env.RUMMY_RETENTION_DAYS || "31", 10);
+		await db.purge_old_runs.run({ retention_days: retentionDays });
+		const dbSizeAfter = statSync(dbPath).size;
+		const dbSizeMB = (dbSizeAfter / 1024 / 1024).toFixed(2);
+		const freed = dbSizeBefore - dbSizeAfter;
+		if (freed > 0) {
+			console.log(`[RUMMY] Hygiene: freed ${(freed / 1024).toFixed(1)}KB, DB is ${dbSizeMB}MB`);
+		} else {
+			console.log(`[RUMMY] DB size: ${dbSizeMB}MB`);
+		}
+		if (dbSizeAfter > 100 * 1024 * 1024) {
+			console.warn(`[RUMMY] WARNING: Database exceeds 100MB. Consider manual cleanup.`);
+		}
+	} catch (err) {
+		console.warn(`[RUMMY] Hygiene skipped: ${err.message}`);
+	}
+
+	// 6b. Abort stuck runs (can't be running if the server just started)
+	await db.reset_active_prompts.run({});
+	const aborted = await db.abort_stuck_runs.run({});
+	if (aborted.changes > 0) {
+		console.log(`[RUMMY] Recovered ${aborted.changes} stuck run(s)`);
+	}
+
+	// 7. Start RPC Server
+	const port = Number.parseInt(process.env.PORT);
+	const server = new SocketServer(db, { port, hooks });
+
+	server.on("error", (err) => {
+		if (err.code === "EADDRINUSE") {
+			console.error(`RUMMY Critical: Port ${port} is already in use.`);
+			process.exit(1);
+		}
+		throw err;
+	});
+
+	console.log(`RUMMY Service Operational`);
+	console.log(`- Home: ${rummyHome}`);
+	console.log(`- DB:   ${dbPath}`);
+	console.log(`- Port: ${port}`);
+}
+
+main().catch((err) => {
+	console.error("RUMMY Failed to boot:", err.message);
+	process.exit(1);
+});