@possumtech/rummy 0.2.7 → 0.3.0

package/migrations/001_initial_schema.sql

@@ -4,13 +4,12 @@ PRAGMA mmap_size = $mmap_size;
 -- INIT: initial_schema
 
 -- Scheme registry: single source of truth for all scheme metadata.
--- fidelity: 'full' = always visible, 'turn' = visible when turn>0, 'null' = never visible.
--- valid_states: JSON array of allowed state values for this scheme.
+-- Status codes are HTTP: 2xx success, 3xx redirect, 4xx model error, 5xx system error.
+-- No valid_states — HTTP semantics are universal.
+-- No fidelity — entries don't decide their own importance.
 CREATE TABLE IF NOT EXISTS schemes (
 	name TEXT PRIMARY KEY
-	, fidelity TEXT NOT NULL CHECK (fidelity IN ('full', 'turn', 'null'))
 	, model_visible BOOLEAN NOT NULL DEFAULT 1
-	, valid_states TEXT NOT NULL
 	, category TEXT
 );
 
@@ -37,15 +36,14 @@ CREATE TABLE IF NOT EXISTS models (
 );
 
 -- Runs: execution units belonging to a project.
--- Each run has its own config (temperature, persona, context_limit).
+-- Status uses HTTP codes: 100=queued, 102=running, 200=completed,
+-- 202=proposed, 500=failed, 499=aborted.
 CREATE TABLE IF NOT EXISTS runs (
 	id INTEGER PRIMARY KEY AUTOINCREMENT
 	, project_id INTEGER NOT NULL REFERENCES projects (id) ON DELETE CASCADE
 	, parent_run_id INTEGER REFERENCES runs (id) ON DELETE SET NULL
 	, model TEXT
-	, status TEXT NOT NULL DEFAULT 'queued' CHECK (
-		status IN ('queued', 'running', 'proposed', 'completed', 'failed', 'aborted')
-	)
+	, status INTEGER NOT NULL DEFAULT 100 CHECK (status BETWEEN 100 AND 599)
 	, alias TEXT NOT NULL UNIQUE
 	, temperature REAL CHECK (
 		temperature IS NULL OR (temperature >= 0 AND temperature <= 2)
@@ -53,17 +51,42 @@ CREATE TABLE IF NOT EXISTS runs (
 	, persona TEXT
 	, context_limit INTEGER CHECK (context_limit IS NULL OR context_limit >= 1024)
 	, next_turn INTEGER NOT NULL DEFAULT 1 CHECK (next_turn >= 1)
+	, next_loop INTEGER NOT NULL DEFAULT 1 CHECK (next_loop >= 1)
 	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
 );
 
 CREATE INDEX IF NOT EXISTS idx_runs_alias ON runs (alias);
 CREATE INDEX IF NOT EXISTS idx_runs_project ON runs (project_id);
 
+-- Loops: execution units within a run. Each ask/act call creates a loop.
+-- Status: 100=pending, 102=running, 200=completed, 202=proposed,
+-- 500=failed, 499=aborted.
+CREATE TABLE IF NOT EXISTS loops (
+	id INTEGER PRIMARY KEY AUTOINCREMENT
+	, run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE
+	, sequence INTEGER NOT NULL CHECK (sequence >= 1)
+	, mode TEXT NOT NULL CHECK (mode IN ('ask', 'act', 'panic'))
+	, model TEXT
+	, prompt TEXT NOT NULL DEFAULT ''
+	, status INTEGER NOT NULL DEFAULT 100 CHECK (status BETWEEN 100 AND 599)
+	, config JSON
+	, result JSON
+	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+	, UNIQUE (run_id, sequence)
+);
+CREATE INDEX IF NOT EXISTS idx_loops_run ON loops (run_id);
+-- Enforce at most one running loop per run.
+CREATE UNIQUE INDEX IF NOT EXISTS idx_loops_one_active
+ON loops (run_id) WHERE status = 102;
+
 -- Turns: usage stats and sequencing (operational, not model-facing)
 CREATE TABLE IF NOT EXISTS turns (
 	id INTEGER PRIMARY KEY AUTOINCREMENT
 	, run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE
+	, loop_id INTEGER NOT NULL REFERENCES loops (id) ON DELETE CASCADE
 	, sequence INTEGER NOT NULL CHECK (sequence >= 1)
+	, context_tokens INTEGER NOT NULL DEFAULT 0 CHECK (context_tokens >= 0)
+	, reasoning_content TEXT
 	, prompt_tokens INTEGER NOT NULL DEFAULT 0 CHECK (prompt_tokens >= 0)
 	, cached_tokens INTEGER NOT NULL DEFAULT 0 CHECK (cached_tokens >= 0)
 	, completion_tokens INTEGER NOT NULL DEFAULT 0 CHECK (completion_tokens >= 0)
@@ -90,15 +113,20 @@ ON file_constraints (project_id);
 -- Known K/V Store: the unified state machine.
 -- Files, knowledge, tool results, audit — everything is a keyed entry.
 -- scheme: derived from path via schemeOf(). Generated column.
--- State validated by trigger against schemes.valid_states.
+-- status: HTTP status code (2xx success, 4xx model error, 5xx system error).
+-- fidelity: visibility level, independently managed by relevance engine.
 CREATE TABLE IF NOT EXISTS known_entries (
 	id INTEGER PRIMARY KEY AUTOINCREMENT
 	, run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE
+	, loop_id INTEGER REFERENCES loops (id) ON DELETE CASCADE
 	, turn INTEGER NOT NULL DEFAULT 0 CHECK (turn >= 0)
-	, path TEXT NOT NULL
+	, path TEXT NOT NULL CHECK (length(path) <= 2048)
 	, body TEXT NOT NULL DEFAULT ''
 	, scheme TEXT GENERATED ALWAYS AS (schemeOf(path)) STORED
-	, state TEXT NOT NULL
+	, status INTEGER NOT NULL DEFAULT 200 CHECK (status BETWEEN 100 AND 599)
+	, fidelity TEXT NOT NULL DEFAULT 'full' CHECK (
+		fidelity IN ('full', 'summary', 'index', 'archive')
+	)
 	, hash TEXT
 	, attributes JSON NOT NULL DEFAULT '{}' CHECK (json_valid(attributes))
 	, tokens INTEGER NOT NULL DEFAULT 0 CHECK (tokens >= 0)
@@ -110,43 +138,14 @@ CREATE TABLE IF NOT EXISTS known_entries (
 );
 CREATE UNIQUE INDEX IF NOT EXISTS idx_known_entries_run_path
 ON known_entries (run_id, path);
-CREATE INDEX IF NOT EXISTS idx_known_entries_scheme_state
-ON known_entries (run_id, scheme, state);
+CREATE INDEX IF NOT EXISTS idx_known_entries_scheme_status
+ON known_entries (run_id, scheme, status);
 CREATE INDEX IF NOT EXISTS idx_known_entries_turn
 ON known_entries (run_id, turn);
 
--- Validate state against schemes.valid_states on insert.
-CREATE TRIGGER IF NOT EXISTS trg_known_entry_state_insert
-BEFORE INSERT ON known_entries
-FOR EACH ROW
-BEGIN
-	SELECT RAISE(ABORT, 'invalid state for scheme')
-	WHERE NOT EXISTS (
-		SELECT 1
-		FROM schemes AS s, json_each(s.valid_states) AS j
-		WHERE
-			s.name = COALESCE(schemeOf(NEW.path), 'file')
-			AND j.value = NEW.state
-	);
-END;
+-- No state validation triggers — HTTP status codes are universal.
 
--- Validate state against schemes.valid_states on update.
-CREATE TRIGGER IF NOT EXISTS trg_known_entry_state_update
-BEFORE UPDATE OF state ON known_entries
-FOR EACH ROW
-WHEN OLD.state != NEW.state
-BEGIN
-	SELECT RAISE(ABORT, 'invalid state for scheme')
-	WHERE NOT EXISTS (
-		SELECT 1
-		FROM schemes AS s, json_each(s.valid_states) AS j
-		WHERE
-			s.name = COALESCE(schemeOf(NEW.path), 'file')
-			AND j.value = NEW.state
-	);
-END;
-
--- UNRESOLVED VIEW: all entries awaiting user action
+-- UNRESOLVED VIEW: all entries awaiting user action (202 Accepted)
 CREATE VIEW IF NOT EXISTS v_unresolved AS
 SELECT
 	run_id
@@ -155,30 +154,31 @@ SELECT
 	, attributes
 	, turn
 FROM known_entries
-WHERE state = 'proposed';
+WHERE status = 202;
 
 -- Turn context: materialized snapshot of what the model sees each turn.
 -- known_entries is the warehouse. turn_context is the shipment.
 CREATE TABLE IF NOT EXISTS turn_context (
 	id INTEGER PRIMARY KEY AUTOINCREMENT
 	, run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE
+	, loop_id INTEGER REFERENCES loops (id) ON DELETE CASCADE
 	, turn INTEGER NOT NULL CHECK (turn >= 1)
 	, ordinal INTEGER NOT NULL CHECK (ordinal >= 0)
 	, path TEXT NOT NULL
 	, scheme TEXT GENERATED ALWAYS AS (schemeOf(path)) STORED
+	, status INTEGER NOT NULL DEFAULT 200 CHECK (status BETWEEN 100 AND 599)
 	, fidelity TEXT NOT NULL CHECK (fidelity IN ('full', 'summary', 'index'))
-	, state TEXT NOT NULL DEFAULT 'full'
 	, body TEXT NOT NULL DEFAULT ''
 	, tokens INTEGER NOT NULL DEFAULT 0 CHECK (tokens >= 0)
 	, attributes JSON NOT NULL DEFAULT '{}' CHECK (json_valid(attributes))
-	, category TEXT NOT NULL DEFAULT 'result'
+	, category TEXT NOT NULL DEFAULT 'logging'
 	, source_turn INTEGER DEFAULT 0
 );
 CREATE INDEX IF NOT EXISTS idx_turn_context_run_turn
 ON turn_context (run_id, turn);
 
--- Enforce valid run state transitions.
--- completed → running: continuation (new turn on finished run)
+-- Enforce valid run state transitions (HTTP status codes).
+-- 100=queued, 102=running, 200=completed, 202=proposed, 499=aborted, 500=failed.
 CREATE TRIGGER IF NOT EXISTS trg_run_state_transition
 BEFORE UPDATE OF status ON runs
 FOR EACH ROW
@@ -186,30 +186,17 @@ WHEN OLD.status != NEW.status
 BEGIN
 	SELECT RAISE(ABORT, 'invalid run state transition')
 	WHERE NOT (
-		(OLD.status = 'queued' AND NEW.status IN ('running', 'aborted'))
-		OR (OLD.status = 'running' AND NEW.status IN ('proposed', 'completed', 'failed', 'aborted'))
-		OR (OLD.status = 'proposed' AND NEW.status IN ('running', 'completed', 'aborted'))
-		OR (OLD.status = 'completed' AND NEW.status IN ('running', 'aborted'))
-		OR (OLD.status = 'failed' AND NEW.status IN ('running', 'aborted'))
-		OR (OLD.status = 'aborted' AND NEW.status IN ('running'))
+		(OLD.status = 100 AND NEW.status IN (102, 499))
+		OR (OLD.status = 102 AND NEW.status IN (200, 202, 500, 499))
+		OR (OLD.status = 202 AND NEW.status IN (102, 200, 499))
+		OR (OLD.status = 200 AND NEW.status IN (102, 499))
+		OR (OLD.status = 500 AND NEW.status IN (102, 499))
+		OR (OLD.status = 499 AND NEW.status IN (102))
 	);
 END;
 
--- Prompt queue. All prompts flow through here. Worker consumes FIFO per run.
-CREATE TABLE IF NOT EXISTS prompt_queue (
-	id INTEGER PRIMARY KEY AUTOINCREMENT
-	, run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE
-	, mode TEXT NOT NULL CHECK (mode IN ('ask', 'act'))
-	, model TEXT
-	, prompt TEXT NOT NULL
-	, config JSON
-	, status TEXT NOT NULL DEFAULT 'pending'
-	CHECK (status IN ('pending', 'active', 'completed', 'aborted'))
-	, result JSON
-	, created_at DATETIME DEFAULT CURRENT_TIMESTAMP
-);
-
-CREATE INDEX IF NOT EXISTS idx_prompt_queue_run ON prompt_queue (run_id, status);
+-- Prompt queue is the loops table (defined above).
+-- Each ask/act enqueues a loop (status=pending). Worker claims FIFO per run.
 
 -- RPC audit log. Every call recorded unconditionally.
 CREATE TABLE IF NOT EXISTS rpc_log (

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@possumtech/rummy",
-	"version": "0.2.7",
+	"version": "0.3.0",
 	"description": "Relational Unknowns Memory Management Yoke",
 	"keywords": [
 		"llm"
@@ -37,22 +37,28 @@
 		"fix:sql": ". .venv/bin/activate && sqlfluff fix . --force",
 		"test": "npm run lint && npm run test:unit && npm run test:intg",
 		"test:all": "npm run lint && npm run test:unit && npm run test:intg && npm run test:e2e",
-		"test:unit": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --experimental-test-coverage --test-coverage-lines=70 --test-coverage-branches=70 --test-coverage-functions=70 --test-concurrency=1 --test $(find src -name '*.test.js')",
+		"test:unit": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --experimental-test-coverage --test-coverage-lines=50 --test-coverage-branches=50 --test-coverage-functions=50 --test-concurrency=1 --test-force-exit --test $(find src -name '*.test.js')",
 		"test:intg": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test $(find test/integration -name '*.test.js')",
-		"test:e2e": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test-force-exit --test-reporter=spec --test $(find test/e2e -name '*.test.js')",
-		"test:live": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test-force-exit --test-reporter=spec --test $(find test/live -name '*.test.js')"
+		"test:e2e": "mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test-force-exit --test-reporter=spec --test $(find test/e2e -name '*.test.js') 2>&1 | tee /tmp/rummy_test_diag/e2e_$(date +%Y%m%dT%H%M%S).log",
+		"test:live": "mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test --test-concurrency=1 --test-force-exit --test-reporter=spec --test $(find test/live -name '*.test.js') 2>&1 | tee /tmp/rummy_test_diag/live_$(date +%Y%m%dT%H%M%S).log",
+		"test:clean": "rm -rf test/lme/results test/mab/results test/tmp /tmp/rummy_test_diag /tmp/rummy_test_*.db /tmp/rummy_test_*.db-shm /tmp/rummy_test_*.db-wal && echo 'Test artifacts cleaned.'",
+		"test:mab:get": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/mab/download.js",
+		"test:mab": "mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/mab/runner.js 2>&1 | tee /tmp/rummy_test_diag/mab_$(date +%Y%m%dT%H%M%S).log",
+		"test:lme:get": "node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/lme/download.js",
+		"test:lme": "mkdir -p /tmp/rummy_test_diag && node --env-file-if-exists=.env.example --env-file-if-exists=.env --env-file-if-exists=.env.test test/lme/runner.js 2>&1 | tee /tmp/rummy_test_diag/lme_$(date +%Y%m%dT%H%M%S).log",
+		"test:mab:clean": "rm -rf test/mab/results/*/",
+		"test:lme:clean": "rm -rf test/lme/results/*/",
+		"test:clear": "rm -rf /tmp/rummy_test_diag /tmp/rummy_test_*.db /tmp/rummy_test_*.db-shm /tmp/rummy_test_*.db-wal /tmp/rummy-stories-*"
 	},
 	"devDependencies": {
 		"@biomejs/biome": "^2.4.6"
 	},
 	"dependencies": {
 		"@possumtech/sqlrite": "^3.1.0",
+		"@xmldom/xmldom": "^0.9.9",
 		"htmlparser2": "^12.0.0",
-		"tiktoken": "^1.0.22",
-		"ws": "^8.19.0"
-	},
-	"optionalDependencies": {
-		"@possumtech/rummy.repo": "^0.0.4",
-		"@possumtech/rummy.web": "^0.0.7"
+		"picomatch": "^4.0.4",
+		"ws": "^8.19.0",
+		"xpath": "^0.0.34"
 	}
 }

package/service.js

@@ -115,7 +115,7 @@ async function main() {
 	}
 
 	// 6b. Abort stuck runs (can't be running if the server just started)
-	await db.reset_active_prompts.run({});
+	await db.reset_active_loops.run({});
 	const aborted = await db.abort_stuck_runs.run({});
 	if (aborted.changes > 0) {
 		console.log(`[RUMMY] Recovered ${aborted.changes} stuck run(s)`);