@possumtech/rummy 0.5.0 → 2.0.0

package/src/agent/known_store.sql

@@ -1,196 +1,346 @@
--- PREP: upsert_known_entry
-INSERT INTO known_entries (
-	run_id, loop_id, turn, path, body, status, fidelity, hash
-	, attributes, tokens, updated_at
+-- PREP: upsert_entry
+-- Content-layer upsert. Returns entry id for the subsequent run_view write.
+-- Null :attributes on UPDATE path means "don't touch existing attributes"
+-- — so UPDATE reads :attributes directly, not excluded.attributes (which
+-- would have been coerced to '{}' by the VALUES clause).
+INSERT INTO entries (
+	scope, path, body, attributes, hash, updated_at
 )
 VALUES (
-	:run_id, :loop_id, :turn, :path, :body, :status, :fidelity, :hash
-	, COALESCE(:attributes, '{}')
-	, countTokens(:body)
-	, COALESCE(:updated_at, CURRENT_TIMESTAMP)
+	:scope, :path, :body, COALESCE(:attributes, '{}'), :hash
+	, CURRENT_TIMESTAMP
 )
-ON CONFLICT (run_id, path) DO UPDATE SET
+ON CONFLICT (scope, path) DO UPDATE SET
 	body = excluded.body
-	, status = excluded.status
-	, fidelity = excluded.fidelity
-	, hash = COALESCE(excluded.hash, known_entries.hash)
-	, attributes = COALESCE(excluded.attributes, known_entries.attributes)
-	, loop_id = excluded.loop_id
+	, attributes = COALESCE(:attributes, entries.attributes)
+	, hash = COALESCE(:hash, entries.hash)
+	, updated_at = CURRENT_TIMESTAMP
+RETURNING id;
+
+-- PREP: upsert_run_view
+-- View-layer upsert. Called after upsert_entry with the returned entry id.
+INSERT INTO run_views (
+	run_id, entry_id, loop_id, turn, state, outcome, visibility, updated_at
+)
+VALUES (
+	:run_id, :entry_id, :loop_id, :turn, :state, :outcome, :visibility
+	, CURRENT_TIMESTAMP
+)
+ON CONFLICT (run_id, entry_id) DO UPDATE SET
+	loop_id = excluded.loop_id
 	, turn = excluded.turn
-	, tokens = countTokens(excluded.body)
-	, write_count = known_entries.write_count + 1
-	, updated_at = COALESCE(excluded.updated_at, CURRENT_TIMESTAMP);
-
--- PREP: recount_tokens
-UPDATE known_entries
-SET tokens = :tokens
-WHERE run_id = :run_id AND path = :path;
-
--- PREP: get_stale_tokens
-SELECT path, body
-FROM known_entries
-WHERE
-	run_id = :run_id
-	AND turn = :turn;
+	, state = excluded.state
+	, outcome = excluded.outcome
+	, visibility = excluded.visibility
+	, write_count = run_views.write_count + 1
+	, updated_at = CURRENT_TIMESTAMP;
+
+-- Helper fragment: "the entry this run's view references at this path".
+-- Every UPDATE/DELETE resolves its target this way so the logic is
+-- correct whether the entry lives in the run's own scope or a shared one.
+
+-- PREP: append_entry_body
+-- Streaming entry body growth. Appends a chunk to the existing body.
+UPDATE entries
+SET
+	body = body || :chunk
+	, updated_at = CURRENT_TIMESTAMP
+WHERE id = (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE rv.run_id = :run_id AND e.path = :path
+	LIMIT 1
+);
 
 -- PREP: delete_known_entry
-DELETE FROM known_entries
-WHERE run_id = :run_id AND path = :path;
+-- Removes the view only. Entry is left for future GC; may be shared.
+DELETE FROM run_views
+WHERE run_id = :run_id AND entry_id = (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE rv.run_id = :run_id AND e.path = :path
+	LIMIT 1
+);
 
 -- PREP: delete_file_entries_by_pattern
-DELETE FROM known_entries
-WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
+DELETE FROM run_views
+WHERE run_id = :run_id AND entry_id IN (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE
+		rv.run_id = :run_id
+		AND hedmatch(:pattern, e.path)
+		AND e.scheme IS NULL
+);
+
+-- PREP: resolve_known_entry_view
+UPDATE run_views
+SET
+	state = :state
+	, outcome = :outcome
+	, updated_at = CURRENT_TIMESTAMP
+WHERE run_id = :run_id AND entry_id = (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE rv.run_id = :run_id AND e.path = :path
+	LIMIT 1
+);
 
--- PREP: resolve_known_entry
-UPDATE known_entries
+-- PREP: resolve_known_entry_body
+UPDATE entries
 SET
-	status = :status
-	, body = :body
+	body = :body
 	, updated_at = CURRENT_TIMESTAMP
-WHERE run_id = :run_id AND path = :path;
+WHERE id = (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE rv.run_id = :run_id AND e.path = :path
+	LIMIT 1
+);
 
--- PREP: set_file_fidelity
-UPDATE known_entries
+-- PREP: set_file_visibility
+UPDATE run_views
 SET
-	fidelity = :fidelity
+	visibility = :visibility
 	, updated_at = CURRENT_TIMESTAMP
-WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL;
+WHERE run_id = :run_id AND entry_id IN (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE
+		rv.run_id = :run_id
+		AND hedmatch(:pattern, e.path)
+		AND e.scheme IS NULL
+);
 
 -- PREP: promote_path
-UPDATE known_entries
+UPDATE run_views
 SET
-	fidelity = 'promoted'
-	, status = 200
+	visibility = 'visible'
+	, state = 'resolved'
+	, outcome = NULL
 	, turn = :turn
 	, updated_at = CURRENT_TIMESTAMP
-WHERE run_id = :run_id AND path = :path;
+WHERE run_id = :run_id AND entry_id = (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE rv.run_id = :run_id AND e.path = :path
+	LIMIT 1
+);
 
 -- PREP: demote_path
-UPDATE known_entries
+UPDATE run_views
 SET
-	fidelity = 'archived'
+	visibility = 'archived'
 	, updated_at = CURRENT_TIMESTAMP
-WHERE run_id = :run_id AND path = :path;
+WHERE run_id = :run_id AND entry_id = (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE rv.run_id = :run_id AND e.path = :path
+	LIMIT 1
+);
 
--- PREP: set_fidelity
--- Tokens unchanged — always reflects full body cost.
-UPDATE known_entries
+-- PREP: set_visibility
+UPDATE run_views
 SET
-	fidelity = :fidelity
+	visibility = :visibility
 	, updated_at = CURRENT_TIMESTAMP
-WHERE run_id = :run_id AND path = :path;
+WHERE run_id = :run_id AND entry_id = (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE rv.run_id = :run_id AND e.path = :path
+	LIMIT 1
+);
 
 -- PREP: get_entry_body
-SELECT body
-FROM known_entries
-WHERE run_id = :run_id AND path = :path;
+SELECT e.body AS body
+FROM run_views AS rv
+JOIN entries AS e ON e.id = rv.entry_id
+WHERE rv.run_id = :run_id AND e.path = :path;
 
 -- PREP: get_entry_state
-SELECT status, fidelity, scheme, turn
-FROM known_entries
-WHERE run_id = :run_id AND path = :path;
+SELECT rv.state, rv.outcome, rv.visibility, e.scheme, rv.turn
+FROM run_views AS rv
+JOIN entries AS e ON e.id = rv.entry_id
+WHERE rv.run_id = :run_id AND e.path = :path;
 
 -- PREP: get_file_states_by_pattern
-SELECT path, status, fidelity, turn
-FROM known_entries
-WHERE run_id = :run_id AND hedmatch(:pattern, path) AND scheme IS NULL
-ORDER BY path;
+SELECT e.path, rv.state, rv.outcome, rv.visibility, rv.turn
+FROM run_views AS rv
+JOIN entries AS e ON e.id = rv.entry_id
+WHERE
+	rv.run_id = :run_id
+	AND hedmatch(:pattern, e.path)
+	AND e.scheme IS NULL
+ORDER BY e.path;
 
 -- PREP: update_entry_attributes
-UPDATE known_entries
+UPDATE entries
 SET
 	attributes = json_patch(attributes, :attributes)
 	, updated_at = CURRENT_TIMESTAMP
-WHERE run_id = :run_id AND path = :path;
+WHERE id = (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE rv.run_id = :run_id AND e.path = :path
+	LIMIT 1
+);
 
 -- PREP: get_entry_attributes
-SELECT attributes
-FROM known_entries
-WHERE run_id = :run_id AND path = :path;
+SELECT e.attributes AS attributes
+FROM run_views AS rv
+JOIN entries AS e ON e.id = rv.entry_id
+WHERE rv.run_id = :run_id AND e.path = :path;
 
 -- PREP: promote_by_pattern
-UPDATE known_entries
+UPDATE run_views
 SET
-	fidelity = 'promoted'
-	, status = 200
+	visibility = 'visible'
+	, state = 'resolved'
+	, outcome = NULL
 	, turn = :turn
 	, updated_at = CURRENT_TIMESTAMP
-WHERE
-	run_id = :run_id
-	AND hedmatch(:path, path)
-	AND (:body IS NULL OR hedsearch(:body, body));
+WHERE run_id = :run_id AND entry_id IN (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE
+		rv.run_id = :run_id
+		AND hedmatch(:path, e.path)
+		AND (:body IS NULL OR hedsearch(:body, e.body))
+);
 
 -- PREP: demote_by_pattern
-UPDATE known_entries
+UPDATE run_views
 SET
-	fidelity = 'archived'
+	visibility = 'archived'
 	, updated_at = CURRENT_TIMESTAMP
-WHERE
-	run_id = :run_id
-	AND hedmatch(:path, path)
-	AND (:body IS NULL OR hedsearch(:body, body));
+WHERE run_id = :run_id AND entry_id IN (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE
+		rv.run_id = :run_id
+		AND hedmatch(:path, e.path)
+		AND (:body IS NULL OR hedsearch(:body, e.body))
+);
 
 -- PREP: get_entries_by_pattern
-SELECT path, body, scheme, status, fidelity, tokens, attributes
-FROM known_entries
+SELECT
+	e.path, e.body, e.scheme, rv.state, rv.outcome, rv.visibility
+	, countTokens(e.body) AS tokens, e.attributes
+FROM run_views AS rv
+JOIN entries AS e ON e.id = rv.entry_id
 WHERE
-	run_id = :run_id
-	AND hedmatch(:path, path)
-	AND (:body IS NULL OR hedsearch(:body, body))
-ORDER BY path
+	rv.run_id = :run_id
+	AND hedmatch(:path, e.path)
+	AND (:body IS NULL OR hedsearch(:body, e.body))
+ORDER BY e.path
 LIMIT
 	COALESCE(:limit, -1)
-	OFFSET
-	COALESCE(:offset, 0);
+	OFFSET COALESCE(:offset, 0);
 
 -- PREP: delete_entries_by_pattern
-DELETE FROM known_entries
-WHERE
-	run_id = :run_id
-	AND hedmatch(:path, path)
-	AND (:body IS NULL OR hedsearch(:body, body));
+DELETE FROM run_views
+WHERE run_id = :run_id AND entry_id IN (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE
+		rv.run_id = :run_id
+		AND hedmatch(:path, e.path)
+		AND (:body IS NULL OR hedsearch(:body, e.body))
+);
 
 -- PREP: update_body_by_pattern
-UPDATE known_entries
+UPDATE entries
 SET
 	body = :new_body
-	, tokens = countTokens(:new_body)
-	, write_count = write_count + 1
 	, updated_at = CURRENT_TIMESTAMP
-WHERE
-	run_id = :run_id
-	AND hedmatch(:path, path)
-	AND (:body IS NULL OR hedsearch(:body, body));
-
--- PREP: restore_summarized_prompts
--- Restore prompt entries demoted by a recovery phase that was
--- interrupted (e.g. server crash). Safe to call unconditionally at loop
--- start: if the full prompt would overflow, Prompt Demotion handles it.
-UPDATE known_entries
+WHERE id IN (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE
+		rv.run_id = :run_id
+		AND hedmatch(:path, e.path)
+		AND (:body IS NULL OR hedsearch(:body, e.body))
+);
+
+-- PREP: bump_write_count_by_pattern
+-- Companion to update_body_by_pattern. write_count lives on run_views.
+UPDATE run_views
 SET
-	fidelity = 'promoted'
+	write_count = write_count + 1
 	, updated_at = CURRENT_TIMESTAMP
-WHERE run_id = :run_id AND scheme = 'prompt' AND fidelity = 'demoted';
+WHERE run_id = :run_id AND entry_id IN (
+	SELECT e.id FROM entries AS e
+	JOIN run_views AS rv ON rv.entry_id = e.id
+	WHERE
+		rv.run_id = :run_id
+		AND hedmatch(:path, e.path)
+		AND (:body IS NULL OR hedsearch(:body, e.body))
+);
+
+-- PREP: get_turn_demotion_targets
+-- Rows that demote_turn_entries is about to flip. Return shape
+-- matches the old RETURNING (path, tokens) for caller compatibility.
+-- State filter: skip failed/cancelled entries (they're already not
+-- contributing visible context — demoting them would be misleading).
+-- Scheme filter: skip known/unknown — these are the model's deliverables,
+-- not housekeeping. Auto-demoting just-created knowns punishes the
+-- correct Distill+Demote pattern.
+SELECT e.path, countTokens(e.body) AS tokens
+FROM run_views AS rv
+JOIN entries AS e ON e.id = rv.entry_id
+WHERE
+	rv.run_id = :run_id
+	AND rv.turn = :turn
+	AND rv.visibility = 'visible'
+	AND rv.state NOT IN ('failed', 'cancelled')
+	AND e.scheme NOT IN ('known', 'unknown');
 
 -- PREP: demote_turn_entries
--- Demote all promoted entries from a turn for budget claw-back.
--- Action schemes (set/rm/mv/cp) at status 200 keep their status — those
--- represent committed side effects (files written/removed) that can't be
--- clawed back; only the body in context is demoted, not the truth of what
--- happened. Everything else flips to 413 since promotion was reversed.
--- Tokens unchanged — always reports full cost regardless of fidelity.
-UPDATE known_entries
+-- View-layer only — visibility lives on run_views. State untouched.
+-- Call get_turn_demotion_targets first if you need the list of what
+-- was demoted (used by budget plugin for the overflow error body).
+-- Scheme filter mirrors get_turn_demotion_targets — never demote the
+-- model's deliverables (known/unknown) along with housekeeping.
+UPDATE run_views
 SET
-	fidelity = 'demoted'
-	, status = CASE
-		WHEN scheme IN ('set', 'rm', 'mv', 'cp') AND status = 200 THEN 200
-		ELSE 413
-	END
+	visibility = 'summarized'
 	, updated_at = CURRENT_TIMESTAMP
 WHERE
 	run_id = :run_id
 	AND turn = :turn
-	AND fidelity = 'promoted'
-	AND status < 400
-RETURNING path, tokens;
+	AND visibility = 'visible'
+	AND state NOT IN ('failed', 'cancelled')
+	AND NOT EXISTS (
+		SELECT 1
+		FROM entries AS e
+		WHERE
+			e.id = run_views.entry_id
+			AND e.scheme IN ('known', 'unknown')
+	);
+
+-- PREP: get_run_visible_targets
+-- All visible entries across the run, oldest promotion first. Used by
+-- budget postDispatch as the fallback demotion set when this-turn
+-- demotion yields nothing but the packet still overflows (promotions
+-- from prior turns the model forgot to demote themselves).
+SELECT e.path, countTokens(e.body) AS tokens, rv.turn
+FROM run_views AS rv
+JOIN entries AS e ON e.id = rv.entry_id
+WHERE
+	rv.run_id = :run_id
+	AND rv.visibility = 'visible'
+	AND rv.state NOT IN ('failed', 'cancelled')
+ORDER BY rv.turn, e.id;
 
+-- PREP: demote_run_visible
+-- Broad cross-turn demotion. Separate prep from demote_turn_entries
+-- so the caller's intent (surgical this-turn vs fallback all-visible)
+-- stays explicit.
+UPDATE run_views
+SET
+	visibility = 'summarized'
+	, updated_at = CURRENT_TIMESTAMP
+WHERE
+	run_id = :run_id
+	AND visibility = 'visible'
+	AND state NOT IN ('failed', 'cancelled');

package/src/agent/materializeContext.js

@@ -0,0 +1,102 @@
+import ContextAssembler from "./ContextAssembler.js";
+import { countTokens } from "./tokens.js";
+
+/**
+ * Rebuild turn_context from v_model_context, then assemble messages.
+ * Called at turn start and again by the budget plugin when it needs a
+ * fresh measurement after mutating visibility.
+ */
+export default async function materializeContext({
+	db,
+	hooks,
+	runId,
+	loopId,
+	turn,
+	systemPrompt,
+	mode,
+	toolSet,
+	contextSize,
+	demoted,
+}) {
+	await db.clear_turn_context.run({ run_id: runId, turn });
+	const viewRows = await db.get_model_context.all({ run_id: runId });
+	// Per-entry token accounting (see SPEC @token_accounting): captured
+	// here while we still have the raw body, then merged onto rows after
+	// the read-back roundtrip through turn_context.
+	const tokenAccounting = new Map();
+	for (const row of viewRows) {
+		// schemeOf() yields NULL (or "") for bare file paths — translate
+		// to "file" so the view lookup finds the file scheme handler.
+		const scheme = row.scheme ? row.scheme : "file";
+		const attrs = row.attributes ? JSON.parse(row.attributes) : null;
+		// Log entries live at log://turn_N/action/slug. Dispatch projection
+		// to the action plugin's view (set, update, search, etc.) by
+		// extracting the action segment from the path.
+		let projectionKey = scheme;
+		if (scheme === "log") {
+			const m = row.path.match(/^log:\/\/turn_\d+\/([^/]+)\//);
+			if (m) projectionKey = m[1];
+		}
+		const baseEntry = {
+			path: row.path,
+			scheme,
+			body: row.body,
+			attributes: attrs,
+			category: row.category,
+		};
+		const visibleProjection = await hooks.tools.view(projectionKey, {
+			...baseEntry,
+			visibility: "visible",
+		});
+		const summarizedProjection = await hooks.tools.view(projectionKey, {
+			...baseEntry,
+			visibility: "summarized",
+		});
+		const vTokens = countTokens(visibleProjection);
+		const sTokens = countTokens(summarizedProjection);
+		tokenAccounting.set(row.path, { vTokens, sTokens });
+		const projectedBody =
+			row.visibility === "visible" ? visibleProjection : summarizedProjection;
+		await db.insert_turn_context.run({
+			run_id: runId,
+			loop_id: loopId,
+			turn,
+			ordinal: row.ordinal,
+			path: row.path,
+			visibility: row.visibility,
+			state: row.state,
+			outcome: row.outcome,
+			body: projectedBody,
+			attributes: row.attributes,
+			category: row.category,
+			source_turn: row.turn,
+		});
+	}
+	const rows = await db.get_turn_context.all({ run_id: runId, turn });
+	for (const row of rows) {
+		const t = tokenAccounting.get(row.path);
+		if (!t) continue;
+		row.vTokens = t.vTokens;
+		row.sTokens = t.sTokens;
+		row.aTokens = t.vTokens - t.sTokens;
+	}
+	const lastCtx = await db.get_last_context_tokens.get({ run_id: runId });
+	// First turn of a new run has no prior context.
+	let lastContextTokens = 0;
+	if (lastCtx) lastContextTokens = lastCtx.context_tokens;
+
+	const messages = await ContextAssembler.assembleFromTurnContext(
+		rows,
+		{
+			type: mode,
+			systemPrompt,
+			contextSize,
+			demoted,
+			toolSet,
+			lastContextTokens,
+			turn,
+		},
+		hooks,
+	);
+	return { rows, messages, lastContextTokens };
+}

package/src/agent/runs.sql

@@ -44,7 +44,7 @@ SELECT
 		FROM known_entries AS ke
 		WHERE
 			ke.run_id = r.id
-			AND ke.scheme = 'summarize'
+			AND ke.path LIKE 'log://turn_%/update/%'
 		ORDER BY ke.id DESC
 		LIMIT 1
 	) AS summary
@@ -79,14 +79,17 @@ WHERE id = :run_id
 RETURNING next_turn - 1 AS turn;
 
 -- PREP: fork_known_entries
-INSERT INTO known_entries (
-	run_id, loop_id, turn, path, body, status, fidelity
-	, hash, attributes, tokens, refs, write_count
+-- Cheap fork: copy only view rows. Entries stay shared between parent
+-- and child. Child's subsequent writes diverge via upsert into a new
+-- run-scoped entry.
+INSERT INTO run_views (
+	run_id, entry_id, loop_id, turn, state, outcome, visibility
+	, write_count, refs
 )
 SELECT
-	:new_run_id, NULL, turn, path, body, status, fidelity
-	, hash, attributes, tokens, refs, write_count
-FROM known_entries
+	:new_run_id, entry_id, NULL, turn, state, outcome, visibility
+	, write_count, refs
+FROM run_views
 WHERE run_id = :parent_run_id;
 
 -- PREP: get_active_runs

package/src/agent/schemes.sql

@@ -1,6 +1,17 @@
 -- PREP: upsert_scheme
-INSERT OR REPLACE INTO schemes (name, model_visible, category)
-VALUES (:name, :model_visible, :category);
+INSERT INTO schemes (name, model_visible, category, default_scope, writable_by)
+VALUES (
+	:name
+	, :model_visible
+	, :category
+	, COALESCE(:default_scope, 'run')
+	, COALESCE(:writable_by, '["model","plugin"]')
+)
+ON CONFLICT (name) DO UPDATE SET
+	model_visible = excluded.model_visible
+	, category = excluded.category
+	, default_scope = excluded.default_scope
+	, writable_by = excluded.writable_by;
 
 -- PREP: get_all_schemes
-SELECT name, model_visible, category FROM schemes;
+SELECT name, model_visible, category, default_scope, writable_by FROM schemes;

package/src/agent/turns.sql

@@ -39,12 +39,12 @@ WHERE run_id = :run_id AND context_tokens > 0
 ORDER BY sequence DESC
 LIMIT 1;
 
--- PREP: get_run_log
-SELECT ke.path, ke.status, ke.body, ke.attributes
-FROM known_entries AS ke
-JOIN schemes AS s ON s.name = COALESCE(ke.scheme, 'file')
-WHERE
-	ke.run_id = :run_id
-	AND ke.scheme IS NOT NULL
-	AND s.category NOT IN ('knowledge')
-ORDER BY ke.id;
+-- PREP: get_turns_by_run
+SELECT
+	id, run_id, loop_id, sequence, context_tokens, prompt_tokens,
+	cached_tokens, completion_tokens, reasoning_tokens, total_tokens, cost
+FROM turns
+WHERE run_id = :run_id
+ORDER BY sequence;
+
+-- get_run_log retired — use get_results (v_run_log) instead.