@rkarim08/sia 1.0.0

package/src/graph/semantic-db.ts

@@ -0,0 +1,101 @@
+// Module: semantic-db — Migration runner and semantic database openers
+
+import { Database } from "bun:sqlite";
+import { existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { BunSqliteDb } from "@/graph/db-interface";
+import { SIA_HOME } from "@/shared/config";
+
+/**
+ * Open (or create) a SQLite database at `dbPath`, apply WAL/NORMAL/FK pragmas,
+ * then run every unapplied `.sql` file found in `migrationsDir` (sorted
+ * alphabetically).  Each migration is tracked in a `_migrations` table so it
+ * is never applied twice.
+ *
+ * Returns a `BunSqliteDb` wrapping the open connection.
+ */
+export function runMigrations(dbPath: string, migrationsDir: string): BunSqliteDb {
+	// Ensure the parent directory for the database file exists.
+	const parentDir = dirname(dbPath);
+	mkdirSync(parentDir, { recursive: true });
+
+	// Open (or create) the database.
+	const db = new Database(dbPath);
+
+	// page_size must come first — only effective on a brand-new file before any
+	// tables exist; silently a no-op on existing databases.
+	db.exec("PRAGMA page_size = 4096");
+
+	// WAL for concurrent reads, synchronous NORMAL for safety,
+	// foreign keys enforced.
+	db.exec("PRAGMA journal_mode = WAL");
+	db.exec("PRAGMA synchronous = NORMAL");
+	db.exec("PRAGMA foreign_keys = ON");
+
+	// Performance hardening: memory-mapped I/O (1 GB virtual window, demand-paged
+	// by the OS), temp tables kept in RAM, and a 64 MB page cache.
+	db.exec("PRAGMA mmap_size = 1073741824");
+	db.exec("PRAGMA temp_store = MEMORY");
+	db.exec("PRAGMA cache_size = -64000");
+
+	// Ensure the bookkeeping table exists.
+	db.exec(`
+		CREATE TABLE IF NOT EXISTS _migrations (
+			name TEXT PRIMARY KEY,
+			applied_at INTEGER NOT NULL
+		)
+	`);
+
+	// If the migrations directory does not exist, return early — the database
+	// is usable but has no application tables yet.
+	if (!existsSync(migrationsDir)) {
+		return new BunSqliteDb(db);
+	}
+
+	// Discover which migrations have already been applied.
+	const applied = new Set<string>(
+		(db.prepare("SELECT name FROM _migrations").all() as Array<{ name: string }>).map(
+			(r) => r.name,
+		),
+	);
+
+	// Read .sql files from the migrations directory, sorted alphabetically so
+	// that numbered prefixes control execution order (001-foo.sql before 002-bar.sql).
+	const files = readdirSync(migrationsDir)
+		.filter((f) => f.endsWith(".sql"))
+		.sort();
+
+	for (const file of files) {
+		if (applied.has(file)) continue;
+
+		const sql = readFileSync(`${migrationsDir}/${file}`, "utf-8");
+		db.exec(sql);
+		db.prepare("INSERT INTO _migrations (name, applied_at) VALUES (?, ?)").run(file, Date.now());
+	}
+
+	return new BunSqliteDb(db);
+}
+
+/**
+ * Open (or create) the semantic graph database for a given repo.
+ * Resolves to `{siaHome}/repos/{repoHash}/graph.db` and applies
+ * migrations from the `migrations/semantic` directory.
+ */
+export function openGraphDb(repoHash: string, siaHome?: string): BunSqliteDb {
+	const home = siaHome ?? SIA_HOME;
+	const dbPath = join(home, "repos", repoHash, "graph.db");
+	const migrationsDir = resolve(import.meta.dirname, "../../migrations/semantic");
+	return runMigrations(dbPath, migrationsDir);
+}
+
+/**
+ * Open (or create) the episodic database for a given repo.
+ * Resolves to `{siaHome}/repos/{repoHash}/episodic.db` and applies
+ * migrations from the `migrations/episodic` directory.
+ */
+export function openEpisodicDb(repoHash: string, siaHome?: string): BunSqliteDb {
+	const home = siaHome ?? SIA_HOME;
+	const dbPath = join(home, "repos", repoHash, "episodic.db");
+	const migrationsDir = resolve(import.meta.dirname, "../../migrations/episodic");
+	return runMigrations(dbPath, migrationsDir);
+}

package/src/graph/session-resume.ts

@@ -0,0 +1,56 @@
+// Module: session-resume — Session resume CRUD operations
+
+import type { SiaDb } from "@/graph/db-interface";
+
+/** A session resume row returned by loadSubgraph. */
+export interface SessionResumeRow {
+	subgraph_json: string;
+	last_prompt: string | null;
+	budget_used: number;
+}
+
+/**
+ * Persist (upsert) a session resume record for the given session.
+ * If a row already exists for the session_id, it is updated in place.
+ */
+export async function saveSubgraph(
+	db: SiaDb,
+	sessionId: string,
+	subgraphJson: string,
+	lastPrompt: string | null,
+	budgetUsed: number,
+): Promise<void> {
+	const now = Date.now();
+	await db.execute(
+		`INSERT INTO session_resume (session_id, subgraph_json, last_prompt, budget_used, created_at, updated_at)
+		 VALUES (?, ?, ?, ?, ?, ?)
+		 ON CONFLICT(session_id) DO UPDATE SET
+		   subgraph_json = excluded.subgraph_json,
+		   last_prompt   = excluded.last_prompt,
+		   budget_used   = excluded.budget_used,
+		   updated_at    = excluded.updated_at`,
+		[sessionId, subgraphJson, lastPrompt, budgetUsed, now, now],
+	);
+}
+
+/**
+ * Load the session resume record for the given session.
+ * Returns null if no record exists.
+ */
+export async function loadSubgraph(db: SiaDb, sessionId: string): Promise<SessionResumeRow | null> {
+	const result = await db.execute(
+		"SELECT subgraph_json, last_prompt, budget_used FROM session_resume WHERE session_id = ?",
+		[sessionId],
+	);
+	if (result.rows.length === 0) {
+		return null;
+	}
+	return result.rows[0] as unknown as SessionResumeRow;
+}
+
+/**
+ * Delete the session resume record for the given session.
+ */
+export async function deleteResume(db: SiaDb, sessionId: string): Promise<void> {
+	await db.execute("DELETE FROM session_resume WHERE session_id = ?", [sessionId]);
+}

package/src/graph/snapshots.ts

@@ -0,0 +1,342 @@
+// Module: snapshots — Daily snapshot creation and rollback + branch-keyed snapshots
+
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { basename, join } from "node:path";
+import { writeAuditEntry } from "@/graph/audit";
+import type { SiaDb } from "@/graph/db-interface";
+import { SIA_HOME } from "@/shared/config";
+
+/** Shape of a serialized snapshot file. */
+export interface SnapshotData {
+	version: 1;
+	timestamp: number;
+	entities: Record<string, unknown>[];
+	edges: Record<string, unknown>[];
+}
+
+/**
+ * Build the snapshot directory path for a given repo.
+ */
+function snapshotDir(repoHash: string, siaHome?: string): string {
+	const home = siaHome ?? SIA_HOME;
+	return join(home, "snapshots", repoHash);
+}
+
+/**
+ * Format a Date as YYYY-MM-DD.
+ */
+function formatDate(d: Date): string {
+	const yyyy = d.getFullYear();
+	const mm = String(d.getMonth() + 1).padStart(2, "0");
+	const dd = String(d.getDate()).padStart(2, "0");
+	return `${yyyy}-${mm}-${dd}`;
+}
+
+/**
+ * Parse a YYYY-MM-DD date string into a Unix ms timestamp (midnight UTC).
+ */
+function parseDateFromFilename(filename: string): number {
+	// Extract YYYY-MM-DD from "YYYY-MM-DD.snapshot"
+	const datePart = basename(filename, ".snapshot");
+	const [yearStr, monthStr, dayStr] = datePart.split("-");
+	const year = Number(yearStr);
+	const month = Number(monthStr) - 1; // JS months are 0-indexed
+	const day = Number(dayStr);
+	return Date.UTC(year, month, day);
+}
+
+/**
+ * Create a snapshot of all active entities and edges.
+ *
+ * Active entities: t_valid_until IS NULL AND archived_at IS NULL
+ * Active edges: t_valid_until IS NULL
+ *
+ * Writes JSON to `{siaHome}/snapshots/{repoHash}/YYYY-MM-DD.snapshot`.
+ * Returns the snapshot file path.
+ */
+export async function createSnapshot(
+	db: SiaDb,
+	repoHash: string,
+	siaHome?: string,
+): Promise<string> {
+	const dir = snapshotDir(repoHash, siaHome);
+	mkdirSync(dir, { recursive: true });
+
+	// Query active entities
+	const entitiesResult = await db.execute(
+		"SELECT * FROM graph_nodes WHERE t_valid_until IS NULL AND archived_at IS NULL",
+	);
+
+	// Query active edges
+	const edgesResult = await db.execute("SELECT * FROM graph_edges WHERE t_valid_until IS NULL");
+
+	const now = Date.now();
+	const data: SnapshotData = {
+		version: 1,
+		timestamp: now,
+		entities: entitiesResult.rows,
+		edges: edgesResult.rows,
+	};
+
+	const filename = `${formatDate(new Date(now))}.snapshot`;
+	const filepath = join(dir, filename);
+
+	writeFileSync(filepath, JSON.stringify(data, null, 2), "utf-8");
+
+	await writeAuditEntry(db, "ADD", { snapshot_id: filepath });
+
+	return filepath;
+}
+
+/**
+ * List all snapshot files for a repo, sorted by date (oldest first).
+ */
+export function listSnapshots(repoHash: string, siaHome?: string): string[] {
+	const dir = snapshotDir(repoHash, siaHome);
+
+	if (!existsSync(dir)) {
+		return [];
+	}
+
+	const files = readdirSync(dir)
+		.filter((f) => f.endsWith(".snapshot"))
+		.sort();
+
+	return files.map((f) => join(dir, f));
+}
+
+/**
+ * Find the snapshot file closest to and before a target timestamp.
+ *
+ * Returns the full path, or null if no snapshot exists before the target.
+ */
+export function findNearestSnapshot(
+	repoHash: string,
+	targetTimestamp: number,
+	siaHome?: string,
+): string | null {
+	const snapshots = listSnapshots(repoHash, siaHome);
+
+	let best: string | null = null;
+	let bestTs = -1;
+
+	for (const snap of snapshots) {
+		const ts = parseDateFromFilename(basename(snap));
+		if (ts <= targetTimestamp && ts > bestTs) {
+			best = snap;
+			bestTs = ts;
+		}
+	}
+
+	return best;
+}
+
+/**
+ * Restore a snapshot: create a pre-rollback snapshot first (atomicity),
+ * then delete all entities and edges, re-insert from the snapshot JSON.
+ */
+export async function restoreSnapshot(
+	db: SiaDb,
+	snapshotPath: string,
+	repoHash: string,
+	siaHome?: string,
+): Promise<void> {
+	// Step 1: Read the snapshot file BEFORE creating the pre-rollback snapshot
+	// to avoid the pre-rollback overwriting the same date-based filename.
+	const raw = readFileSync(snapshotPath, "utf-8");
+	const data = JSON.parse(raw) as SnapshotData;
+
+	// Step 2: Create a pre-rollback snapshot for safety
+	await createSnapshot(db, repoHash, siaHome);
+
+	// Step 3: Delete all existing entities and edges, then re-insert from snapshot.
+	await db.transaction(async (tx) => {
+		// Delete edges first (FK constraint: edges reference entities)
+		await tx.execute("DELETE FROM graph_edges");
+		// Delete entities
+		await tx.execute("DELETE FROM graph_nodes");
+
+		// Re-insert entities
+		for (const entity of data.entities) {
+			const columns = Object.keys(entity);
+			const safeColumns = columns.map((c) => `"${c.replace(/"/g, '""')}"`);
+			const placeholders = columns.map(() => "?").join(", ");
+			const values = columns.map((col) => entity[col] ?? null);
+			const sql = `INSERT INTO graph_nodes (${safeColumns.join(", ")}) VALUES (${placeholders})`;
+			await tx.execute(sql, values);
+		}
+
+		// Re-insert edges
+		for (const edge of data.edges) {
+			const columns = Object.keys(edge);
+			const safeColumns = columns.map((c) => `"${c.replace(/"/g, '""')}"`);
+			const placeholders = columns.map(() => "?").join(", ");
+			const values = columns.map((col) => edge[col] ?? null);
+			const sql = `INSERT INTO graph_edges (${safeColumns.join(", ")}) VALUES (${placeholders})`;
+			await tx.execute(sql, values);
+		}
+	});
+
+	// Step 4: Write audit entry for the restore
+	await writeAuditEntry(db, "UPDATE", { snapshot_id: snapshotPath });
+}
+
+// ---------------------------------------------------------------------------
+// Branch-keyed snapshots (stored in SQLite, not on disk)
+// ---------------------------------------------------------------------------
+
+/** Shape of a branch snapshot row. */
+export interface BranchSnapshot {
+	id: number;
+	branch_name: string;
+	commit_hash: string;
+	node_count: number;
+	edge_count: number;
+	snapshot_data: string;
+	created_at: number;
+	updated_at: number;
+}
+
+/**
+ * Create (or update) a branch snapshot.
+ * Serializes all active nodes and edges into JSON and UPSERTs into
+ * the branch_snapshots table. Each branch gets exactly one snapshot row.
+ */
+export async function createBranchSnapshot(
+	db: SiaDb,
+	branchName: string,
+	commitHash: string,
+): Promise<void> {
+	const { rows: nodes } = await db.execute(
+		"SELECT * FROM graph_nodes WHERE t_valid_until IS NULL AND archived_at IS NULL",
+	);
+	const { rows: edges } = await db.execute("SELECT * FROM graph_edges WHERE t_valid_until IS NULL");
+
+	const snapshotData = JSON.stringify({ nodes, edges });
+	const now = Date.now();
+
+	await db.execute(
+		`INSERT INTO branch_snapshots (branch_name, commit_hash, node_count, edge_count, snapshot_data, created_at, updated_at)
+		 VALUES (?, ?, ?, ?, ?, ?, ?)
+		 ON CONFLICT(branch_name) DO UPDATE SET
+		   commit_hash = excluded.commit_hash,
+		   node_count = excluded.node_count,
+		   edge_count = excluded.edge_count,
+		   snapshot_data = excluded.snapshot_data,
+		   updated_at = excluded.updated_at`,
+		[branchName, commitHash, nodes.length, edges.length, snapshotData, now, now],
+	);
+
+	await writeAuditEntry(db, "ADD", { snapshot_id: `branch:${branchName}:${commitHash}` });
+}
+
+/**
+ * Restore graph state from a branch snapshot.
+ * Deletes all current nodes/edges, then re-inserts from the snapshot.
+ * Returns true if a snapshot was found and restored, false otherwise.
+ */
+export async function restoreBranchSnapshot(db: SiaDb, branchName: string): Promise<boolean> {
+	const { rows } = await db.execute(
+		"SELECT snapshot_data FROM branch_snapshots WHERE branch_name = ?",
+		[branchName],
+	);
+
+	if (rows.length === 0) return false;
+
+	let data: { nodes: Record<string, unknown>[]; edges: Record<string, unknown>[] };
+	try {
+		const parsed = JSON.parse(rows[0].snapshot_data as string);
+		if (!parsed || !Array.isArray(parsed.nodes) || !Array.isArray(parsed.edges)) {
+			throw new Error("Invalid snapshot structure: missing nodes or edges arrays");
+		}
+		data = parsed;
+	} catch (err) {
+		throw new Error(
+			`Failed to parse branch snapshot for "${branchName}": ${err instanceof Error ? err.message : err}`,
+		);
+	}
+
+	await db.transaction(async (tx) => {
+		await tx.execute("DELETE FROM graph_edges");
+		await tx.execute("DELETE FROM graph_nodes");
+
+		for (const node of data.nodes) {
+			const columns = Object.keys(node);
+			const safeColumns = columns.map((c) => `"${c.replace(/"/g, '""')}"`);
+			const placeholders = columns.map(() => "?").join(", ");
+			const values = columns.map((col) => node[col] ?? null);
+			await tx.execute(
+				`INSERT INTO graph_nodes (${safeColumns.join(", ")}) VALUES (${placeholders})`,
+				values,
+			);
+		}
+
+		for (const edge of data.edges) {
+			const columns = Object.keys(edge);
+			const safeColumns = columns.map((c) => `"${c.replace(/"/g, '""')}"`);
+			const placeholders = columns.map(() => "?").join(", ");
+			const values = columns.map((col) => edge[col] ?? null);
+			await tx.execute(
+				`INSERT INTO graph_edges (${safeColumns.join(", ")}) VALUES (${placeholders})`,
+				values,
+			);
+		}
+	});
+
+	await writeAuditEntry(db, "UPDATE", { snapshot_id: `branch-restore:${branchName}` });
+	return true;
+}
+
+/**
+ * List all branch snapshots, ordered by updated_at descending.
+ */
+export async function listBranchSnapshots(db: SiaDb): Promise<BranchSnapshot[]> {
+	const { rows } = await db.execute(
+		"SELECT id, branch_name, commit_hash, node_count, edge_count, snapshot_data, created_at, updated_at FROM branch_snapshots ORDER BY updated_at DESC",
+	);
+	return rows as unknown as BranchSnapshot[];
+}
+
+/**
+ * Prune snapshots for specific branches (e.g., deleted branches).
+ * Returns the number of snapshots deleted.
+ */
+export async function pruneBranchSnapshots(db: SiaDb, branchNames: string[]): Promise<number> {
+	if (branchNames.length === 0) return 0;
+
+	const { rows: before } = await db.execute("SELECT COUNT(*) as cnt FROM branch_snapshots");
+
+	const placeholders = branchNames.map(() => "?").join(", ");
+	await db.execute(
+		`DELETE FROM branch_snapshots WHERE branch_name IN (${placeholders})`,
+		branchNames,
+	);
+
+	const { rows: after } = await db.execute("SELECT COUNT(*) as cnt FROM branch_snapshots");
+
+	const deleted = Number(before[0].cnt) - Number(after[0].cnt);
+	if (deleted > 0) {
+		await writeAuditEntry(db, "ARCHIVE", { snapshot_id: `branch-prune:${branchNames.join(",")}` });
+	}
+	return deleted;
+}
+
+/**
+ * Garbage-collect branch snapshots older than ttlDays.
+ * Returns the number of snapshots deleted.
+ */
+export async function gcBranchSnapshots(db: SiaDb, ttlDays: number): Promise<number> {
+	const cutoff = Date.now() - ttlDays * 24 * 60 * 60 * 1000;
+
+	const { rows: before } = await db.execute("SELECT COUNT(*) as cnt FROM branch_snapshots");
+
+	await db.execute("DELETE FROM branch_snapshots WHERE updated_at < ?", [cutoff]);
+
+	const { rows: after } = await db.execute("SELECT COUNT(*) as cnt FROM branch_snapshots");
+
+	const deleted = Number(before[0].cnt) - Number(after[0].cnt);
+	if (deleted > 0) {
+		await writeAuditEntry(db, "ARCHIVE", { snapshot_id: `branch-gc:${deleted}:ttl=${ttlDays}d` });
+	}
+	return deleted;
+}

package/src/graph/staging.ts

@@ -0,0 +1,151 @@
+// Module: staging — CRUD for the memory_staging table (Tier 4 security staging area)
+
+import { randomUUID } from "node:crypto";
+import { writeAuditEntry } from "@/graph/audit";
+import type { SiaDb } from "@/graph/db-interface";
+
+/** Row shape matching all columns of the `memory_staging` table. */
+export interface StagedFact {
+	id: string;
+	source_episode: string | null;
+	proposed_type: string;
+	proposed_name: string;
+	proposed_content: string;
+	proposed_tags: string;
+	proposed_file_paths: string;
+	trust_tier: number;
+	raw_confidence: number;
+	validation_status: string;
+	rejection_reason: string | null;
+	created_at: number;
+	expires_at: number;
+}
+
+/** Fields the caller provides when inserting a staged fact. */
+export interface InsertStagedFactInput {
+	source_episode?: string;
+	proposed_type: string;
+	proposed_name: string;
+	proposed_content: string;
+	proposed_tags?: string;
+	proposed_file_paths?: string;
+	trust_tier?: number;
+	raw_confidence: number;
+}
+
+/** 7-day TTL in milliseconds. */
+const SEVEN_DAYS_MS = 7 * 86_400_000;
+
+/**
+ * Insert a new staged fact into `memory_staging`.
+ *
+ * Generates a UUID, sets `created_at = Date.now()`, computes
+ * `expires_at = created_at + 7 days`, defaults `validation_status = 'pending'`.
+ * Writes a STAGE audit entry. Returns the generated id.
+ */
+export async function insertStagedFact(db: SiaDb, input: InsertStagedFactInput): Promise<string> {
+	const id = randomUUID();
+	const createdAt = Date.now();
+	const expiresAt = createdAt + SEVEN_DAYS_MS;
+
+	await db.execute(
+		`INSERT INTO memory_staging (
+			id, source_episode, proposed_type, proposed_name, proposed_content,
+			proposed_tags, proposed_file_paths, trust_tier, raw_confidence,
+			validation_status, created_at, expires_at
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		[
+			id,
+			input.source_episode ?? null,
+			input.proposed_type,
+			input.proposed_name,
+			input.proposed_content,
+			input.proposed_tags ?? "[]",
+			input.proposed_file_paths ?? "[]",
+			input.trust_tier ?? 4,
+			input.raw_confidence,
+			"pending",
+			createdAt,
+			expiresAt,
+		],
+	);
+
+	await writeAuditEntry(db, "STAGE", {
+		entity_id: id,
+		trust_tier: input.trust_tier ?? 4,
+		source_episode: input.source_episode,
+	});
+
+	return id;
+}
+
+/**
+ * Retrieve pending staged facts that have not yet expired.
+ *
+ * Returns rows WHERE `validation_status = 'pending' AND expires_at > now`,
+ * ordered by `created_at ASC`, limited to `limit` rows (default 100).
+ */
+export async function getPendingStagedFacts(db: SiaDb, limit = 100): Promise<StagedFact[]> {
+	const now = Date.now();
+	const result = await db.execute(
+		`SELECT * FROM memory_staging
+		 WHERE validation_status = 'pending' AND expires_at > ?
+		 ORDER BY created_at ASC
+		 LIMIT ?`,
+		[now, limit],
+	);
+	return result.rows as unknown as StagedFact[];
+}
+
+/**
+ * Update the validation status of a staged fact.
+ *
+ * Optionally sets `rejection_reason`. Writes a QUARANTINE audit entry
+ * when the status is 'rejected' or 'quarantined'.
+ */
+export async function updateStagingStatus(
+	db: SiaDb,
+	id: string,
+	status: string,
+	rejectionReason?: string,
+): Promise<void> {
+	await db.execute(
+		`UPDATE memory_staging
+		 SET validation_status = ?, rejection_reason = ?
+		 WHERE id = ?`,
+		[status, rejectionReason ?? null, id],
+	);
+
+	if (status === "rejected" || status === "quarantined") {
+		await writeAuditEntry(db, "QUARANTINE", { entity_id: id });
+	}
+}
+
+/**
+ * Expire stale staged facts whose `expires_at` has passed while still pending.
+ *
+ * Sets `validation_status = 'expired'` for all matching rows.
+ * Returns the number of rows affected.
+ */
+export async function expireStaleStagedFacts(db: SiaDb): Promise<number> {
+	const now = Date.now();
+
+	// Get count of rows that will be affected before updating
+	const countResult = await db.execute(
+		`SELECT COUNT(*) as cnt FROM memory_staging
+		 WHERE expires_at <= ? AND validation_status = 'pending'`,
+		[now],
+	);
+	const count = (countResult.rows[0]?.cnt as number) ?? 0;
+
+	if (count > 0) {
+		await db.execute(
+			`UPDATE memory_staging
+			 SET validation_status = 'expired'
+			 WHERE expires_at <= ? AND validation_status = 'pending'`,
+			[now],
+		);
+	}
+
+	return count;
+}