opencode-starterkit 1.0.1

package/baseline/plugin/lib/db/maintenance.ts

@@ -0,0 +1,312 @@
+/**
+ * Database Maintenance & Auxiliary Operations
+ *
+ * Memory files, FTS5 maintenance, archiving, and full maintenance cycle.
+ */
+
+import { purgeOldTemporalMessages } from "./pipeline.js";
+import { getMemoryDB } from "./schema.js";
+import type {
+	ArchiveOptions,
+	MaintenanceStats,
+	MemoryFileRow,
+} from "./types.js";
+
+// ============================================================================
+// Memory File Operations
+// ============================================================================
+
+/**
+ * Store or update a memory file.
+ */
+export function upsertMemoryFile(
+	filePath: string,
+	content: string,
+	mode: "replace" | "append" = "replace",
+): void {
+	const db = getMemoryDB();
+	const now = new Date();
+
+	db.run(
+		`
+    INSERT INTO memory_files (file_path, content, mode, created_at, created_at_epoch)
+    VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(file_path) DO UPDATE SET
+      content = CASE WHEN excluded.mode = 'append' THEN memory_files.content || '\n\n' || excluded.content ELSE excluded.content END,
+      mode = excluded.mode,
+      updated_at = ?,
+      updated_at_epoch = ?
+  `,
+		[
+			filePath,
+			content,
+			mode,
+			now.toISOString(),
+			now.getTime(),
+			now.toISOString(),
+			now.getTime(),
+		],
+	);
+}
+
+/**
+ * Get a memory file by path.
+ */
+export function getMemoryFile(filePath: string): MemoryFileRow | null {
+	const db = getMemoryDB();
+	return db
+		.query("SELECT * FROM memory_files WHERE file_path = ?")
+		.get(filePath) as MemoryFileRow | null;
+}
+
+// ============================================================================
+// FTS5 Maintenance
+// ============================================================================
+
+/**
+ * Optimize FTS5 indexes (run periodically).
+ */
+export function optimizeFTS5(): void {
+	const db = getMemoryDB();
+	db.run("INSERT INTO observations_fts(observations_fts) VALUES('optimize')");
+	try {
+		db.run(
+			"INSERT INTO distillations_fts(distillations_fts) VALUES('optimize')",
+		);
+	} catch {
+		// distillations_fts may not exist yet
+	}
+}
+
+/**
+ * Rebuild FTS5 indexes from scratch.
+ */
+export function rebuildFTS5(): void {
+	const db = getMemoryDB();
+	db.run("INSERT INTO observations_fts(observations_fts) VALUES('rebuild')");
+	try {
+		db.run(
+			"INSERT INTO distillations_fts(distillations_fts) VALUES('rebuild')",
+		);
+	} catch {
+		// distillations_fts may not exist yet
+	}
+}
+
+/**
+ * Check if FTS5 is available and working.
+ */
+export function checkFTS5Available(): boolean {
+	try {
+		const db = getMemoryDB();
+		db.query("SELECT * FROM observations_fts LIMIT 1").get();
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+// ============================================================================
+// Database Maintenance
+// ============================================================================
+
+/**
+ * Archive old observations to a separate table.
+ */
+export function archiveOldObservations(options: ArchiveOptions = {}): number {
+	const db = getMemoryDB();
+	const olderThanDays = options.olderThanDays ?? 90;
+	const includeSuperseded = options.includeSuperseded ?? true;
+	const dryRun = options.dryRun ?? false;
+
+	const cutoffEpoch = Date.now() - olderThanDays * 24 * 60 * 60 * 1000;
+
+	// Create archive table if not exists
+	db.run(`
+		CREATE TABLE IF NOT EXISTS observations_archive (
+			id INTEGER PRIMARY KEY,
+			type TEXT NOT NULL,
+			title TEXT NOT NULL,
+			subtitle TEXT,
+			facts TEXT,
+			narrative TEXT,
+			concepts TEXT,
+			files_read TEXT,
+			files_modified TEXT,
+			confidence TEXT,
+			bead_id TEXT,
+			supersedes INTEGER,
+			superseded_by INTEGER,
+			valid_until TEXT,
+			markdown_file TEXT,
+			source TEXT DEFAULT 'manual',
+			created_at TEXT NOT NULL,
+			created_at_epoch INTEGER NOT NULL,
+			updated_at TEXT,
+			archived_at TEXT NOT NULL
+		)
+	`);
+
+	// Build WHERE clause
+	let whereClause = `created_at_epoch < ${cutoffEpoch}`;
+	if (includeSuperseded) {
+		whereClause = `(${whereClause} OR superseded_by IS NOT NULL)`;
+	}
+
+	// Count candidates
+	const countResult = db
+		.query(`SELECT COUNT(*) as count FROM observations WHERE ${whereClause}`)
+		.get() as { count: number };
+
+	if (dryRun || countResult.count === 0) {
+		return countResult.count;
+	}
+
+	// Move to archive
+	const now = new Date().toISOString();
+	db.run(`
+		INSERT INTO observations_archive
+		SELECT *, '${now}' as archived_at FROM observations WHERE ${whereClause}
+	`);
+
+	// Delete from main table (triggers will remove from FTS)
+	db.run(`DELETE FROM observations WHERE ${whereClause}`);
+
+	return countResult.count;
+}
+
+/**
+ * Checkpoint WAL file back to main database.
+ */
+export function checkpointWAL(): { walSize: number; checkpointed: boolean } {
+	const db = getMemoryDB();
+
+	try {
+		const result = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get() as {
+			busy: number;
+			log: number;
+			checkpointed: number;
+		};
+
+		return {
+			walSize: result.log,
+			checkpointed: result.busy === 0,
+		};
+	} catch {
+		return { walSize: 0, checkpointed: false };
+	}
+}
+
+/**
+ * Vacuum database to reclaim space and defragment.
+ */
+export function vacuumDatabase(): boolean {
+	const db = getMemoryDB();
+	try {
+		db.run("VACUUM");
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Get database file sizes.
+ */
+export function getDatabaseSizes(): {
+	mainDb: number;
+	wal: number;
+	shm: number;
+	total: number;
+} {
+	const db = getMemoryDB();
+
+	try {
+		const pageCount = db.query("PRAGMA page_count").get() as {
+			page_count: number;
+		};
+		const pageSize = db.query("PRAGMA page_size").get() as {
+			page_size: number;
+		};
+		const mainDb = pageCount.page_count * pageSize.page_size;
+
+		const walResult = db.query("PRAGMA wal_checkpoint").get() as {
+			busy: number;
+			log: number;
+			checkpointed: number;
+		};
+		const wal = walResult.log * pageSize.page_size;
+
+		return {
+			mainDb,
+			wal,
+			shm: 32768,
+			total: mainDb + wal + 32768,
+		};
+	} catch {
+		return { mainDb: 0, wal: 0, shm: 0, total: 0 };
+	}
+}
+
+/**
+ * Get list of markdown files that exist in SQLite (for pruning).
+ */
+export function getMarkdownFilesInSqlite(): string[] {
+	const db = getMemoryDB();
+	const rows = db
+		.query(
+			"SELECT markdown_file FROM observations WHERE markdown_file IS NOT NULL",
+		)
+		.all() as { markdown_file: string }[];
+
+	return rows.map((r) => r.markdown_file);
+}
+
+/**
+ * Run full maintenance cycle.
+ */
+export function runFullMaintenance(
+	options: ArchiveOptions = {},
+): MaintenanceStats {
+	const sizesBefore = getDatabaseSizes();
+
+	// 1. Archive old observations
+	const archived = archiveOldObservations(options);
+
+	// 2. Purge old temporal messages
+	let purgedMessages = 0;
+	if (!options.dryRun) {
+		purgedMessages = purgeOldTemporalMessages();
+	}
+
+	// 3. Optimize FTS5
+	if (!options.dryRun) {
+		optimizeFTS5();
+	}
+
+	// 4. Checkpoint WAL
+	let checkpointed = false;
+	if (!options.dryRun) {
+		const walResult = checkpointWAL();
+		checkpointed = walResult.checkpointed;
+	}
+
+	// 5. Vacuum
+	let vacuumed = false;
+	if (!options.dryRun) {
+		vacuumed = vacuumDatabase();
+	}
+
+	const sizesAfter = getDatabaseSizes();
+
+	return {
+		archived,
+		vacuumed,
+		checkpointed,
+		prunedMarkdown: 0,
+		purgedMessages,
+		freedBytes: sizesBefore.total - sizesAfter.total,
+		dbSizeBefore: sizesBefore.total,
+		dbSizeAfter: sizesAfter.total,
+	};
+}

package/baseline/plugin/lib/db/observations.ts

@@ -0,0 +1,299 @@
+/**
+ * Observation Operations
+ *
+ * CRUD, search, timeline, and stats for the observations table.
+ * Uses FTS5 with porter stemming for full-text search with BM25 ranking.
+ */
+
+import { getMemoryDB } from "./schema.js";
+import type {
+	ObservationInput,
+	ObservationRow,
+	ObservationType,
+	SearchIndexResult,
+} from "./types.js";
+
+// ============================================================================
+// CRUD
+// ============================================================================
+
+/**
+ * Store a new observation in the database.
+ */
+export function storeObservation(input: ObservationInput): number {
+	const db = getMemoryDB();
+	const now = new Date();
+
+	const result = db
+		.query(
+			`
+    INSERT INTO observations (
+      type, title, subtitle, facts, narrative, concepts,
+      files_read, files_modified, confidence, bead_id,
+      supersedes, markdown_file, source, created_at, created_at_epoch
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `,
+		)
+		.run(
+			input.type,
+			input.title,
+			input.subtitle ?? null,
+			input.facts ? JSON.stringify(input.facts) : null,
+			input.narrative ?? null,
+			input.concepts ? JSON.stringify(input.concepts) : null,
+			input.files_read ? JSON.stringify(input.files_read) : null,
+			input.files_modified ? JSON.stringify(input.files_modified) : null,
+			input.confidence ?? "high",
+			input.bead_id ?? null,
+			input.supersedes ?? null,
+			input.markdown_file ?? null,
+			input.source ?? "manual",
+			now.toISOString(),
+			now.getTime(),
+		);
+
+	const insertedId = Number(result.lastInsertRowid);
+
+	// Update supersedes relationship
+	if (input.supersedes) {
+		db.run("UPDATE observations SET superseded_by = ? WHERE id = ?", [
+			insertedId,
+			input.supersedes,
+		]);
+	}
+
+	return insertedId;
+}
+
+/**
+ * Get observation by ID.
+ */
+export function getObservationById(id: number): ObservationRow | null {
+	const db = getMemoryDB();
+	return db
+		.query("SELECT * FROM observations WHERE id = ?")
+		.get(id) as ObservationRow | null;
+}
+
+/**
+ * Get multiple observations by IDs.
+ */
+export function getObservationsByIds(ids: number[]): ObservationRow[] {
+	if (ids.length === 0) return [];
+
+	const db = getMemoryDB();
+	const placeholders = ids.map(() => "?").join(",");
+	return db
+		.query(`SELECT * FROM observations WHERE id IN (${placeholders})`)
+		.all(...ids) as ObservationRow[];
+}
+
+// ============================================================================
+// Search
+// ============================================================================
+
+/**
+ * Search observations using FTS5.
+ * Returns compact index results for progressive disclosure.
+ */
+export function searchObservationsFTS(
+	query: string,
+	options: {
+		type?: ObservationType;
+		concepts?: string[];
+		limit?: number;
+	} = {},
+): SearchIndexResult[] {
+	const db = getMemoryDB();
+	const limit = options.limit ?? 10;
+
+	// Build FTS5 query — porter stemming handles word forms automatically
+	const ftsQuery = query
+		.replace(/['"]/g, '""')
+		.split(/\s+/)
+		.filter((term) => term.length > 0)
+		.map((term) => `"${term}"*`)
+		.join(" OR ");
+
+	if (!ftsQuery) {
+		// Empty query — return recent observations
+		return db
+			.query(
+				`
+      SELECT id, type, title,
+             substr(COALESCE(narrative, ''), 1, 100) as snippet,
+             created_at,
+             0 as relevance_score
+      FROM observations
+      WHERE superseded_by IS NULL
+      ${options.type ? "AND type = ?" : ""}
+      ORDER BY created_at_epoch DESC
+      LIMIT ?
+    `,
+			)
+			.all(
+				...(options.type ? [options.type, limit] : [limit]),
+			) as SearchIndexResult[];
+	}
+
+	try {
+		// Use FTS5 with BM25 ranking
+		let sql = `
+      SELECT o.id, o.type, o.title,
+             substr(COALESCE(o.narrative, ''), 1, 100) as snippet,
+             o.created_at,
+             bm25(observations_fts) as relevance_score
+      FROM observations o
+      JOIN observations_fts fts ON fts.rowid = o.id
+      WHERE observations_fts MATCH ?
+        AND o.superseded_by IS NULL
+    `;
+
+		const params: (string | number)[] = [ftsQuery];
+
+		if (options.type) {
+			sql += " AND o.type = ?";
+			params.push(options.type);
+		}
+
+		sql += " ORDER BY relevance_score LIMIT ?";
+		params.push(limit);
+
+		return db.query(sql).all(...params) as SearchIndexResult[];
+	} catch {
+		// FTS5 query failed, fallback to LIKE search
+		return fallbackLikeSearch(db, query, options.type, limit);
+	}
+}
+
+/**
+ * Fallback search using LIKE (for when FTS5 fails).
+ */
+function fallbackLikeSearch(
+	db: ReturnType<typeof getMemoryDB>,
+	query: string,
+	type: ObservationType | undefined,
+	limit: number,
+): SearchIndexResult[] {
+	const likePattern = `%${query}%`;
+
+	let sql = `
+    SELECT id, type, title,
+           substr(COALESCE(narrative, ''), 1, 100) as snippet,
+           created_at,
+           0 as relevance_score
+    FROM observations
+    WHERE superseded_by IS NULL
+      AND (title LIKE ? OR narrative LIKE ? OR concepts LIKE ?)
+  `;
+
+	const params: (string | number)[] = [likePattern, likePattern, likePattern];
+
+	if (type) {
+		sql += " AND type = ?";
+		params.push(type);
+	}
+
+	sql += " ORDER BY created_at_epoch DESC LIMIT ?";
+	params.push(limit);
+
+	return db.query(sql).all(...params) as SearchIndexResult[];
+}
+
+// ============================================================================
+// Timeline & Stats
+// ============================================================================
+
+/**
+ * Get timeline around an anchor observation.
+ */
+export function getTimelineAroundObservation(
+	anchorId: number,
+	depthBefore = 5,
+	depthAfter = 5,
+): {
+	anchor: ObservationRow | null;
+	before: SearchIndexResult[];
+	after: SearchIndexResult[];
+} {
+	const db = getMemoryDB();
+
+	const anchor = getObservationById(anchorId);
+	if (!anchor) {
+		return { anchor: null, before: [], after: [] };
+	}
+
+	const before = db
+		.query(
+			`
+    SELECT id, type, title,
+           substr(COALESCE(narrative, ''), 1, 100) as snippet,
+           created_at,
+           0 as relevance_score
+    FROM observations
+    WHERE created_at_epoch < ?
+      AND superseded_by IS NULL
+    ORDER BY created_at_epoch DESC
+    LIMIT ?
+  `,
+		)
+		.all(anchor.created_at_epoch, depthBefore) as SearchIndexResult[];
+
+	const after = db
+		.query(
+			`
+    SELECT id, type, title,
+           substr(COALESCE(narrative, ''), 1, 100) as snippet,
+           created_at,
+           0 as relevance_score
+    FROM observations
+    WHERE created_at_epoch > ?
+      AND superseded_by IS NULL
+    ORDER BY created_at_epoch ASC
+    LIMIT ?
+  `,
+		)
+		.all(anchor.created_at_epoch, depthAfter) as SearchIndexResult[];
+
+	return {
+		anchor,
+		before: before.reverse(),
+		after,
+	};
+}
+
+/**
+ * Get most recent observation.
+ */
+export function getMostRecentObservation(): ObservationRow | null {
+	const db = getMemoryDB();
+	return db
+		.query(
+			"SELECT * FROM observations WHERE superseded_by IS NULL ORDER BY created_at_epoch DESC LIMIT 1",
+		)
+		.get() as ObservationRow | null;
+}
+
+/**
+ * Get observation count by type.
+ */
+export function getObservationStats(): Record<string, number> {
+	const db = getMemoryDB();
+	const rows = db
+		.query(
+			`
+    SELECT type, COUNT(*) as count
+    FROM observations
+    WHERE superseded_by IS NULL
+    GROUP BY type
+  `,
+		)
+		.all() as { type: string; count: number }[];
+
+	const stats: Record<string, number> = { total: 0 };
+	for (const row of rows) {
+		stats[row.type] = row.count;
+		stats.total += row.count;
+	}
+	return stats;
+}