@myrialabs/clopen 0.0.1

package/backend/lib/database/README.md

@@ -0,0 +1,77 @@
+# Database System
+
+## ➕ Adding New Migration
+
+1. Create file: `migrations/006_description.ts`
+```typescript
+export const description = 'Create new table';
+
+export function up(db) {
+    db.exec('CREATE TABLE ...');
+}
+
+export function down(db) {
+    db.exec('DROP TABLE ...');
+}
+```
+
+2. Add to `migrations/index.ts`:
+```typescript
+import * as migration006 from './006_description';
+
+export const migrations = [
+    // ... existing migrations
+    {
+        id: '006',
+        description: migration006.description,
+        up: migration006.up,
+        down: migration006.down
+    }
+];
+```
+
+3. **Restart app** → Migration runs automatically ✅
+
+## ➕ Adding New Seeder
+
+1. Create file: `seeders/new_seeder.ts`
+```typescript
+export const description = 'Seed default data';
+
+export function seed(db) {
+    // Insert default data
+}
+```
+
+2. Add to `seeders/index.ts`:
+```typescript
+import * as newSeeder from './new_seeder';
+
+export const seeders = [
+    // ... existing seeders
+    {
+        name: 'new_seeder',
+        description: newSeeder.description,
+        seed: newSeeder.seed
+    }
+];
+```
+
+3. **Restart app** → Seeder runs automatically ✅
+
+## 🔧 Important Info
+
+- **Automatic**: Migrations/seeders run on app startup
+- **Safe**: Only new migrations/seeders execute
+- **Tracked**: System remembers what's already done
+- **Order**: Migrations run by ID (001, 002, 003...)
+
+## 🛠️ Development Commands
+
+```typescript
+// Reset database (dev only)
+await resetDatabase();
+
+// Check status
+const pending = migrationRunner.getPendingMigrations();
+```

package/backend/lib/database/index.ts

@@ -0,0 +1,119 @@
+import type { DatabaseConnection } from '$shared/types/database/connection';
+import { DatabaseManager, MigrationRunner, SeederRunner } from './utils';
+import { migrations } from './migrations';
+import { seeders } from './seeders';
+import { debug } from '$shared/utils/logger';
+
+// Database manager instance
+let dbManager: DatabaseManager | null = null;
+
+export async function initializeDatabase(): Promise<DatabaseConnection> {
+	debug.log('database', '🔧 Initializing database system...');
+
+	try {
+		// Get database manager instance
+		dbManager = DatabaseManager.getInstance();
+
+		// Connect to database
+		const db = await dbManager.connect();
+
+		// Run migrations
+		await runMigrations(db);
+
+		// Run seeders
+		await runSeeders(db);
+
+		debug.log('database', '✅ Database system initialized successfully');
+		return db;
+
+	} catch (error) {
+		debug.error('database', '❌ Failed to initialize database:', error);
+		throw error;
+	}
+}
+
+async function runMigrations(db: DatabaseConnection): Promise<void> {
+	debug.log('database', '📋 Setting up migrations...');
+
+	const migrationRunner = new MigrationRunner(db);
+
+	// Add all migrations
+	for (const migration of migrations) {
+		migrationRunner.addMigration(migration);
+	}
+
+	// Run migrations
+	await migrationRunner.runMigrations();
+
+	debug.log('database', '✅ Migrations completed');
+}
+
+async function runSeeders(db: DatabaseConnection): Promise<void> {
+	debug.log('database', '🌱 Setting up seeders...');
+
+	const seederRunner = new SeederRunner(db);
+
+	// Add all seeders
+	for (const seeder of seeders) {
+		seederRunner.addSeeder(seeder);
+	}
+
+	// Run seeders (only if not already executed)
+	await seederRunner.runSeeders();
+
+	debug.log('database', '✅ Seeders completed');
+}
+
+export function getDatabase(): DatabaseConnection {
+	if (!dbManager || !dbManager.isConnected()) {
+		throw new Error('Database not initialized. Call initializeDatabase() first.');
+	}
+	return dbManager.getConnection();
+}
+
+export function closeDatabase(): void {
+	if (dbManager) {
+		dbManager.close();
+		dbManager = null;
+		debug.log('database', '✅ Database system shutdown');
+	}
+}
+
+export async function resetDatabase(): Promise<void> {
+	debug.log('database', '⚠️ Resetting database system...');
+
+	if (!dbManager) {
+		throw new Error('Database not initialized');
+	}
+
+	// Reset database (drop all tables)
+	await dbManager.resetDatabase();
+
+	// Re-run migrations and seeders
+	const db = dbManager.getConnection();
+	await runMigrations(db);
+	await runSeeders(db);
+
+	debug.log('database', '✅ Database system reset completed');
+}
+
+export async function getDatabaseInfo(): Promise<object> {
+	if (!dbManager) {
+		throw new Error('Database not initialized');
+	}
+
+	return dbManager.getDatabaseInfo();
+}
+
+export async function vacuumDatabase(): Promise<void> {
+	if (!dbManager) {
+		throw new Error('Database not initialized');
+	}
+
+	await dbManager.vacuum();
+}
+
+// Export utilities for advanced usage
+export { DatabaseManager, MigrationRunner, SeederRunner } from './utils';
+export { migrations } from './migrations';
+export { seeders } from './seeders';

package/backend/lib/database/migrations/001_create_projects_table.ts

@@ -0,0 +1,31 @@
+import type { DatabaseConnection } from '$shared/types/database/connection';
+
+import { debug } from '$shared/utils/logger';
+export const description = 'Create projects table for storing project information';
+
+export const up = (db: DatabaseConnection): void => {
+	debug.log('migration', '📋 Creating projects table...');
+	
+	db.exec(`
+		CREATE TABLE projects (
+			id TEXT PRIMARY KEY,
+			name TEXT NOT NULL,
+			path TEXT NOT NULL UNIQUE,
+			created_at TEXT NOT NULL,
+			last_opened_at TEXT NOT NULL
+		)
+	`);
+
+	// Create index for performance
+	db.exec(`
+		CREATE INDEX idx_projects_last_opened ON projects(last_opened_at)
+	`);
+
+	debug.log('migration', '✅ Projects table created');
+};
+
+export const down = (db: DatabaseConnection): void => {
+	debug.log('migration', '🗑️ Dropping projects table...');
+	db.exec('DROP TABLE IF EXISTS projects');
+	debug.log('migration', '✅ Projects table dropped');
+};

package/backend/lib/database/migrations/002_create_chat_sessions_table.ts

@@ -0,0 +1,33 @@
+import type { DatabaseConnection } from '$shared/types/database/connection';
+
+import { debug } from '$shared/utils/logger';
+export const description = 'Create chat_sessions table for storing chat conversation sessions';
+
+export const up = (db: DatabaseConnection): void => {
+	debug.log('migration', '📋 Creating chat_sessions table...');
+	
+	db.exec(`
+		CREATE TABLE chat_sessions (
+			id TEXT PRIMARY KEY,
+			project_id TEXT NOT NULL,
+			title TEXT,
+			latest_sdk_session_id TEXT,
+			started_at TEXT NOT NULL,
+			ended_at TEXT,
+			FOREIGN KEY (project_id) REFERENCES projects(id)
+		)
+	`);
+
+	// Create index for performance
+	db.exec(`
+		CREATE INDEX idx_chat_sessions_project_id ON chat_sessions(project_id)
+	`);
+
+	debug.log('migration', '✅ Chat sessions table created');
+};
+
+export const down = (db: DatabaseConnection): void => {
+	debug.log('migration', '🗑️ Dropping chat_sessions table...');
+	db.exec('DROP TABLE IF EXISTS chat_sessions');
+	debug.log('migration', '✅ Chat sessions table dropped');
+};

package/backend/lib/database/migrations/003_create_messages_table.ts

@@ -0,0 +1,32 @@
+import type { DatabaseConnection } from '$shared/types/database/connection';
+
+import { debug } from '$shared/utils/logger';
+export const description = 'Create messages table for storing SDKMessage data as JSON';
+
+export const up = (db: DatabaseConnection): void => {
+	debug.log('migration', '📋 Creating messages table...');
+	
+	db.exec(`
+		CREATE TABLE messages (
+			id TEXT PRIMARY KEY,
+			session_id TEXT NOT NULL,
+			timestamp TEXT NOT NULL,
+			sdk_message TEXT NOT NULL,
+			FOREIGN KEY (session_id) REFERENCES chat_sessions(id)
+		)
+	`);
+
+	// Create indexes for performance
+	db.exec(`
+		CREATE INDEX idx_messages_session_id ON messages(session_id);
+		CREATE INDEX idx_messages_timestamp ON messages(timestamp)
+	`);
+
+	debug.log('migration', '✅ Messages table created');
+};
+
+export const down = (db: DatabaseConnection): void => {
+	debug.log('migration', '🗑️ Dropping messages table...');
+	db.exec('DROP TABLE IF EXISTS messages');
+	debug.log('migration', '✅ Messages table dropped');
+};

package/backend/lib/database/migrations/004_create_prompt_templates_table.ts

@@ -0,0 +1,34 @@
+import type { DatabaseConnection } from '$shared/types/database/connection';
+
+import { debug } from '$shared/utils/logger';
+export const description = 'Create prompt_templates table for storing reusable prompt templates';
+
+export const up = (db: DatabaseConnection): void => {
+	debug.log('migration', '📋 Creating prompt_templates table...');
+
+	db.exec(`
+		CREATE TABLE IF NOT EXISTS prompt_templates (
+			id TEXT PRIMARY KEY,
+			name TEXT NOT NULL,
+			category TEXT NOT NULL,
+			content TEXT NOT NULL,
+			description TEXT NOT NULL,
+			created_at TEXT NOT NULL,
+			tags TEXT,
+			usage_count INTEGER DEFAULT 0
+		)
+	`);
+
+	// Create index for performance
+	db.exec(`
+		CREATE INDEX IF NOT EXISTS idx_prompt_templates_category ON prompt_templates(category)
+	`);
+
+	debug.log('migration', '✅ Prompt templates table created');
+};
+
+export const down = (db: DatabaseConnection): void => {
+	debug.log('migration', '🗑️ Dropping prompt_templates table...');
+	db.exec('DROP TABLE IF EXISTS prompt_templates');
+	debug.log('migration', '✅ Prompt templates table dropped');
+};

package/backend/lib/database/migrations/005_create_settings_table.ts

@@ -0,0 +1,24 @@
+import type { DatabaseConnection } from '$shared/types/database/connection';
+
+import { debug } from '$shared/utils/logger';
+export const description = 'Create settings table for storing application configuration';
+
+export const up = (db: DatabaseConnection): void => {
+	debug.log('migration', '📋 Creating settings table...');
+	
+	db.exec(`
+		CREATE TABLE settings (
+			key TEXT PRIMARY KEY,
+			value TEXT NOT NULL,
+			updated_at TEXT NOT NULL
+		)
+	`);
+
+	debug.log('migration', '✅ Settings table created');
+};
+
+export const down = (db: DatabaseConnection): void => {
+	debug.log('migration', '🗑️ Dropping settings table...');
+	db.exec('DROP TABLE IF EXISTS settings');
+	debug.log('migration', '✅ Settings table dropped');
+};

package/backend/lib/database/migrations/006_add_user_to_messages.ts

@@ -0,0 +1,58 @@
+import type { DatabaseConnection } from '$shared/types/database/connection';
+
+import { debug } from '$shared/utils/logger';
+export const description = 'Add user identification fields to messages table for shared chat';
+
+export const up = (db: DatabaseConnection): void => {
+	debug.log('migration', '📋 Adding user fields to messages table...');
+	
+	// Add sender_id and sender_name columns to messages table
+	db.exec(`
+		ALTER TABLE messages 
+		ADD COLUMN sender_id TEXT;
+	`);
+	
+	db.exec(`
+		ALTER TABLE messages 
+		ADD COLUMN sender_name TEXT;
+	`);
+	
+	// Create index for sender_id for performance
+	db.exec(`
+		CREATE INDEX idx_messages_sender_id ON messages(sender_id);
+	`);
+	
+	debug.log('migration', '✅ User fields added to messages table');
+};
+
+export const down = (db: DatabaseConnection): void => {
+	debug.log('migration', '🗑️ Removing user fields from messages table...');
+	
+	// SQLite doesn't support dropping columns directly, need to recreate table
+	db.exec(`
+		-- Create temporary table without user fields
+		CREATE TABLE messages_temp (
+			id TEXT PRIMARY KEY,
+			session_id TEXT NOT NULL,
+			timestamp TEXT NOT NULL,
+			sdk_message TEXT NOT NULL,
+			FOREIGN KEY (session_id) REFERENCES chat_sessions(id)
+		);
+		
+		-- Copy data
+		INSERT INTO messages_temp (id, session_id, timestamp, sdk_message)
+		SELECT id, session_id, timestamp, sdk_message FROM messages;
+		
+		-- Drop original table
+		DROP TABLE messages;
+		
+		-- Rename temp table
+		ALTER TABLE messages_temp RENAME TO messages;
+		
+		-- Recreate indexes
+		CREATE INDEX idx_messages_session_id ON messages(session_id);
+		CREATE INDEX idx_messages_timestamp ON messages(timestamp);
+	`);
+	
+	debug.log('migration', '✅ User fields removed from messages table');
+};

package/backend/lib/database/migrations/007_create_stream_states_table.ts

@@ -0,0 +1,41 @@
+import type { DatabaseConnection } from '$shared/types/database/connection';
+
+import { debug } from '$shared/utils/logger';
+export const description = 'Create stream states table for persistent background streaming';
+
+export const up = (db: DatabaseConnection): void => {
+  debug.log('migration', '📋 Creating stream_states table...');
+  
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS stream_states (
+      stream_id TEXT PRIMARY KEY,
+      chat_session_id TEXT NOT NULL,
+      project_id TEXT,
+      process_id TEXT NOT NULL,
+      status TEXT NOT NULL CHECK(status IN ('active', 'completed', 'error', 'cancelled')),
+      started_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+      completed_at DATETIME,
+      last_message_index INTEGER NOT NULL DEFAULT 0,
+      error TEXT,
+      created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+      updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+      FOREIGN KEY (chat_session_id) REFERENCES chat_sessions (id) ON DELETE CASCADE
+    )
+  `);
+
+  db.exec(`
+    CREATE INDEX idx_stream_states_session ON stream_states(chat_session_id)
+  `);
+  
+  db.exec(`
+    CREATE INDEX idx_stream_states_status ON stream_states(status)
+  `);
+  
+  debug.log('migration', '✅ stream_states table created successfully');
+};
+
+export const down = (db: DatabaseConnection): void => {
+  debug.log('migration', '🗑️ Dropping stream_states table...');
+  db.exec(`DROP TABLE IF EXISTS stream_states`);
+  debug.log('migration', '✅ stream_states table dropped');
+};

package/backend/lib/database/migrations/008_create_message_snapshots_table.ts

@@ -0,0 +1,62 @@
+/**
+ * Migration: Create message snapshots table for time travel feature
+ * Purpose: Store file snapshots for each user message to enable restoration
+ */
+
+export const description = 'Create message snapshots table';
+
+export function up(db: any): void {
+	// Create message_snapshots table for storing project state at each message
+	db.prepare(`
+		CREATE TABLE IF NOT EXISTS message_snapshots (
+			id TEXT PRIMARY KEY,
+			message_id TEXT NOT NULL,
+			session_id TEXT NOT NULL,
+			project_id TEXT NOT NULL,
+			files_snapshot TEXT NOT NULL, -- JSON object with file paths and contents
+			project_metadata TEXT, -- JSON object with project info (name, path, etc)
+			created_at TEXT NOT NULL,
+			FOREIGN KEY (message_id) REFERENCES messages (id) ON DELETE CASCADE,
+			FOREIGN KEY (session_id) REFERENCES chat_sessions (id) ON DELETE CASCADE,
+			FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE
+		)
+	`).run();
+
+	// Create indexes for efficient queries
+	db.prepare('CREATE INDEX IF NOT EXISTS idx_message_snapshots_message_id ON message_snapshots (message_id)').run();
+	db.prepare('CREATE INDEX IF NOT EXISTS idx_message_snapshots_session_id ON message_snapshots (session_id)').run();
+	db.prepare('CREATE INDEX IF NOT EXISTS idx_message_snapshots_project_id ON message_snapshots (project_id)').run();
+	db.prepare('CREATE INDEX IF NOT EXISTS idx_message_snapshots_created_at ON message_snapshots (created_at)').run();
+
+	// Create session_relationships table for tracking parent-child relationships
+	db.prepare(`
+		CREATE TABLE IF NOT EXISTS session_relationships (
+			id TEXT PRIMARY KEY,
+			parent_session_id TEXT NOT NULL,
+			child_session_id TEXT NOT NULL,
+			branched_from_message_id TEXT,
+			created_at TEXT NOT NULL,
+			FOREIGN KEY (parent_session_id) REFERENCES chat_sessions (id) ON DELETE CASCADE,
+			FOREIGN KEY (child_session_id) REFERENCES chat_sessions (id) ON DELETE CASCADE,
+			FOREIGN KEY (branched_from_message_id) REFERENCES messages (id) ON DELETE SET NULL
+		)
+	`).run();
+
+	// Create indexes for session relationships
+	db.prepare('CREATE INDEX IF NOT EXISTS idx_session_relationships_parent ON session_relationships (parent_session_id)').run();
+	db.prepare('CREATE INDEX IF NOT EXISTS idx_session_relationships_child ON session_relationships (child_session_id)').run();
+}
+
+export function down(db: any): void {
+	// Drop indexes
+	db.prepare('DROP INDEX IF EXISTS idx_session_relationships_child').run();
+	db.prepare('DROP INDEX IF EXISTS idx_session_relationships_parent').run();
+	db.prepare('DROP INDEX IF EXISTS idx_message_snapshots_created_at').run();
+	db.prepare('DROP INDEX IF EXISTS idx_message_snapshots_project_id').run();
+	db.prepare('DROP INDEX IF EXISTS idx_message_snapshots_session_id').run();
+	db.prepare('DROP INDEX IF EXISTS idx_message_snapshots_message_id').run();
+
+	// Drop tables
+	db.prepare('DROP TABLE IF EXISTS session_relationships').run();
+	db.prepare('DROP TABLE IF EXISTS message_snapshots').run();
+}

package/backend/lib/database/migrations/009_add_delta_snapshot_fields.ts

@@ -0,0 +1,41 @@
+import type { DatabaseConnection } from '$shared/types/database/connection';
+
+import { debug } from '$shared/utils/logger';
+export const description = 'Add delta snapshot support for efficient storage';
+
+export const up = (db: DatabaseConnection): void => {
+	debug.log('migration', '📋 Adding delta snapshot fields...');
+
+	// Add new columns for delta snapshots
+	db.exec(`
+		ALTER TABLE message_snapshots
+		ADD COLUMN snapshot_type TEXT DEFAULT 'full' CHECK(snapshot_type IN ('full', 'delta'))
+	`);
+
+	db.exec(`
+		ALTER TABLE message_snapshots
+		ADD COLUMN parent_snapshot_id TEXT
+	`);
+
+	db.exec(`
+		ALTER TABLE message_snapshots
+		ADD COLUMN delta_changes TEXT
+	`);
+
+	// Create index for parent lookup
+	db.exec(`
+		CREATE INDEX idx_message_snapshots_parent ON message_snapshots(parent_snapshot_id)
+	`);
+
+	debug.log('migration', '✅ Delta snapshot fields added');
+};
+
+export const down = (db: DatabaseConnection): void => {
+	debug.log('migration', '🗑️ Removing delta snapshot fields...');
+
+	// SQLite doesn't support DROP COLUMN easily, so we'd need to recreate table
+	// For now, this is a forward-only migration
+	debug.warn('migration', '⚠️ Rollback not implemented for delta snapshot fields');
+
+	debug.log('migration', '✅ Delta snapshot fields rollback skipped');
+};

package/backend/lib/database/migrations/010_add_soft_delete_and_branch_support.ts

@@ -0,0 +1,70 @@
+/**
+ * Migration: Add soft delete and branch support for undo/redo feature
+ * Purpose: Enable multi-branch redo functionality with soft deletes
+ */
+
+import type { DatabaseConnection } from '$shared/types/database/connection';
+
+import { debug } from '$shared/utils/logger';
+export const description = 'Add soft delete and branch support for multi-branch redo';
+
+export const up = (db: DatabaseConnection): void => {
+	debug.log('migration', '📋 Adding soft delete and branch support...');
+
+	// Add columns to messages table
+	db.exec(`
+		ALTER TABLE messages
+		ADD COLUMN is_deleted INTEGER DEFAULT 0 CHECK(is_deleted IN (0, 1))
+	`);
+
+	db.exec(`
+		ALTER TABLE messages
+		ADD COLUMN branch_id TEXT
+	`);
+
+	// Add columns to message_snapshots table
+	db.exec(`
+		ALTER TABLE message_snapshots
+		ADD COLUMN is_deleted INTEGER DEFAULT 0 CHECK(is_deleted IN (0, 1))
+	`);
+
+	db.exec(`
+		ALTER TABLE message_snapshots
+		ADD COLUMN branch_id TEXT
+	`);
+
+	// Create indexes for efficient queries
+	db.exec(`
+		CREATE INDEX idx_messages_is_deleted ON messages(is_deleted)
+	`);
+
+	db.exec(`
+		CREATE INDEX idx_messages_branch_id ON messages(branch_id)
+	`);
+
+	db.exec(`
+		CREATE INDEX idx_message_snapshots_is_deleted ON message_snapshots(is_deleted)
+	`);
+
+	db.exec(`
+		CREATE INDEX idx_message_snapshots_branch_id ON message_snapshots(branch_id)
+	`);
+
+	debug.log('migration', '✅ Soft delete and branch support added');
+};
+
+export const down = (db: DatabaseConnection): void => {
+	debug.log('migration', '🗑️ Removing soft delete and branch support...');
+
+	// Drop indexes
+	db.exec('DROP INDEX IF EXISTS idx_message_snapshots_branch_id');
+	db.exec('DROP INDEX IF EXISTS idx_message_snapshots_is_deleted');
+	db.exec('DROP INDEX IF EXISTS idx_messages_branch_id');
+	db.exec('DROP INDEX IF EXISTS idx_messages_is_deleted');
+
+	// SQLite doesn't support DROP COLUMN easily without recreating table
+	// For rollback, we'd need to recreate tables without these columns
+	debug.warn('migration', '⚠️ Full rollback not implemented - columns remain but unused');
+
+	debug.log('migration', '✅ Soft delete and branch support rollback completed');
+};