@exaudeus/workrail 0.1.0 → 0.1.2

package/dist/infrastructure/storage/sqlite-metadata-storage.d.ts

@@ -0,0 +1,35 @@
+import { IMetadataStorage, ContextStorageConfig, MetadataStats } from './context-storage';
+import { CheckpointMetadata, SessionInfo } from '../../types/context-types';
+export declare class SqliteMetadataStorage implements IMetadataStorage {
+    private db;
+    private migrator;
+    private config;
+    private isInitialized;
+    private heartbeatIntervals;
+    private cleanupInterval;
+    private statements;
+    constructor(config: ContextStorageConfig);
+    initialize(): Promise<void>;
+    saveCheckpointMetadata(metadata: CheckpointMetadata): Promise<void>;
+    getCheckpointMetadata(checkpointId: string): Promise<CheckpointMetadata | null>;
+    listCheckpointMetadata(sessionId: string, limit?: number, offset?: number): Promise<CheckpointMetadata[]>;
+    deleteCheckpointMetadata(checkpointId: string): Promise<void>;
+    upsertSessionInfo(session: SessionInfo): Promise<void>;
+    getSessionInfo(sessionId: string): Promise<SessionInfo | null>;
+    deleteSessionInfo(sessionId: string): Promise<void>;
+    acquireSessionLock(sessionId: string, operationType: string, timeoutMs?: number): Promise<string>;
+    releaseSessionLock(operationId: string): Promise<void>;
+    updateOperationHeartbeat(operationId: string): Promise<void>;
+    cleanupStaleOperations(): Promise<number>;
+    getMetadataStats(): Promise<MetadataStats>;
+    close(): Promise<void>;
+    private ensureInitialized;
+    private ensureDataDirectory;
+    private prepareStatements;
+    private mapRowToCheckpointMetadata;
+    private cleanupInterruptedOperations;
+    private startHeartbeat;
+    private stopHeartbeat;
+    private setupAutomaticCleanup;
+    private cleanup;
+}

package/dist/infrastructure/storage/sqlite-metadata-storage.js

@@ -0,0 +1,410 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SqliteMetadataStorage = void 0;
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const crypto_1 = __importDefault(require("crypto"));
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const sqlite_migrator_1 = require("./sqlite-migrator");
+class SqliteMetadataStorage {
+    constructor(config) {
+        this.db = null;
+        this.migrator = null;
+        this.isInitialized = false;
+        this.heartbeatIntervals = new Map();
+        this.cleanupInterval = null;
+        this.statements = {};
+        this.config = config;
+    }
+    async initialize() {
+        if (this.isInitialized) {
+            return;
+        }
+        try {
+            await this.ensureDataDirectory();
+            const dbPath = path_1.default.join(this.config.dataDirectory, this.config.database.path);
+            this.db = new better_sqlite3_1.default(dbPath, {
+                timeout: this.config.database.timeout,
+                verbose: undefined
+            });
+            if (this.config.database.walMode) {
+                this.db.pragma('journal_mode = WAL');
+            }
+            this.db.pragma('synchronous = NORMAL');
+            this.db.pragma('cache_size = 1000');
+            this.db.pragma('temp_store = memory');
+            this.db.pragma('foreign_keys = ON');
+            this.migrator = new sqlite_migrator_1.SqliteMigrator(dbPath);
+            await this.migrator.initialize();
+            const migrationResults = await this.migrator.migrate();
+            if (migrationResults.some(result => !result.success)) {
+                throw new Error('Database migration failed');
+            }
+            const isValid = await this.migrator.validateIntegrity();
+            if (!isValid) {
+                throw new Error('Database integrity validation failed');
+            }
+            await this.prepareStatements();
+            await this.cleanupInterruptedOperations();
+            this.setupAutomaticCleanup();
+            this.isInitialized = true;
+            console.log('✅ SQLite metadata storage initialized');
+        }
+        catch (error) {
+            await this.cleanup();
+            throw new Error(`Failed to initialize SQLite metadata storage: ${error}`);
+        }
+    }
+    async saveCheckpointMetadata(metadata) {
+        this.ensureInitialized();
+        try {
+            this.statements.insertCheckpoint.run(metadata.id, metadata.sessionId, metadata.name || null, metadata.agentId || null, metadata.createdAt, metadata.tags ? JSON.stringify(metadata.tags) : null, metadata.contextSizeBytes, metadata.contextHash, metadata.blobPath, metadata.status, null, 1.0, metadata.classificationSummary ? JSON.stringify(metadata.classificationSummary) : null);
+            console.log(`📦 Saved checkpoint metadata: ${metadata.id}`);
+        }
+        catch (error) {
+            throw new Error(`Failed to save checkpoint metadata: ${error}`);
+        }
+    }
+    async getCheckpointMetadata(checkpointId) {
+        this.ensureInitialized();
+        try {
+            const row = this.statements.selectCheckpoint.get(checkpointId);
+            if (!row) {
+                return null;
+            }
+            return this.mapRowToCheckpointMetadata(row);
+        }
+        catch (error) {
+            throw new Error(`Failed to get checkpoint metadata: ${error}`);
+        }
+    }
+    async listCheckpointMetadata(sessionId, limit = 20, offset = 0) {
+        this.ensureInitialized();
+        try {
+            const rows = this.statements.selectCheckpointsBySession.all(sessionId, limit, offset);
+            return rows.map(row => this.mapRowToCheckpointMetadata(row));
+        }
+        catch (error) {
+            throw new Error(`Failed to list checkpoint metadata: ${error}`);
+        }
+    }
+    async deleteCheckpointMetadata(checkpointId) {
+        this.ensureInitialized();
+        try {
+            const result = this.statements.deleteCheckpoint.run(checkpointId);
+            if (result.changes === 0) {
+                throw new Error(`Checkpoint ${checkpointId} not found`);
+            }
+            console.log(`🗑️ Deleted checkpoint metadata: ${checkpointId}`);
+        }
+        catch (error) {
+            throw new Error(`Failed to delete checkpoint metadata: ${error}`);
+        }
+    }
+    async upsertSessionInfo(session) {
+        this.ensureInitialized();
+        try {
+            this.statements.upsertSession.run(session.id, session.createdAt, session.lastAccessedAt, session.tags ? JSON.stringify(session.tags) : null, session.totalSizeBytes);
+            console.log(`📋 Upserted session: ${session.id}`);
+        }
+        catch (error) {
+            throw new Error(`Failed to upsert session: ${error}`);
+        }
+    }
+    async getSessionInfo(sessionId) {
+        this.ensureInitialized();
+        try {
+            const row = this.statements.selectSession.get(sessionId);
+            if (!row) {
+                return null;
+            }
+            return {
+                id: row.id,
+                createdAt: row.created_at,
+                lastAccessedAt: row.last_accessed_at,
+                tags: row.tags ? JSON.parse(row.tags) : undefined,
+                totalSizeBytes: row.total_size_bytes
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to get session info: ${error}`);
+        }
+    }
+    async deleteSessionInfo(sessionId) {
+        this.ensureInitialized();
+        try {
+            const result = this.statements.deleteSession.run(sessionId);
+            if (result.changes === 0) {
+                throw new Error(`Session ${sessionId} not found`);
+            }
+            console.log(`🗑️ Deleted session: ${sessionId}`);
+        }
+        catch (error) {
+            throw new Error(`Failed to delete session: ${error}`);
+        }
+    }
+    async acquireSessionLock(sessionId, operationType, timeoutMs) {
+        this.ensureInitialized();
+        const operationId = crypto_1.default.randomUUID();
+        const timeout = timeoutMs || this.config.concurrency.operationTimeoutMs;
+        const expiresAt = new Date(Date.now() + timeout).toISOString();
+        const maxRetries = this.config.concurrency.maxRetries;
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+            try {
+                await this.cleanupStaleOperations();
+                this.statements.insertActiveOperation.run(operationId, sessionId, operationType, new Date().toISOString(), new Date().toISOString(), timeout, JSON.stringify({ attempt, maxRetries }));
+                this.statements.acquireSessionLock.run(sessionId, operationId, new Date().toISOString(), expiresAt);
+                this.startHeartbeat(operationId);
+                console.log(`🔒 Acquired lock for session ${sessionId}, operation: ${operationId}`);
+                return operationId;
+            }
+            catch (error) {
+                if (attempt === maxRetries) {
+                    throw new Error(`Failed to acquire session lock for ${sessionId} after ${maxRetries} attempts: ${error}`);
+                }
+                const delay = Math.min(1000 * Math.pow(2, attempt - 1), 5000);
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
+        }
+        throw new Error(`Unable to acquire session lock for ${sessionId}`);
+    }
+    async releaseSessionLock(operationId) {
+        this.ensureInitialized();
+        try {
+            this.stopHeartbeat(operationId);
+            this.statements.releaseSessionLock.run(operationId);
+            this.statements.deleteActiveOperation.run(operationId);
+            console.log(`🔓 Released lock for operation: ${operationId}`);
+        }
+        catch (error) {
+            console.warn(`Warning: Failed to release session lock ${operationId}:`, error);
+        }
+    }
+    async updateOperationHeartbeat(operationId) {
+        this.ensureInitialized();
+        try {
+            this.statements.updateHeartbeat.run(new Date().toISOString(), operationId);
+        }
+        catch (error) {
+            console.warn(`Warning: Failed to update heartbeat for ${operationId}:`, error);
+        }
+    }
+    async cleanupStaleOperations() {
+        this.ensureInitialized();
+        try {
+            const result = this.statements.cleanupStaleOperations.run();
+            if (result.changes > 0) {
+                console.log(`🧹 Cleaned up ${result.changes} stale operations`);
+            }
+            return result.changes;
+        }
+        catch (error) {
+            console.warn('Warning: Failed to cleanup stale operations:', error);
+            return 0;
+        }
+    }
+    async getMetadataStats() {
+        this.ensureInitialized();
+        try {
+            const countsQuery = `
+        SELECT 
+          (SELECT COUNT(*) FROM sessions) as sessions,
+          (SELECT COUNT(*) FROM checkpoint_metadata) as checkpoints,
+          (SELECT COUNT(*) FROM active_operations) as active_ops,
+          (SELECT AVG(context_size_bytes) FROM checkpoint_metadata) as avg_size
+      `;
+            const counts = this.db.prepare(countsQuery).get();
+            const dbPath = path_1.default.join(this.config.dataDirectory, this.config.database.path);
+            const dbStats = fs_1.default.statSync(dbPath);
+            return {
+                databaseSizeBytes: dbStats.size,
+                totalSessions: counts.sessions || 0,
+                totalCheckpoints: counts.checkpoints || 0,
+                activeOperations: counts.active_ops || 0,
+                averageCheckpointSize: counts.avg_size || 0,
+                indexEfficiency: 1.0
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to get metadata stats: ${error}`);
+        }
+    }
+    async close() {
+        await this.cleanup();
+    }
+    ensureInitialized() {
+        if (!this.isInitialized || !this.db) {
+            throw new Error('SqliteMetadataStorage not initialized');
+        }
+    }
+    async ensureDataDirectory() {
+        try {
+            await fs_1.default.promises.mkdir(this.config.dataDirectory, {
+                recursive: true,
+                mode: this.config.blobs.directoryPermissions
+            });
+        }
+        catch (error) {
+            throw new Error(`Failed to create data directory: ${error}`);
+        }
+    }
+    async prepareStatements() {
+        if (!this.db)
+            return;
+        this.statements = {
+            insertCheckpoint: this.db.prepare(`
+        INSERT OR REPLACE INTO checkpoint_metadata (
+          id, session_id, name, agent_id, created_at, tags, 
+          context_size_bytes, context_hash, blob_path, status,
+          created_by_operation, compression_ratio, classification_info
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `),
+            selectCheckpoint: this.db.prepare(`
+        SELECT * FROM checkpoint_metadata WHERE id = ?
+      `),
+            selectCheckpointsBySession: this.db.prepare(`
+        SELECT * FROM checkpoint_metadata 
+        WHERE session_id = ? 
+        ORDER BY created_at DESC 
+        LIMIT ? OFFSET ?
+      `),
+            deleteCheckpoint: this.db.prepare(`
+        DELETE FROM checkpoint_metadata WHERE id = ?
+      `),
+            upsertSession: this.db.prepare(`
+        INSERT OR REPLACE INTO sessions (
+          id, created_at, last_accessed_at, tags, total_size_bytes
+        ) VALUES (?, ?, ?, ?, ?)
+      `),
+            selectSession: this.db.prepare(`
+        SELECT * FROM sessions WHERE id = ?
+      `),
+            deleteSession: this.db.prepare(`
+        DELETE FROM sessions WHERE id = ?
+      `),
+            insertActiveOperation: this.db.prepare(`
+        INSERT INTO active_operations (
+          id, session_id, operation_type, started_at, heartbeat_at, timeout_ms, metadata
+        ) VALUES (?, ?, ?, ?, ?, ?, ?)
+      `),
+            deleteActiveOperation: this.db.prepare(`
+        DELETE FROM active_operations WHERE id = ?
+      `),
+            updateHeartbeat: this.db.prepare(`
+        UPDATE active_operations SET heartbeat_at = ? WHERE id = ?
+      `),
+            cleanupStaleOperations: this.db.prepare(`
+        DELETE FROM active_operations 
+        WHERE heartbeat_at < datetime('now', '-5 minutes') 
+           OR started_at < datetime('now', '-10 minutes')
+      `),
+            acquireSessionLock: this.db.prepare(`
+        UPDATE sessions 
+        SET locked_at = ?, locked_by = ?, lock_timeout_at = ?
+        WHERE id = ? AND (locked_at IS NULL OR lock_timeout_at < CURRENT_TIMESTAMP)
+      `),
+            releaseSessionLock: this.db.prepare(`
+        UPDATE sessions 
+        SET locked_at = NULL, locked_by = NULL, lock_timeout_at = NULL
+        WHERE locked_by = ?
+      `),
+            checkSessionLock: this.db.prepare(`
+        SELECT locked_by, lock_timeout_at FROM sessions 
+        WHERE id = ? AND locked_at IS NOT NULL
+      `)
+        };
+    }
+    mapRowToCheckpointMetadata(row) {
+        return {
+            id: row.id,
+            sessionId: row.session_id,
+            name: row.name || undefined,
+            agentId: row.agent_id || undefined,
+            createdAt: row.created_at,
+            tags: row.tags ? JSON.parse(row.tags) : undefined,
+            contextSizeBytes: row.context_size_bytes,
+            contextHash: row.context_hash,
+            blobPath: row.blob_path,
+            status: row.status,
+            classificationSummary: row.classification_info ? JSON.parse(row.classification_info) : undefined
+        };
+    }
+    async cleanupInterruptedOperations() {
+        try {
+            const cleanupCount = await this.cleanupStaleOperations();
+            if (cleanupCount > 0) {
+                console.log(`🔄 Cleaned up ${cleanupCount} interrupted operations from previous session`);
+            }
+            this.db.prepare(`
+        UPDATE sessions 
+        SET locked_at = NULL, locked_by = NULL, lock_timeout_at = NULL
+        WHERE lock_timeout_at < CURRENT_TIMESTAMP
+      `).run();
+        }
+        catch (error) {
+            console.warn('Warning: Failed to cleanup interrupted operations:', error);
+        }
+    }
+    startHeartbeat(operationId) {
+        const interval = setInterval(async () => {
+            try {
+                await this.updateOperationHeartbeat(operationId);
+            }
+            catch (error) {
+                console.warn(`Heartbeat failed for operation ${operationId}:`, error);
+                this.stopHeartbeat(operationId);
+            }
+        }, this.config.concurrency.heartbeatIntervalMs);
+        this.heartbeatIntervals.set(operationId, interval);
+    }
+    stopHeartbeat(operationId) {
+        const interval = this.heartbeatIntervals.get(operationId);
+        if (interval) {
+            clearInterval(interval);
+            this.heartbeatIntervals.delete(operationId);
+        }
+    }
+    setupAutomaticCleanup() {
+        this.cleanupInterval = setInterval(async () => {
+            try {
+                await this.cleanupStaleOperations();
+            }
+            catch (error) {
+                console.warn('Automatic cleanup failed:', error);
+            }
+        }, this.config.concurrency.cleanupIntervalMs);
+        process.on('exit', () => {
+            if (this.cleanupInterval) {
+                clearInterval(this.cleanupInterval);
+            }
+        });
+    }
+    async cleanup() {
+        try {
+            for (const interval of this.heartbeatIntervals.values()) {
+                clearInterval(interval);
+            }
+            this.heartbeatIntervals.clear();
+            if (this.cleanupInterval) {
+                clearInterval(this.cleanupInterval);
+                this.cleanupInterval = null;
+            }
+            if (this.migrator) {
+                this.migrator.close();
+                this.migrator = null;
+            }
+            if (this.db) {
+                this.db.close();
+                this.db = null;
+            }
+            this.isInitialized = false;
+        }
+        catch (error) {
+            console.warn('Warning during SQLite metadata storage cleanup:', error);
+        }
+    }
+}
+exports.SqliteMetadataStorage = SqliteMetadataStorage;

package/dist/infrastructure/storage/sqlite-migrator.d.ts

@@ -0,0 +1,46 @@
+export interface Migration {
+    version: number;
+    filename: string;
+    description: string;
+    sql: string;
+    checksum: string;
+}
+export interface MigrationResult {
+    success: boolean;
+    appliedVersion?: number;
+    error?: string;
+    rollbackPerformed?: boolean;
+    duration?: number;
+}
+export interface MigrationStatus {
+    currentVersion: number;
+    targetVersion: number;
+    pendingMigrations: Migration[];
+    appliedMigrations: AppliedMigration[];
+}
+export interface AppliedMigration {
+    version: number;
+    description: string;
+    appliedAt: string;
+    checksum: string;
+}
+export declare class SqliteMigrator {
+    private db;
+    private migrationsDir;
+    private readonly maxRetries;
+    private readonly lockTimeout;
+    constructor(databasePath: string, migrationsDir?: string);
+    initialize(): Promise<void>;
+    getCurrentVersion(): number;
+    loadAvailableMigrations(): Migration[];
+    getMigrationStatus(): MigrationStatus;
+    migrate(): Promise<MigrationResult[]>;
+    private applyMigration;
+    private acquireMigrationLock;
+    private releaseMigrationLock;
+    private validateMigrationChecksum;
+    private calculateChecksum;
+    validateIntegrity(): Promise<boolean>;
+    close(): void;
+    getStats(): Record<string, any>;
+}