lsh-framework 1.2.0 → 1.3.0

package/dist/cicd/data-retention.js

@@ -1,305 +0,0 @@
-import * as cron from 'node-cron';
-export class DataRetentionService {
-    pool;
-    redis;
-    policies;
-    archiveConfig;
-    cronJobs = [];
-    constructor(pool, redis) {
-        this.pool = pool;
-        this.redis = redis;
-        // Define retention policies
-        this.policies = [
-            { tableName: 'pipeline_events', retentionDays: 90, dateColumn: 'created_at', archiveBeforeDelete: true },
-            { tableName: 'build_metrics', retentionDays: 180, dateColumn: 'recorded_at', archiveBeforeDelete: true },
-            { tableName: 'audit_logs', retentionDays: 365, dateColumn: 'timestamp', archiveBeforeDelete: true },
-            { tableName: 'pipeline_stages', retentionDays: 90, dateColumn: 'started_at' },
-            { tableName: 'alerts', retentionDays: 30, dateColumn: 'created_at' },
-            { tableName: 'webhook_logs', retentionDays: 7, dateColumn: 'received_at' }
-        ];
-        this.archiveConfig = {
-            enabled: process.env.ENABLE_ARCHIVE === 'true',
-            s3Bucket: process.env.ARCHIVE_S3_BUCKET,
-            localPath: process.env.ARCHIVE_LOCAL_PATH || '/var/backups/cicd',
-            compress: true
-        };
-        this.setupCronJobs();
-    }
-    setupCronJobs() {
-        // Daily cleanup at 2 AM
-        const dailyCleanup = cron.schedule('0 2 * * *', async () => {
-            console.warn('Running daily data cleanup...');
-            await this.runCleanup();
-        });
-        // Weekly archive at Sunday 3 AM
-        const weeklyArchive = cron.schedule('0 3 * * 0', async () => {
-            console.warn('Running weekly data archive...');
-            await this.runArchive();
-        });
-        // Hourly Redis cleanup
-        const hourlyRedisCleanup = cron.schedule('0 * * * *', async () => {
-            console.warn('Running Redis cleanup...');
-            await this.cleanupRedis();
-        });
-        this.cronJobs.push(dailyCleanup, weeklyArchive, hourlyRedisCleanup);
-    }
-    async runCleanup() {
-        const results = [];
-        for (const policy of this.policies) {
-            try {
-                const result = await this.cleanupTable(policy);
-                results.push(result);
-            }
-            catch (error) {
-                console.error(`Error cleaning up ${policy.tableName}:`, error);
-                results.push({
-                    table: policy.tableName,
-                    status: 'error',
-                    error: error instanceof Error ? error.message : 'Unknown error'
-                });
-            }
-        }
-        // Log cleanup results
-        await this.logCleanupResults(results);
-        return results;
-    }
-    async cleanupTable(policy) {
-        const cutoffDate = new Date();
-        cutoffDate.setDate(cutoffDate.getDate() - policy.retentionDays);
-        // Archive if needed
-        if (policy.archiveBeforeDelete && this.archiveConfig.enabled) {
-            await this.archiveTableData(policy.tableName, policy.dateColumn, cutoffDate);
-        }
-        // Delete old records
-        const deleteQuery = `
-      DELETE FROM ${policy.tableName}
-      WHERE ${policy.dateColumn} < $1
-      RETURNING COUNT(*) as deleted_count
-    `;
-        const result = await this.pool.query(deleteQuery, [cutoffDate]);
-        const deletedCount = result.rowCount || 0;
-        // Vacuum the table to reclaim space
-        await this.pool.query(`VACUUM ANALYZE ${policy.tableName}`);
-        return {
-            table: policy.tableName,
-            status: 'success',
-            deletedRecords: deletedCount,
-            cutoffDate: cutoffDate.toISOString()
-        };
-    }
-    async archiveTableData(tableName, dateColumn, cutoffDate) {
-        const selectQuery = `
-      SELECT * FROM ${tableName}
-      WHERE ${dateColumn} < $1
-    `;
-        const result = await this.pool.query(selectQuery, [cutoffDate]);
-        if (result.rows.length === 0) {
-            return;
-        }
-        const archiveData = {
-            tableName,
-            archiveDate: new Date().toISOString(),
-            recordCount: result.rows.length,
-            cutoffDate: cutoffDate.toISOString(),
-            data: result.rows
-        };
-        // Save archive based on configuration
-        if (this.archiveConfig.s3Bucket) {
-            await this.saveToS3(archiveData);
-        }
-        else if (this.archiveConfig.localPath) {
-            await this.saveToLocal(archiveData);
-        }
-    }
-    async saveToS3(archiveData) {
-        // Implementation would use AWS SDK
-        console.warn(`Would save ${archiveData.recordCount} records from ${archiveData.tableName} to S3`);
-        // Placeholder for S3 upload logic
-    }
-    async saveToLocal(archiveData) {
-        const fs = await import('fs/promises');
-        const path = await import('path');
-        const zlib = await import('zlib');
-        const { promisify } = await import('util');
-        const gzip = promisify(zlib.gzip);
-        const timestamp = 'archiveDate' in archiveData && typeof archiveData.archiveDate === 'string'
-            ? archiveData.archiveDate.replace(/:/g, '-')
-            : 'weekOf' in archiveData && typeof archiveData.weekOf === 'string'
-                ? archiveData.weekOf
-                : new Date().toISOString().replace(/:/g, '-');
-        const fileName = `${archiveData.tableName}_${timestamp}.json`;
-        const filePath = path.join(this.archiveConfig.localPath, fileName);
-        let data = JSON.stringify(archiveData);
-        if (this.archiveConfig.compress) {
-            data = (await gzip(data)).toString('base64');
-            const compressedPath = filePath + '.gz';
-            await fs.writeFile(compressedPath, data);
-            console.warn(`Archived ${archiveData.recordCount} records to ${compressedPath}`);
-        }
-        else {
-            await fs.writeFile(filePath, data);
-            console.warn(`Archived ${archiveData.recordCount} records to ${filePath}`);
-        }
-    }
-    async cleanupRedis() {
-        const patterns = [
-            'metrics:*',
-            'durations:*',
-            'stage_durations:*',
-            'webhook:*'
-        ];
-        const cutoffTimestamp = Date.now() - (7 * 24 * 60 * 60 * 1000); // 7 days
-        for (const pattern of patterns) {
-            const keys = await this.redis.keys(pattern);
-            for (const key of keys) {
-                // Extract date from key if possible
-                const dateMatch = key.match(/(\d{4}-\d{2}-\d{2})/);
-                if (dateMatch) {
-                    const keyDate = new Date(dateMatch[1]);
-                    if (keyDate.getTime() < cutoffTimestamp) {
-                        await this.redis.del(key);
-                    }
-                }
-            }
-        }
-        // Clean up expired cache entries
-        const cacheKeys = await this.redis.keys('cicd:cache:*');
-        for (const key of cacheKeys) {
-            const ttl = await this.redis.ttl(key);
-            if (ttl === -1) {
-                // No expiration set, check age
-                await this.redis.expire(key, 3600); // Set 1 hour expiration
-            }
-        }
-        console.warn('Redis cleanup completed');
-    }
-    async runArchive() {
-        if (!this.archiveConfig.enabled) {
-            console.warn('Archiving is disabled');
-            return;
-        }
-        const tables = ['pipeline_events', 'build_metrics', 'audit_logs'];
-        const results = [];
-        for (const tableName of tables) {
-            try {
-                // Archive last week's data
-                const startDate = new Date();
-                startDate.setDate(startDate.getDate() - 7);
-                const endDate = new Date();
-                const query = `
-          SELECT * FROM ${tableName}
-          WHERE created_at >= $1 AND created_at < $2
-        `;
-                const result = await this.pool.query(query, [startDate, endDate]);
-                if (result.rows.length > 0) {
-                    const archiveData = {
-                        tableName,
-                        weekOf: startDate.toISOString().split('T')[0],
-                        recordCount: result.rows.length,
-                        data: result.rows
-                    };
-                    if (this.archiveConfig.s3Bucket) {
-                        await this.saveToS3(archiveData);
-                    }
-                    else if (this.archiveConfig.localPath) {
-                        await this.saveToLocal(archiveData);
-                    }
-                    results.push({
-                        table: tableName,
-                        status: 'success',
-                        recordsArchived: result.rows.length
-                    });
-                }
-            }
-            catch (error) {
-                console.error(`Error archiving ${tableName}:`, error);
-                results.push({
-                    table: tableName,
-                    status: 'error',
-                    error: error instanceof Error ? error.message : 'Unknown error'
-                });
-            }
-        }
-        return results;
-    }
-    async logCleanupResults(results) {
-        const query = `
-      INSERT INTO data_retention_logs (
-        execution_time, tables_processed, records_deleted, status, details
-      ) VALUES ($1, $2, $3, $4, $5)
-    `;
-        const totalDeleted = results
-            .filter(r => r.status === 'success')
-            .reduce((sum, r) => sum + (r.deletedRecords || 0), 0);
-        const status = results.every(r => r.status === 'success') ? 'success' :
-            results.some(r => r.status === 'success') ? 'partial' : 'failed';
-        await this.pool.query(query, [
-            new Date(),
-            results.length,
-            totalDeleted,
-            status,
-            JSON.stringify(results)
-        ]);
-    }
-    async getRetentionStats() {
-        const stats = [];
-        for (const policy of this.policies) {
-            const countQuery = `
-        SELECT
-          COUNT(*) as total_records,
-          MIN(${policy.dateColumn}) as oldest_record,
-          MAX(${policy.dateColumn}) as newest_record,
-          pg_size_pretty(pg_total_relation_size('${policy.tableName}')) as table_size
-        FROM ${policy.tableName}
-      `;
-            try {
-                const result = await this.pool.query(countQuery);
-                const row = result.rows[0];
-                stats.push({
-                    table: policy.tableName,
-                    retentionDays: policy.retentionDays,
-                    totalRecords: parseInt(row.total_records),
-                    oldestRecord: row.oldest_record,
-                    newestRecord: row.newest_record,
-                    tableSize: row.table_size,
-                    estimatedDeletions: await this.estimateDeletions(policy)
-                });
-            }
-            catch (error) {
-                console.error(`Error getting stats for ${policy.tableName}:`, error);
-            }
-        }
-        return stats;
-    }
-    async estimateDeletions(policy) {
-        const cutoffDate = new Date();
-        cutoffDate.setDate(cutoffDate.getDate() - policy.retentionDays);
-        const query = `
-      SELECT COUNT(*) as count
-      FROM ${policy.tableName}
-      WHERE ${policy.dateColumn} < $1
-    `;
-        try {
-            const result = await this.pool.query(query, [cutoffDate]);
-            return parseInt(result.rows[0].count);
-        }
-        catch (_error) {
-            return 0;
-        }
-    }
-    // Manual trigger for cleanup
-    async triggerCleanup(tableName) {
-        if (tableName) {
-            const policy = this.policies.find(p => p.tableName === tableName);
-            if (policy) {
-                return await this.cleanupTable(policy);
-            }
-            throw new Error(`No retention policy found for table: ${tableName}`);
-        }
-        return await this.runCleanup();
-    }
-    // Stop all cron jobs
-    stop() {
-        this.cronJobs.forEach(job => job.stop());
-    }
-}

package/dist/cicd/performance-monitor.js

@@ -1,224 +0,0 @@
-import { performance } from 'perf_hooks';
-import * as os from 'os';
-export class PerformanceMonitor {
-    requestMetrics = [];
-    dbQueryTimes = [];
-    eventLoopDelay = 0;
-    startTime;
-    alertThresholds;
-    constructor() {
-        this.startTime = Date.now();
-        this.alertThresholds = {
-            cpuUsage: 80,
-            memoryUsage: 85,
-            responseTime: 1000,
-            errorRate: 5
-        };
-        this.setupEventLoopMonitoring();
-        this.setupMemoryMonitoring();
-    }
-    setupEventLoopMonitoring() {
-        let lastCheck = process.hrtime.bigint();
-        setInterval(() => {
-            const now = process.hrtime.bigint();
-            const delay = Number(now - lastCheck) / 1e6 - 100; // Convert to ms and subtract interval
-            this.eventLoopDelay = Math.max(0, delay);
-            lastCheck = now;
-        }, 100);
-    }
-    setupMemoryMonitoring() {
-        // Monitor heap usage periodically
-        setInterval(() => {
-            const memUsage = process.memoryUsage();
-            const totalMem = os.totalmem();
-            const percentUsed = (memUsage.rss / totalMem) * 100;
-            if (percentUsed > this.alertThresholds.memoryUsage) {
-                this.triggerAlert('memory', `Memory usage at ${percentUsed.toFixed(2)}%`);
-            }
-        }, 30000); // Check every 30 seconds
-    }
-    recordRequest(metrics) {
-        this.requestMetrics.push(metrics);
-        // Keep only last 1000 requests in memory
-        if (this.requestMetrics.length > 1000) {
-            this.requestMetrics = this.requestMetrics.slice(-1000);
-        }
-        // Check for high response time
-        if (metrics.responseTime > this.alertThresholds.responseTime) {
-            this.triggerAlert('response_time', `Slow request: ${metrics.method} ${metrics.path} took ${metrics.responseTime}ms`);
-        }
-    }
-    recordDatabaseQuery(queryTime, query) {
-        this.dbQueryTimes.push(queryTime);
-        if (this.dbQueryTimes.length > 1000) {
-            this.dbQueryTimes = this.dbQueryTimes.slice(-1000);
-        }
-        // Alert for slow queries (> 100ms)
-        if (queryTime > 100) {
-            this.triggerAlert('slow_query', `Slow database query: ${queryTime}ms${query ? ` - ${query.substring(0, 100)}` : ''}`);
-        }
-    }
-    getMetrics() {
-        const now = Date.now();
-        const cpuUsage = process.cpuUsage();
-        const memUsage = process.memoryUsage();
-        // Calculate request metrics
-        const recentRequests = this.requestMetrics.filter(r => r.timestamp > now - 60000 // Last minute
-        );
-        const totalRequests = recentRequests.length;
-        const errors = recentRequests.filter(r => r.statusCode >= 500).length;
-        const avgResponseTime = totalRequests > 0
-            ? recentRequests.reduce((sum, r) => sum + r.responseTime, 0) / totalRequests
-            : 0;
-        // Calculate database metrics
-        const avgQueryTime = this.dbQueryTimes.length > 0
-            ? this.dbQueryTimes.reduce((a, b) => a + b, 0) / this.dbQueryTimes.length
-            : 0;
-        const slowQueries = this.dbQueryTimes.filter(t => t > 100).length;
-        const cpuPercent = (cpuUsage.user + cpuUsage.system) / 1000000 * 100;
-        const memPercent = (memUsage.rss / os.totalmem()) * 100;
-        // Check alert thresholds
-        if (cpuPercent > this.alertThresholds.cpuUsage) {
-            this.triggerAlert('cpu', `CPU usage at ${cpuPercent.toFixed(2)}%`);
-        }
-        const errorRate = totalRequests > 0 ? (errors / totalRequests) * 100 : 0;
-        if (errorRate > this.alertThresholds.errorRate) {
-            this.triggerAlert('error_rate', `Error rate at ${errorRate.toFixed(2)}%`);
-        }
-        return {
-            timestamp: now,
-            cpu: {
-                usage: cpuPercent,
-                loadAverage: os.loadavg()
-            },
-            memory: {
-                heapUsed: memUsage.heapUsed,
-                heapTotal: memUsage.heapTotal,
-                external: memUsage.external,
-                rss: memUsage.rss,
-                percentUsed: memPercent
-            },
-            eventLoop: {
-                delay: this.eventLoopDelay,
-                utilization: this.eventLoopDelay / 16.67 * 100 // Percentage of 60fps frame time
-            },
-            requests: {
-                total: totalRequests,
-                avgResponseTime,
-                errors,
-                errorRate
-            },
-            database: {
-                activeConnections: 0, // To be updated by connection pool
-                avgQueryTime,
-                slowQueries
-            }
-        };
-    }
-    getHealthStatus() {
-        const metrics = this.getMetrics();
-        let status = 'healthy';
-        const issues = [];
-        if (metrics.cpu.usage > this.alertThresholds.cpuUsage) {
-            status = 'degraded';
-            issues.push(`High CPU usage: ${metrics.cpu.usage.toFixed(2)}%`);
-        }
-        if (metrics.memory.percentUsed > this.alertThresholds.memoryUsage) {
-            status = 'degraded';
-            issues.push(`High memory usage: ${metrics.memory.percentUsed.toFixed(2)}%`);
-        }
-        if (metrics.requests.errorRate > this.alertThresholds.errorRate) {
-            status = 'unhealthy';
-            issues.push(`High error rate: ${metrics.requests.errorRate.toFixed(2)}%`);
-        }
-        if (metrics.eventLoop.delay > 50) {
-            status = status === 'healthy' ? 'degraded' : status;
-            issues.push(`Event loop delay: ${metrics.eventLoop.delay.toFixed(2)}ms`);
-        }
-        return {
-            status,
-            details: {
-                uptime: Math.floor((Date.now() - this.startTime) / 1000),
-                issues,
-                metrics: {
-                    cpu: `${metrics.cpu.usage.toFixed(2)}%`,
-                    memory: `${metrics.memory.percentUsed.toFixed(2)}%`,
-                    requests: metrics.requests.total,
-                    errors: metrics.requests.errors,
-                    avgResponseTime: `${metrics.requests.avgResponseTime.toFixed(2)}ms`
-                }
-            }
-        };
-    }
-    triggerAlert(type, message) {
-        console.warn(`[PERFORMANCE ALERT] ${type}: ${message}`);
-        // Here you could send to monitoring service, trigger notifications, etc.
-    }
-    // Middleware for Express
-    middleware() {
-        return (req, res, next) => {
-            const start = Date.now();
-            // Override res.end to capture response time
-            const originalEnd = res.end;
-            res.end = (...args) => {
-                const responseTime = Date.now() - start;
-                this.recordRequest({
-                    method: req.method,
-                    path: req.path,
-                    statusCode: res.statusCode,
-                    responseTime,
-                    timestamp: Date.now()
-                });
-                originalEnd.apply(res, args);
-            };
-            next();
-        };
-    }
-    // Graceful shutdown
-    async shutdown() {
-        const finalMetrics = this.getMetrics();
-        console.log('Final performance metrics:', finalMetrics);
-        // Could save to database or send to monitoring service
-        return finalMetrics;
-    }
-}
-// Singleton instance
-export const performanceMonitor = new PerformanceMonitor();
-// Helper to measure function execution time
-export function measurePerformance(target, propertyKey, descriptor) {
-    const originalMethod = descriptor.value;
-    descriptor.value = async function (...args) {
-        const start = performance.now();
-        try {
-            const result = await originalMethod.apply(this, args);
-            const duration = performance.now() - start;
-            console.log(`[PERF] ${propertyKey} took ${duration.toFixed(2)}ms`);
-            return result;
-        }
-        catch (error) {
-            const duration = performance.now() - start;
-            console.error(`[PERF] ${propertyKey} failed after ${duration.toFixed(2)}ms`);
-            throw error;
-        }
-    };
-    return descriptor;
-}
-// Helper to profile memory usage
-export function profileMemory(label) {
-    const before = process.memoryUsage();
-    return () => {
-        const after = process.memoryUsage();
-        const diff = {
-            rss: (after.rss - before.rss) / 1024 / 1024,
-            heapTotal: (after.heapTotal - before.heapTotal) / 1024 / 1024,
-            heapUsed: (after.heapUsed - before.heapUsed) / 1024 / 1024,
-            external: (after.external - before.external) / 1024 / 1024
-        };
-        console.log(`[MEMORY] ${label}:`, {
-            rss: `${diff.rss.toFixed(2)} MB`,
-            heapTotal: `${diff.heapTotal.toFixed(2)} MB`,
-            heapUsed: `${diff.heapUsed.toFixed(2)} MB`,
-            external: `${diff.external.toFixed(2)} MB`
-        });
-    };
-}