@kadi.build/file-manager 1.0.0

package/src/utils/FileStreamingUtils.js

@@ -0,0 +1,757 @@
+/**
+ * File Streaming Utilities
+ *
+ * Reusable utilities for efficient file streaming with range request support,
+ * progress tracking, and proper caching headers.
+ */
+
+import fs from 'fs';
+import { promises as fsPromises } from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+import { EventEmitter } from 'events';
+
+class FileStreamingUtils extends EventEmitter {
+  constructor(options = {}) {
+    super();
+
+    this.options = {
+      bufferSize: options.bufferSize || 64 * 1024,
+      progressInterval: options.progressInterval || 1024 * 1024,
+      enableETag: options.enableETag !== false,
+      enableLastModified: options.enableLastModified !== false,
+      enableCaching: options.enableCaching !== false,
+      maxCacheAge: options.maxCacheAge || 3600,
+      ...options
+    };
+  }
+
+  // ============================================================================
+  // MIME TYPE DETECTION
+  // ============================================================================
+
+  getMimeType(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+
+    const mimeTypes = {
+      // Text files
+      '.txt': 'text/plain',
+      '.md': 'text/markdown',
+      '.html': 'text/html',
+      '.htm': 'text/html',
+      '.css': 'text/css',
+      '.xml': 'application/xml',
+      '.yaml': 'application/x-yaml',
+      '.yml': 'application/x-yaml',
+
+      // JavaScript/JSON
+      '.js': 'application/javascript',
+      '.mjs': 'application/javascript',
+      '.json': 'application/json',
+      '.jsonl': 'application/jsonlines',
+
+      // Images
+      '.png': 'image/png',
+      '.jpg': 'image/jpeg',
+      '.jpeg': 'image/jpeg',
+      '.gif': 'image/gif',
+      '.webp': 'image/webp',
+      '.svg': 'image/svg+xml',
+      '.ico': 'image/x-icon',
+      '.bmp': 'image/bmp',
+      '.tiff': 'image/tiff',
+      '.tif': 'image/tiff',
+
+      // Archives
+      '.zip': 'application/zip',
+      '.tar': 'application/x-tar',
+      '.gz': 'application/gzip',
+      '.tgz': 'application/gzip',
+      '.bz2': 'application/x-bzip2',
+      '.xz': 'application/x-xz',
+      '.7z': 'application/x-7z-compressed',
+      '.rar': 'application/vnd.rar',
+
+      // Container registry specific
+      '.layer': 'application/vnd.docker.image.rootfs.diff.tar.gzip',
+      '.manifest': 'application/vnd.docker.distribution.manifest.v2+json',
+      '.config': 'application/vnd.docker.container.image.v1+json',
+
+      // Documents
+      '.pdf': 'application/pdf',
+      '.doc': 'application/msword',
+      '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      '.xls': 'application/vnd.ms-excel',
+      '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      '.ppt': 'application/vnd.ms-powerpoint',
+      '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+
+      // Audio/Video
+      '.mp3': 'audio/mpeg',
+      '.wav': 'audio/wav',
+      '.ogg': 'audio/ogg',
+      '.flac': 'audio/flac',
+      '.mp4': 'video/mp4',
+      '.avi': 'video/x-msvideo',
+      '.mov': 'video/quicktime',
+      '.wmv': 'video/x-ms-wmv',
+      '.webm': 'video/webm',
+      '.mkv': 'video/x-matroska',
+
+      // Programming languages
+      '.py': 'text/x-python',
+      '.java': 'text/x-java-source',
+      '.cpp': 'text/x-c++src',
+      '.c': 'text/x-csrc',
+      '.h': 'text/x-chdr',
+      '.sh': 'application/x-sh',
+      '.bat': 'application/x-bat',
+      '.ps1': 'application/x-powershell',
+
+      // Configuration files
+      '.ini': 'text/plain',
+      '.conf': 'text/plain',
+      '.cfg': 'text/plain',
+      '.env': 'text/plain',
+      '.properties': 'text/plain',
+      '.dockerfile': 'text/plain'
+    };
+
+    return mimeTypes[ext] || 'application/octet-stream';
+  }
+
+  // ============================================================================
+  // FILE INFORMATION AND METADATA
+  // ============================================================================
+
+  async getFileInfo(filePath) {
+    try {
+      const stats = await fsPromises.stat(filePath);
+      const fileName = path.basename(filePath);
+      const fileExt = path.extname(filePath).toLowerCase();
+      const mimeType = this.getMimeType(filePath);
+
+      return {
+        filePath,
+        fileName,
+        fileExt,
+        mimeType,
+        size: stats.size,
+        mtime: stats.mtime,
+        ctime: stats.ctime,
+        atime: stats.atime,
+        isFile: stats.isFile(),
+        isDirectory: stats.isDirectory(),
+        mode: stats.mode,
+        uid: stats.uid,
+        gid: stats.gid,
+        dev: stats.dev,
+        ino: stats.ino,
+        nlink: stats.nlink,
+
+        etag: this.calculateETag(stats),
+        lastModified: stats.mtime.toUTCString(),
+        formattedSize: this.formatBytes(stats.size),
+
+        isLargeFile: stats.size > 100 * 1024 * 1024,
+        isContainerLayer: this.isContainerLayer(filePath),
+        supportedRanges: true
+      };
+    } catch (error) {
+      throw new Error(`Failed to get file info for ${filePath}: ${error.message}`);
+    }
+  }
+
+  isContainerLayer(filePath) {
+    const fileName = path.basename(filePath).toLowerCase();
+    const isDigestPath = /^[a-f0-9]{64}$/.test(fileName);
+    const isLayerExt = ['.layer', '.tar', '.gz', '.tgz'].includes(path.extname(filePath).toLowerCase());
+    const hasLayerInPath = filePath.toLowerCase().includes('layer');
+    const hasDigestInPath = filePath.toLowerCase().includes('sha256');
+
+    return isDigestPath || isLayerExt || hasLayerInPath || hasDigestInPath;
+  }
+
+  // ============================================================================
+  // ETAG AND CACHING UTILITIES
+  // ============================================================================
+
+  calculateETag(stats) {
+    if (!this.options.enableETag) return null;
+    const etag = `"${stats.size.toString(16)}-${stats.mtime.getTime().toString(16)}"`;
+    return etag;
+  }
+
+  generateResponseHeaders(fileInfo, options = {}) {
+    const headers = {
+      'Content-Type': fileInfo.mimeType,
+      'Content-Length': fileInfo.size.toString(),
+      'Accept-Ranges': 'bytes'
+    };
+
+    if (this.options.enableETag && fileInfo.etag) {
+      headers['ETag'] = fileInfo.etag;
+    }
+
+    if (this.options.enableLastModified && fileInfo.lastModified) {
+      headers['Last-Modified'] = fileInfo.lastModified;
+    }
+
+    if (this.options.enableCaching) {
+      headers['Cache-Control'] = `public, max-age=${this.options.maxCacheAge}`;
+      const expires = new Date(Date.now() + this.options.maxCacheAge * 1000);
+      headers['Expires'] = expires.toUTCString();
+    } else {
+      headers['Cache-Control'] = 'no-cache, no-store, must-revalidate';
+      headers['Pragma'] = 'no-cache';
+      headers['Expires'] = '0';
+    }
+
+    if (options.additionalHeaders) {
+      Object.assign(headers, options.additionalHeaders);
+    }
+
+    if (options.forceDownload) {
+      headers['Content-Disposition'] = `attachment; filename="${fileInfo.fileName}"`;
+    } else if (options.inlineDisposition !== false) {
+      headers['Content-Disposition'] = `inline; filename="${fileInfo.fileName}"`;
+    }
+
+    return headers;
+  }
+
+  // ============================================================================
+  // RANGE REQUEST PROCESSING
+  // ============================================================================
+
+  parseRangeHeader(rangeHeader, fileSize) {
+    if (!rangeHeader || !rangeHeader.startsWith('bytes=')) {
+      return null;
+    }
+
+    const ranges = [];
+    const rangeSpec = rangeHeader.replace(/bytes=/, '').split(',');
+
+    for (const spec of rangeSpec) {
+      const range = this.parseRangeSpec(spec.trim(), fileSize);
+      if (range) {
+        ranges.push(range);
+      }
+    }
+
+    return ranges.length > 0 ? ranges : null;
+  }
+
+  parseRangeSpec(spec, fileSize) {
+    const parts = spec.split('-');
+    if (parts.length !== 2) return null;
+
+    let start = parts[0] ? parseInt(parts[0], 10) : null;
+    let end = parts[1] ? parseInt(parts[1], 10) : null;
+
+    if ((parts[0] && isNaN(start)) || (parts[1] && isNaN(end))) {
+      return null;
+    }
+
+    // Suffix range: -500 (last 500 bytes)
+    if (start === null && end !== null) {
+      start = Math.max(0, fileSize - end);
+      end = fileSize - 1;
+    }
+    // Prefix range: 1000- (from byte 1000 to end)
+    else if (start !== null && end === null) {
+      end = fileSize - 1;
+    }
+    // Full range: 1000-2000
+    else if (start !== null && end !== null) {
+      if (start > end || start >= fileSize || end >= fileSize) {
+        return null;
+      }
+    } else {
+      return null;
+    }
+
+    if (start < 0 || end < 0 || start >= fileSize || end >= fileSize || start > end) {
+      return null;
+    }
+
+    return {
+      start,
+      end,
+      length: end - start + 1
+    };
+  }
+
+  formatContentRange(start, end, total) {
+    return `bytes ${start}-${end}/${total}`;
+  }
+
+  validateRanges(ranges, fileSize) {
+    if (!ranges || ranges.length === 0) return false;
+
+    return ranges.every(range => {
+      return range.start >= 0 &&
+        range.end < fileSize &&
+        range.start <= range.end &&
+        range.length > 0;
+    });
+  }
+
+  // ============================================================================
+  // UTILITY FUNCTIONS
+  // ============================================================================
+
+  formatBytes(bytes, decimals = 2) {
+    if (bytes === 0) return '0 B';
+
+    const k = 1024;
+    const dm = decimals < 0 ? 0 : decimals;
+    const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+  }
+
+  formatDuration(ms) {
+    if (ms < 1000) return `${ms}ms`;
+    if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
+    if (ms < 3600000) return `${(ms / 60000).toFixed(1)}m`;
+    return `${(ms / 3600000).toFixed(1)}h`;
+  }
+
+  calculateSpeed(bytes, timeMs) {
+    if (timeMs === 0) return 0;
+    return bytes / (timeMs / 1000);
+  }
+
+  generateOperationId() {
+    return `stream_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  }
+}
+
+// ============================================================================
+// ENHANCED FILE STREAMING WITH PROGRESS TRACKING
+// ============================================================================
+
+class FileStreamer extends EventEmitter {
+  constructor(fileStreamingUtils, options = {}) {
+    super();
+
+    this.utils = fileStreamingUtils;
+    this.options = {
+      bufferSize: options.bufferSize || 64 * 1024,
+      progressInterval: options.progressInterval || 1024 * 1024,
+      timeout: options.timeout || 30000,
+      retryAttempts: options.retryAttempts || 3,
+      retryDelay: options.retryDelay || 1000,
+      ...options
+    };
+
+    this.activeStreams = new Map();
+  }
+
+  async createReadStream(filePath, options = {}) {
+    const operationId = this.utils.generateOperationId();
+    const fileInfo = await this.utils.getFileInfo(filePath);
+
+    let range = null;
+    if (options.range) {
+      const ranges = this.utils.parseRangeHeader(options.range, fileInfo.size);
+      if (ranges && ranges.length > 0) {
+        range = ranges[0];
+      }
+    }
+
+    const start = range ? range.start : 0;
+    const end = range ? range.end : fileInfo.size - 1;
+    const streamLength = end - start + 1;
+
+    const streamOptions = {
+      start,
+      end,
+      highWaterMark: this.options.bufferSize
+    };
+
+    const readStream = fs.createReadStream(filePath, streamOptions);
+
+    const streamInfo = {
+      operationId,
+      filePath,
+      fileInfo,
+      range,
+      streamLength,
+      bytesTransferred: 0,
+      startTime: Date.now(),
+      lastProgressUpdate: 0,
+      speed: 0,
+      estimatedTimeRemaining: 0,
+      completed: false,
+      error: null,
+      stream: readStream
+    };
+
+    this.activeStreams.set(operationId, streamInfo);
+
+    this.setupProgressTracking(streamInfo, options);
+    this.setupErrorHandling(streamInfo, options);
+
+    return {
+      operationId,
+      stream: readStream,
+      fileInfo,
+      range,
+      streamLength,
+      cancel: () => this.cancelStream(operationId),
+      getProgress: () => this.getStreamProgress(operationId)
+    };
+  }
+
+  setupProgressTracking(streamInfo, options) {
+    const { operationId, stream, streamLength, fileInfo } = streamInfo;
+
+    stream.on('data', (chunk) => {
+      streamInfo.bytesTransferred += chunk.length;
+
+      const now = Date.now();
+      const elapsed = now - streamInfo.startTime;
+
+      if (elapsed > 0) {
+        streamInfo.speed = streamInfo.bytesTransferred / (elapsed / 1000);
+      }
+
+      if (streamInfo.speed > 0) {
+        const remaining = streamLength - streamInfo.bytesTransferred;
+        streamInfo.estimatedTimeRemaining = (remaining / streamInfo.speed) * 1000;
+      }
+
+      const shouldEmitProgress = (
+        streamInfo.bytesTransferred - streamInfo.lastProgressUpdate >= this.options.progressInterval ||
+        streamInfo.bytesTransferred === chunk.length ||
+        streamInfo.bytesTransferred >= streamLength
+      );
+
+      if (shouldEmitProgress) {
+        const progress = {
+          operationId,
+          filePath: streamInfo.filePath,
+          fileName: fileInfo.fileName,
+          bytesTransferred: streamInfo.bytesTransferred,
+          totalBytes: streamLength,
+          percentage: Math.round((streamInfo.bytesTransferred / streamLength) * 100),
+          speed: streamInfo.speed,
+          speedFormatted: this.utils.formatBytes(streamInfo.speed) + '/s',
+          estimatedTimeRemaining: streamInfo.estimatedTimeRemaining,
+          elapsed: elapsed,
+          isRangeRequest: !!streamInfo.range,
+          timestamp: new Date()
+        };
+
+        this.emit('progress', progress);
+        streamInfo.lastProgressUpdate = streamInfo.bytesTransferred;
+      }
+    });
+
+    stream.on('end', () => {
+      if (streamInfo.completed) return;
+      streamInfo.completed = true;
+
+      const elapsed = Date.now() - streamInfo.startTime;
+      const finalSpeed = streamInfo.bytesTransferred / (elapsed / 1000);
+
+      const completion = {
+        operationId,
+        filePath: streamInfo.filePath,
+        fileName: fileInfo.fileName,
+        bytesTransferred: streamInfo.bytesTransferred,
+        totalBytes: streamLength,
+        duration: elapsed,
+        speed: finalSpeed,
+        speedFormatted: this.utils.formatBytes(finalSpeed) + '/s',
+        durationFormatted: this.utils.formatDuration(elapsed),
+        isRangeRequest: !!streamInfo.range,
+        success: true,
+        timestamp: new Date()
+      };
+
+      this.emit('complete', completion);
+      this.activeStreams.delete(operationId);
+    });
+  }
+
+  setupErrorHandling(streamInfo, options) {
+    const { operationId, stream } = streamInfo;
+
+    stream.on('error', (error) => {
+      streamInfo.error = error;
+      streamInfo.completed = true;
+
+      const errorInfo = {
+        operationId,
+        filePath: streamInfo.filePath,
+        error: error.message,
+        errorCode: error.code,
+        bytesTransferred: streamInfo.bytesTransferred,
+        totalBytes: streamInfo.streamLength,
+        elapsed: Date.now() - streamInfo.startTime,
+        timestamp: new Date()
+      };
+
+      this.emit('error', errorInfo);
+      this.activeStreams.delete(operationId);
+    });
+
+    if (this.options.timeout > 0) {
+      const timeout = setTimeout(() => {
+        if (!streamInfo.completed) {
+          const timeoutError = new Error(`Stream timeout after ${this.options.timeout}ms`);
+          stream.destroy(timeoutError);
+        }
+      }, this.options.timeout);
+
+      stream.on('end', () => clearTimeout(timeout));
+      stream.on('error', () => clearTimeout(timeout));
+    }
+  }
+
+  cancelStream(operationId) {
+    const streamInfo = this.activeStreams.get(operationId);
+    if (streamInfo && !streamInfo.completed) {
+      streamInfo.stream.destroy();
+      streamInfo.completed = true;
+
+      this.emit('cancelled', {
+        operationId,
+        filePath: streamInfo.filePath,
+        bytesTransferred: streamInfo.bytesTransferred,
+        totalBytes: streamInfo.streamLength,
+        timestamp: new Date()
+      });
+
+      this.activeStreams.delete(operationId);
+    }
+  }
+
+  getStreamProgress(operationId) {
+    const streamInfo = this.activeStreams.get(operationId);
+    if (!streamInfo) return null;
+
+    const elapsed = Date.now() - streamInfo.startTime;
+
+    return {
+      operationId,
+      filePath: streamInfo.filePath,
+      fileName: streamInfo.fileInfo.fileName,
+      bytesTransferred: streamInfo.bytesTransferred,
+      totalBytes: streamInfo.streamLength,
+      percentage: Math.round((streamInfo.bytesTransferred / streamInfo.streamLength) * 100),
+      speed: streamInfo.speed,
+      speedFormatted: this.utils.formatBytes(streamInfo.speed) + '/s',
+      estimatedTimeRemaining: streamInfo.estimatedTimeRemaining,
+      elapsed: elapsed,
+      elapsedFormatted: this.utils.formatDuration(elapsed),
+      isRangeRequest: !!streamInfo.range,
+      completed: streamInfo.completed,
+      timestamp: new Date()
+    };
+  }
+
+  listActiveStreams() {
+    return Array.from(this.activeStreams.keys()).map(id => this.getStreamProgress(id));
+  }
+
+  cancelAllStreams() {
+    const activeIds = Array.from(this.activeStreams.keys());
+    activeIds.forEach(id => this.cancelStream(id));
+  }
+}
+
+// ============================================================================
+// DOWNLOAD PROGRESS TRACKER
+// ============================================================================
+
+class DownloadTracker extends EventEmitter {
+  constructor(options = {}) {
+    super();
+
+    this.options = {
+      trackingInterval: options.trackingInterval || 1000,
+      historySize: options.historySize || 100,
+      enableRealTimeStats: options.enableRealTimeStats !== false,
+      ...options
+    };
+
+    this.downloads = new Map();
+    this.history = [];
+    this.stats = {
+      totalDownloads: 0,
+      totalBytes: 0,
+      totalDuration: 0,
+      averageSpeed: 0,
+      activeDownloads: 0,
+      completedDownloads: 0,
+      failedDownloads: 0
+    };
+
+    if (this.options.enableRealTimeStats) {
+      this.startRealTimeTracking();
+    }
+  }
+
+  startTracking(downloadInfo) {
+    const trackingId = `download_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+
+    const download = {
+      trackingId,
+      startTime: Date.now(),
+      ...downloadInfo,
+      bytesTransferred: 0,
+      speed: 0,
+      percentage: 0,
+      status: 'active',
+      lastUpdate: Date.now()
+    };
+
+    this.downloads.set(trackingId, download);
+    this.stats.activeDownloads++;
+
+    this.emit('downloadStarted', { trackingId, download });
+
+    return trackingId;
+  }
+
+  updateProgress(trackingId, progress) {
+    const download = this.downloads.get(trackingId);
+    if (!download || download.status !== 'active') return;
+
+    const now = Date.now();
+    const elapsed = now - download.startTime;
+
+    Object.assign(download, {
+      bytesTransferred: progress.bytesTransferred || download.bytesTransferred,
+      totalBytes: progress.totalBytes || download.totalBytes,
+      speed: progress.speed || (elapsed > 0 ? download.bytesTransferred / (elapsed / 1000) : 0),
+      percentage: progress.percentage || (download.totalBytes > 0 ?
+        Math.round((download.bytesTransferred / download.totalBytes) * 100) : 0),
+      estimatedTimeRemaining: progress.estimatedTimeRemaining,
+      lastUpdate: now
+    });
+
+    this.emit('downloadProgress', { trackingId, download, progress });
+  }
+
+  completeDownload(trackingId, completionInfo = {}) {
+    const download = this.downloads.get(trackingId);
+    if (!download) return;
+
+    const now = Date.now();
+    const duration = now - download.startTime;
+    const finalSpeed = download.bytesTransferred / (duration / 1000);
+
+    Object.assign(download, {
+      status: 'completed',
+      endTime: now,
+      duration: duration,
+      speed: finalSpeed,
+      percentage: 100,
+      ...completionInfo
+    });
+
+    this.history.unshift(download);
+    if (this.history.length > this.options.historySize) {
+      this.history.pop();
+    }
+
+    this.stats.activeDownloads--;
+    this.stats.completedDownloads++;
+    this.stats.totalDownloads++;
+    this.stats.totalBytes += download.bytesTransferred;
+    this.stats.totalDuration += duration;
+    this.stats.averageSpeed = this.stats.totalBytes / (this.stats.totalDuration / 1000);
+
+    this.downloads.delete(trackingId);
+
+    this.emit('downloadCompleted', { trackingId, download });
+  }
+
+  failDownload(trackingId, errorInfo = {}) {
+    const download = this.downloads.get(trackingId);
+    if (!download) return;
+
+    const now = Date.now();
+    const duration = now - download.startTime;
+
+    Object.assign(download, {
+      status: 'failed',
+      endTime: now,
+      duration: duration,
+      error: errorInfo.error || 'Unknown error',
+      errorCode: errorInfo.errorCode,
+      ...errorInfo
+    });
+
+    this.history.unshift(download);
+    if (this.history.length > this.options.historySize) {
+      this.history.pop();
+    }
+
+    this.stats.activeDownloads--;
+    this.stats.failedDownloads++;
+
+    this.downloads.delete(trackingId);
+
+    this.emit('downloadFailed', { trackingId, download, error: errorInfo });
+  }
+
+  getDownload(trackingId) {
+    return this.downloads.get(trackingId) || null;
+  }
+
+  listActiveDownloads() {
+    return Array.from(this.downloads.values());
+  }
+
+  getHistory(limit = 50) {
+    return this.history.slice(0, limit);
+  }
+
+  getStats() {
+    return { ...this.stats };
+  }
+
+  startRealTimeTracking() {
+    this.trackingInterval = setInterval(() => {
+      const now = Date.now();
+
+      for (const [trackingId, download] of this.downloads.entries()) {
+        if (now - download.lastUpdate > 30000) {
+          this.failDownload(trackingId, {
+            error: 'Download stalled - no progress updates received',
+            errorCode: 'STALLED'
+          });
+        }
+      }
+
+      this.emit('statsUpdate', this.getStats());
+    }, this.options.trackingInterval);
+  }
+
+  stopRealTimeTracking() {
+    if (this.trackingInterval) {
+      clearInterval(this.trackingInterval);
+      this.trackingInterval = null;
+    }
+  }
+
+  shutdown() {
+    this.stopRealTimeTracking();
+    this.downloads.clear();
+    this.emit('shutdown');
+  }
+}
+
+export {
+  FileStreamingUtils,
+  FileStreamer,
+  DownloadTracker
+};