mes-engine 0.0.1

package/dist/types/cache/internalCache.d.ts

@@ -0,0 +1,13 @@
+import { CacheStrategy, CacheOptions } from './cacheStrategy';
+import { StorageProvider } from '../storage/StorageProvider';
+export declare class InternalCache extends CacheStrategy {
+    private cache;
+    private options;
+    private storage;
+    constructor(options: CacheOptions, storage: StorageProvider);
+    set(key: string, value: Buffer): Promise<void>;
+    get(key: string): Promise<Buffer | null>;
+    preload(key: string): Promise<void>;
+    clear(): Promise<void>;
+    private getNextChunkKey;
+}

package/dist/types/core/VideoEngine.d.ts

@@ -0,0 +1,6 @@
+import { EventEmitter } from 'events';
+import { QualityLevel } from './types';
+export declare abstract class VideoEngine extends EventEmitter {
+    abstract processChunk(inputPath: string, outputPath: string, startTime: number, quality: QualityLevel): Promise<void>;
+    abstract getDuration(inputPath: string): Promise<number>;
+}

package/dist/types/core/events.d.ts

@@ -0,0 +1,6 @@
+export declare enum VideoEvent {
+    CHUNK_PROCESSED = "chunkProcessed",
+    QUALITY_PROCESSED = "qualityProcessed",
+    PROCESSING_COMPLETE = "processingComplete",
+    ERROR = "error"
+}

package/dist/types/core/types.d.ts

@@ -0,0 +1,20 @@
+export interface VideoConfig {
+    chunkSize: number;
+    cacheDir: string;
+    maxCacheSize: number;
+    defaultQualities: QualityLevel[];
+}
+export interface QualityLevel {
+    height: number;
+    bitrate: string;
+}
+export interface VideoChunk {
+    quality: number;
+    number: number;
+    path: string;
+}
+export interface VideoManifest {
+    videoId: string;
+    qualities: QualityLevel[];
+    chunks: VideoChunk[];
+}

package/dist/types/engines/FFmpegEngine.d.ts

@@ -0,0 +1,6 @@
+import { VideoEngine } from '../core/VideoEngine';
+import { QualityLevel } from '../core/types';
+export declare class FFmpegEngine extends VideoEngine {
+    processChunk(inputPath: string, outputPath: string, startTime: number, quality: QualityLevel): Promise<void>;
+    getDuration(inputPath: string): Promise<number>;
+}

package/dist/types/index.d.ts

@@ -0,0 +1,10 @@
+export * from './core/types';
+export * from './core/events';
+export * from './core/VideoEngine';
+export * from './engines/FFmpegEngine';
+export * from './storage/StorageProvider';
+export * from './storage/FileSystemStorage';
+export * from './cache/cacheStrategy';
+export * from './cache/internalCache';
+export * from './cache/ExternalCache';
+export * from './processor';

package/dist/types/processor.d.ts

@@ -0,0 +1,19 @@
+import { VideoEngine } from './core/VideoEngine';
+import { EventEmitter } from 'events';
+import { StorageProvider } from './storage/StorageProvider';
+import { VideoConfig, VideoManifest } from './core/types';
+import { Readable } from 'stream';
+export declare class VideoProcessor extends EventEmitter {
+    private engine;
+    private storage;
+    private streamManager;
+    private config;
+    constructor(engine: VideoEngine, storage: StorageProvider, config: VideoConfig);
+    processVideo(inputPath: string): Promise<VideoManifest>;
+    streamChunk(videoId: string, quality: number, chunkNumber: number, range?: {
+        start: number;
+        end: number;
+    }): Promise<Readable>;
+    private getChunkPath;
+    private generateVideoId;
+}

package/dist/types/storage/FileSystemStorage.d.ts

@@ -0,0 +1,6 @@
+import { StorageProvider } from './StorageProvider';
+export declare class FileSystemStorage extends StorageProvider {
+    saveChunk(chunkPath: string, data: Buffer): Promise<void>;
+    getChunk(chunkPath: string): Promise<Buffer>;
+    deleteChunk(chunkPath: string): Promise<void>;
+}

package/dist/types/storage/StorageProvider.d.ts

@@ -0,0 +1,5 @@
+export declare abstract class StorageProvider {
+    abstract saveChunk(chunkPath: string, data: Buffer): Promise<void>;
+    abstract getChunk(chunkPath: string): Promise<Buffer>;
+    abstract deleteChunk(chunkPath: string): Promise<void>;
+}

package/dist/types/streaming/StreamManager.d.ts

@@ -0,0 +1,10 @@
+import { Readable } from 'stream';
+import { StorageProvider } from '../storage/StorageProvider';
+export declare class StreamManager {
+    private storage;
+    constructor(storage: StorageProvider);
+    createStream(chunkPath: string, range?: {
+        start: number;
+        end: number;
+    }): Promise<Readable>;
+}

package/docs/README.md

@@ -0,0 +1,170 @@
+# Video Processing Framework Documentation
+
+Comprehensive documentation for the mes-engine video processing framework.
+
+## Table of Contents
+
+1. [Getting Started](#getting-started)
+2. [Core Concepts](#core-concepts)
+3. [Components](#components)
+4. [Advanced Usage](#advanced-usage)
+5. [API Reference](#api-reference)
+
+## Getting Started
+
+### Installation
+```bash
+npm install mes-engine
+```
+
+### Engine Requirements
+
+#### FFmpeg Engine
+```bash
+npm install ffmpeg-static
+```
+
+#### GStreamer Engine
+Requires GStreamer to be installed on your system:
+- Ubuntu/Debian: `apt-get install gstreamer1.0-tools`
+- MacOS: `brew install gstreamer`
+- Windows: Download from GStreamer website
+
+### Basic Configuration
+
+```typescript
+import { VideoProcessor, FFmpegEngine, FileSystemStorage } from 'mes-engine';
+
+const config = {
+  chunkSize: 10,          // seconds
+  cacheDir: './cache',
+  maxCacheSize: 1000,     // MB
+  defaultQualities: [
+    { height: 1080, bitrate: '4000k' },
+    { height: 720, bitrate: '2500k' },
+    { height: 480, bitrate: '1000k' }
+  ]
+};
+
+const processor = new VideoProcessor({
+  engine: new FFmpegEngine(),
+  storage: new FileSystemStorage(),
+  config
+});
+```
+
+## Core Concepts
+
+### Video Processing Flow
+
+1. **Input**: Raw video file
+2. **Chunking**: Video split into segments
+3. **Transcoding**: Multiple quality versions
+4. **Storage**: Chunks saved to storage provider
+5. **Streaming**: Adaptive delivery
+
+### Events System
+
+```typescript
+processor.on(VideoEvent.CHUNK_PROCESSED, (data) => {
+  console.log(`Chunk ${data.chunkNumber} processed at ${data.quality}p`);
+});
+
+processor.on(VideoEvent.PROCESSING_COMPLETE, (manifest) => {
+  console.log('Processing complete:', manifest);
+});
+```
+
+## Components
+
+### Processing Engines
+
+- **FFmpegEngine**: Standard video processing
+- **GStreamerEngine**: High-performance alternative
+- **Custom Engines**: Extend `VideoEngine` class
+
+```typescript
+class CustomEngine extends VideoEngine {
+  async processChunk(
+    inputPath: string,
+    outputPath: string,
+    startTime: number,
+    quality: QualityLevel
+  ): Promise<void> {
+    // Implementation
+  }
+
+  async getDuration(inputPath: string): Promise<number> {
+    // Implementation
+  }
+}
+```
+
+### Storage Providers
+
+- **FileSystemStorage**: Local file system storage
+- **Custom Storage**: Implement `StorageProvider`
+
+```typescript
+class CustomStorage extends StorageProvider {
+  async saveChunk(chunkPath: string, data: Buffer): Promise<void> {
+    // Implementation
+  }
+
+  async getChunk(chunkPath: string): Promise<Buffer> {
+    // Implementation
+  }
+
+  async deleteChunk(chunkPath: string): Promise<void> {
+    // Implementation
+  }
+}
+```
+
+### Caching Strategies
+
+- **InternalCache**: Memory-based LRU cache
+- **ExternalCache**: Remote cache service
+- **Custom Cache**: Extend `CacheStrategy`
+
+## Advanced Usage
+
+### Custom Quality Levels
+
+```typescript
+const customQualities = [
+  { height: 2160, bitrate: '8000k' }, // 4K
+  { height: 1440, bitrate: '6000k' }, // 2K
+  { height: 1080, bitrate: '4000k' }, // Full HD
+];
+
+const manifest = await processor.processVideo('input.mp4', {
+  qualities: customQualities
+});
+```
+
+### Bandwidth-Aware Streaming
+
+```typescript
+import { BandwidthDetector } from 'mes-engine';
+
+const detector = new BandwidthDetector();
+detector.addSample(bytesTransferred, durationMs);
+const estimatedBandwidth = detector.getEstimatedBandwidth();
+```
+
+## API Reference
+
+See [API.md](./API.md) for detailed API documentation.
+
+## Testing
+
+```bash
+npm run test        # Run all tests
+npm run test:watch  # Watch mode
+npm run test:coverage # Coverage report
+```
+
+## Contributing
+
+See [CONTRIBUTING.md](../CONTRIBUTING.md) for contribution guidelines.

package/docs/engines.md

@@ -0,0 +1,58 @@
+# Engines Support
+
+## FFmpegEngine
+The `FFmpegEngine` uses `ffmpeg` for video processing. You will need to install an FFmpeg binary to use this engine.
+
+### Installation of FFmpeg
+To use the `FFmpegEngine`, you must install FFmpeg or use `ffmpeg-static` to include the FFmpeg binary. You can install `ffmpeg-static` by running:
+
+```bash
+npm install ffmpeg-static
+```
+
+Alternatively, you can use any other FFmpeg binary that is compatible with your system.
+
+### Example Usage:
+
+```typescript
+import { FFmpegEngine } from 'mes-engine';
+
+// Create an FFmpeg engine instance
+const engine = new FFmpegEngine();
+
+// Use it to process video chunks
+engine.processChunk('input.mp4', 'output.mp4', 0, { height: 720, bitrate: '2000k' })
+  .then(() => {
+    console.log('Chunk processed successfully');
+  })
+  .catch(error => {
+    console.error('Error processing chunk:', error);
+  });
+```
+
+## Custom Engines
+You can implement your own video processing engine by extending the `VideoEngine` class. Here’s an example of how to create a custom engine:
+
+### Example Custom Engine:
+
+```typescript
+import { VideoEngine } from 'mes-engine';
+
+class CustomVideoEngine extends VideoEngine {
+  async processChunk(inputPath: string, outputPath: string, startTime: number, quality: any): Promise<void> {
+    // Implement custom video processing logic here
+  }
+
+  async getDuration(inputPath: string): Promise<number> {
+    // Implement custom duration calculation logic here
+    return 120; // Example
+  }
+}
+```
+
+For more details on implementing custom engines, see the [VideoEngine class](../src/core/VideoEngine.ts) in the codebase.
+
+## Other Supported Engines
+You can extend this framework with any other video processing engine that implements the `VideoEngine` interface.
+
+For more details, check the [source code](https://github.com/Bum-Ho12/mes-engine) and adapt it as needed for your requirements.

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "mes-engine",
+  "version": "0.0.1",
+  "description": "mes-engine is a video processing engine that handles video chunking, processing and other video high density and intensive processing and caching processes.",
+  "keywords": [
+    "mes",
+    "mes-engine",
+    "video",
+    "processing",
+    "video-framework",
+    "framework"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+"
+  },
+  "license": "MIT",
+  "author": "Bumho Nisubire",
+  "main": "dist/index.js",
+  "scripts": {
+    "test": "mocha -r ts-node/register 'tests/**/*.test.ts' --timeout 10000",
+    "build": "rollup -c rollup.config.js"
+  },
+  "dependencies": {
+    "@types/node-fetch": "^2.6.12",
+    "mes-engine": "file:",
+    "node-fetch": "^3.3.2",
+    "tslib": "^2.8.1"
+  },
+  "devDependencies": {
+    "@rollup/plugin-commonjs": "^28.0.2",
+    "@rollup/plugin-node-resolve": "^16.0.0",
+    "@rollup/plugin-typescript": "^12.1.2",
+    "@types/chai": "^4.3.11",
+    "@types/jest": "^29.5.14",
+    "@types/mocha": "^10.0.6",
+    "@types/node": "^22.10.5",
+    "chai": "^4.3.10",
+    "jest": "^29.7.0",
+    "mocha": "^10.8.2",
+    "rollup": "^4.30.1",
+    "rollup-plugin-peer-deps-external": "^2.2.4",
+    "ts-jest": "^29.2.5",
+    "ts-mocha": "^10.0.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3"
+  }
+}

package/rollup.config.js

@@ -0,0 +1,24 @@
+import typescript from '@rollup/plugin-typescript';
+import { nodeResolve } from '@rollup/plugin-node-resolve';
+import commonjs from '@rollup/plugin-commonjs';
+import peerDepsExternal from 'rollup-plugin-peer-deps-external';
+
+export default {
+    input: 'src/index.ts', // Entry point
+    output: [
+        {
+            file: 'dist/index.js', // Output file
+            format: 'es', // ES module format
+            sourcemap: true, // Enable source maps
+        },
+    ],
+    plugins: [
+        peerDepsExternal(), // Externalize peer dependencies
+        nodeResolve(), // Resolve dependencies from node_modules
+        commonjs(), // Convert CommonJS to ES6
+        typescript({ tsconfig: './tsconfig.json' }), // TypeScript plugin
+    ],
+    external: [
+        'node-fetch', // Prevent bundling node-fetch
+    ],
+};

package/src/bandwidth.ts

@@ -0,0 +1,30 @@
+
+// bandwidth.ts
+class BandwidthDetector {
+    private samples: Array<[number, number]> = [];
+    private readonly windowSize = 10000; // 10 seconds in ms
+
+    addSample(bytesTransferred: number, durationMs: number): void {
+      const bandwidth = (bytesTransferred * 8) / (durationMs * 1000); // Mbps
+        const timestamp = Date.now();
+        this.samples.push([timestamp, bandwidth]);
+        this._cleanOldSamples();
+    }
+
+    getEstimatedBandwidth(): number {
+        if (this.samples.length === 0) return Infinity;
+        const bandwidths = this.samples.map(([, b]) => b);
+        return this._median(bandwidths);
+    }
+
+    private _cleanOldSamples(): void {
+        const cutoff = Date.now() - this.windowSize;
+        this.samples = this.samples.filter(([t]) => t > cutoff);
+    }
+
+    private _median(values: number[]): number {
+        const sorted = [...values].sort((a, b) => a - b);
+        const mid = Math.floor(sorted.length / 2);
+        return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+    }
+}

package/src/cache/ExternalCache.ts

@@ -0,0 +1,49 @@
+
+// cache/ExternalCache.ts
+import fetch from 'node-fetch';
+import { CacheStrategy, CacheOptions } from './cacheStrategy';
+
+export class ExternalCache extends CacheStrategy {
+    private baseUrl: string;
+    private options: CacheOptions;
+
+    constructor(options: CacheOptions) {
+        super();
+        this.options = options;
+        this.baseUrl = options.externalCacheUrl!;
+    }
+
+    async set(key: string, value: Buffer): Promise<void> {
+        await fetch(`${this.baseUrl}/cache/${key}`, {
+        method: 'POST',
+        body: value,
+        headers: {
+            'Content-Type': 'application/octet-stream',
+            'TTL': this.options.ttl.toString()
+        }
+        });
+    }
+
+    async get(key: string): Promise<Buffer | null> {
+        const response = await fetch(`${this.baseUrl}/cache/${key}`);
+        return response.ok ? Buffer.from(await response.arrayBuffer()) : null;
+    }
+
+    async preload(key: string): Promise<void> {
+        if (this.options.preloadNextChunk) {
+        const nextChunkKey = this.getNextChunkKey(key);
+        await fetch(`${this.baseUrl}/preload/${nextChunkKey}`);
+        }
+    }
+
+    async clear(): Promise<void> {
+        await fetch(`${this.baseUrl}/cache`, { method: 'DELETE' });
+    }
+
+    private getNextChunkKey(currentKey: string): string {
+        const parts = currentKey.split('_');
+        const currentChunk = parseInt(parts[parts.length - 1]);
+        parts[parts.length - 1] = (currentChunk + 1).toString();
+        return parts.join('_');
+    }
+}

package/src/cache/LRU.ts

@@ -0,0 +1,35 @@
+
+// cache/LRU.ts
+export class LRU<T> {
+    private maxSize: number;
+    private cache: Map<string, T>;
+
+    constructor(maxSize: number) {
+        this.maxSize = maxSize;
+        this.cache = new Map();
+    }
+
+    set(key: string, value: T): void {
+        if (this.cache.has(key)) {
+            this.cache.delete(key);
+        } else if (this.cache.size >= this.maxSize) {
+            const oldestKey = this.cache.keys().next().value;
+            if (oldestKey !== undefined) {
+                this.cache.delete(oldestKey);
+            }
+        }
+        this.cache.set(key, value);
+    }
+
+    get(key: string): T | undefined {
+        if (!this.cache.has(key)) return undefined;
+        const value = this.cache.get(key)!;
+        this.cache.delete(key);
+        this.cache.set(key, value);
+        return value;
+    }
+
+    clear(): void {
+        this.cache.clear();
+    }
+}

package/src/cache/cacheStrategy.ts

@@ -0,0 +1,15 @@
+
+// cache/CacheStrategy.ts
+export interface CacheOptions {
+    maxSize: number;
+    ttl: number;
+    preloadNextChunk: boolean;
+    externalCacheUrl?: string;
+}
+
+export abstract class CacheStrategy {
+    abstract set(key: string, value: Buffer): Promise<void>;
+    abstract get(key: string): Promise<Buffer | null>;
+    abstract preload(key: string): Promise<void>;
+    abstract clear(): Promise<void>;
+}

package/src/cache/internalCache.ts

@@ -0,0 +1,60 @@
+
+// cache/internalCache.ts
+import { LRU } from './LRU';
+import { CacheStrategy, CacheOptions } from './cacheStrategy';
+import { StorageProvider } from '../storage/StorageProvider';
+
+export class InternalCache extends CacheStrategy {
+    private cache: LRU<Buffer>;
+    private options: CacheOptions;
+    private storage: StorageProvider;
+
+    constructor(options: CacheOptions, storage: StorageProvider) {
+        super();
+        this.options = options;
+        this.cache = new LRU(options.maxSize);
+        this.storage = storage;
+    }
+
+    async set(key: string, value: Buffer): Promise<void> {
+        this.cache.set(key, value);
+    }
+
+    async get(key: string): Promise<Buffer | null> {
+        const cached = this.cache.get(key);
+        if (cached) return cached;
+
+        try {
+        const data = await this.storage.getChunk(key);
+        await this.set(key, data);
+        return data;
+        } catch {
+        return null;
+        }
+    }
+
+    async preload(key: string): Promise<void> {
+        if (this.options.preloadNextChunk) {
+        const nextChunkKey = this.getNextChunkKey(key);
+        if (!this.cache.get(nextChunkKey)) {
+            try {
+            const data = await this.storage.getChunk(nextChunkKey);
+            await this.set(nextChunkKey, data);
+            } catch {
+            // Ignore preload failures
+            }
+        }
+        }
+    }
+
+    async clear(): Promise<void> {
+        this.cache.clear();
+    }
+
+    private getNextChunkKey(currentKey: string): string {
+        const parts = currentKey.split('_');
+        const currentChunk = parseInt(parts[parts.length - 1]);
+        parts[parts.length - 1] = (currentChunk + 1).toString();
+        return parts.join('_');
+    }
+}

package/src/core/VideoEngine.ts

@@ -0,0 +1,16 @@
+
+// core/VideoEngine.ts
+import { EventEmitter } from 'events';
+import { VideoConfig, QualityLevel, VideoManifest } from './types';
+import { VideoEvent } from './events';
+
+export abstract class VideoEngine extends EventEmitter {
+    abstract processChunk(
+        inputPath: string,
+        outputPath: string,
+        startTime: number,
+        quality: QualityLevel
+    ): Promise<void>;
+
+    abstract getDuration(inputPath: string): Promise<number>;
+}

package/src/core/events.ts

@@ -0,0 +1,7 @@
+
+export enum VideoEvent {
+    CHUNK_PROCESSED = 'chunkProcessed',
+    QUALITY_PROCESSED = 'qualityProcessed',
+    PROCESSING_COMPLETE = 'processingComplete',
+    ERROR = 'error'
+}

package/src/core/types.ts

@@ -0,0 +1,26 @@
+
+// core/types.ts
+
+export interface VideoConfig {
+    chunkSize: number;
+    cacheDir: string;
+    maxCacheSize: number;
+    defaultQualities: QualityLevel[];
+}
+
+export interface QualityLevel {
+    height: number;
+    bitrate: string;
+}
+
+export interface VideoChunk {
+    quality: number;
+    number: number;
+    path: string;
+}
+
+export interface VideoManifest {
+    videoId: string;
+    qualities: QualityLevel[];
+    chunks: VideoChunk[];
+}