digitaltwin-core 0.14.3 → 1.0.1

package/dist/engine/digital_twin_engine.js

@@ -2,14 +2,18 @@ import express from 'ultimate-express';
 import multer from 'multer';
 import fs from 'fs/promises';
 import cors from 'cors';
+import compression from 'compression';
 import { initializeComponents, initializeAssetsManagers } from './initializer.js';
+import { detectComponentType, isCollector, isHarvester, isHandler, isAssetsManager, isCustomTableManager } from './component_types.js';
 import { UserService } from '../auth/user_service.js';
 import { exposeEndpoints } from './endpoints.js';
 import { scheduleComponents } from './scheduler.js';
 import { LogLevel } from '../utils/logger.js';
 import { QueueManager } from './queue_manager.js';
 import { UploadProcessor } from './upload_processor.js';
+import { engineEventBus } from './events.js';
 import { isAsyncUploadable } from '../components/async_upload.js';
+import { HealthChecker, createDatabaseCheck, createRedisCheck, createStorageCheck, livenessCheck } from './health.js';
 /**
  * Digital Twin Engine - Core orchestrator for collectors, harvesters, and handlers
  *
@@ -52,18 +56,47 @@ export class DigitalTwinEngine {
     /** uWebSockets.js TemplatedApp - has close() method to shut down all connections */
     #server;
     #workers = [];
+    #isShuttingDown = false;
+    #shutdownTimeout = 30000;
+    #healthChecker = new HealthChecker();
+    // Mutable arrays for dynamically registered components
+    #dynamicCollectors = [];
+    #dynamicHarvesters = [];
+    #dynamicHandlers = [];
+    #dynamicAssetsManagers = [];
+    #dynamicCustomTableManagers = [];
+    /** Get all collectors (from constructor + register()) */
+    get #allCollectors() {
+        return [...this.#collectors, ...this.#dynamicCollectors];
+    }
+    /** Get all harvesters (from constructor + register()) */
+    get #allHarvesters() {
+        return [...this.#harvesters, ...this.#dynamicHarvesters];
+    }
+    /** Get all handlers (from constructor + register()) */
+    get #allHandlers() {
+        return [...this.#handlers, ...this.#dynamicHandlers];
+    }
+    /** Get all assets managers (from constructor + register()) */
+    get #allAssetsManagers() {
+        return [...this.#assetsManagers, ...this.#dynamicAssetsManagers];
+    }
+    /** Get all custom table managers (from constructor + register()) */
+    get #allCustomTableManagers() {
+        return [...this.#customTableManagers, ...this.#dynamicCustomTableManagers];
+    }
     /** Get all active components (collectors and harvesters) */
     get #activeComponents() {
-        return [...this.#collectors, ...this.#harvesters];
+        return [...this.#allCollectors, ...this.#allHarvesters];
     }
     /** Get all components (collectors + harvesters + handlers + assetsManagers + customTableManagers) */
     get #allComponents() {
         return [
-            ...this.#collectors,
-            ...this.#harvesters,
-            ...this.#handlers,
-            ...this.#assetsManagers,
-            ...this.#customTableManagers
+            ...this.#allCollectors,
+            ...this.#allHarvesters,
+            ...this.#allHandlers,
+            ...this.#allAssetsManagers,
+            ...this.#allCustomTableManagers
         ];
     }
     /** Check if multi-queue mode is enabled */
@@ -139,6 +172,8 @@ export class DigitalTwinEngine {
     }
     #createQueueManager() {
         // Create queue manager if we have collectors, harvesters, OR assets managers that may need async uploads
+        // Note: At construction time, only constructor-provided components are available
+        // Dynamic components registered via register() will be handled at start() time
         const hasActiveComponents = this.#collectors.length > 0 || this.#harvesters.length > 0;
         const hasAssetsManagers = this.#assetsManagers.length > 0;
         if (!hasActiveComponents && !hasAssetsManagers) {
@@ -181,20 +216,32 @@ export class DigitalTwinEngine {
      * @private
      */
     #setupMonitoringEndpoints() {
-        // Health check endpoint
+        // Register default health checks
+        this.#healthChecker.registerCheck('database', createDatabaseCheck(this.#database));
+        if (this.#queueManager) {
+            this.#healthChecker.registerCheck('redis', createRedisCheck(this.#queueManager));
+        }
+        this.#healthChecker.registerCheck('storage', createStorageCheck(this.#storage));
+        // Set component counts (includes both constructor and dynamically registered components)
+        this.#healthChecker.setComponentCounts({
+            collectors: this.#allCollectors.length,
+            harvesters: this.#allHarvesters.length,
+            handlers: this.#allHandlers.length,
+            assetsManagers: this.#allAssetsManagers.length
+        });
+        // Liveness probe - shallow check, always returns ok if process is running
+        this.#router.get('/api/health/live', (req, res) => {
+            res.status(200).json(livenessCheck());
+        });
+        // Readiness probe - deep check with database and redis verification
+        this.#router.get('/api/health/ready', async (req, res) => {
+            const health = await this.#healthChecker.performCheck();
+            const statusCode = health.status === 'unhealthy' ? 503 : 200;
+            res.status(statusCode).json(health);
+        });
+        // Full health check endpoint (detailed)
         this.#router.get('/api/health', async (req, res) => {
-            const health = {
-                status: 'ok',
-                timestamp: new Date().toISOString(),
-                uptime: process.uptime(),
-                components: {
-                    collectors: this.#collectors.length,
-                    harvesters: this.#harvesters.length,
-                    handlers: this.#handlers.length,
-                    assetsManagers: this.#assetsManagers.length,
-                    customTableManagers: this.#customTableManagers.length
-                }
-            };
+            const health = await this.#healthChecker.performCheck();
             res.json(health);
         });
         // Queue statistics endpoint
@@ -263,8 +310,7 @@ export class DigitalTwinEngine {
         }
         // Inject upload queue to components that support async uploads
         if (this.#queueManager) {
-            const allManagers = [...this.#assetsManagers];
-            for (const manager of allManagers) {
+            for (const manager of this.#allAssetsManagers) {
                 if (isAsyncUploadable(manager)) {
                     manager.setUploadQueue(this.#queueManager.uploadQueue);
                 }
@@ -288,6 +334,24 @@ export class DigitalTwinEngine {
         this.#setupMonitoringEndpoints();
         // Ensure temporary upload directory exists
         await this.#ensureTempUploadDir();
+        // HTTP compression - disabled by default as API gateways (APISIX, Kong, etc.) typically handle this
+        // Enable with DIGITALTWIN_ENABLE_COMPRESSION=true for standalone deployments without a gateway
+        const enableCompression = process.env.DIGITALTWIN_ENABLE_COMPRESSION === 'true';
+        if (enableCompression) {
+            const compressionMiddleware = compression({
+                filter: (req, res) => {
+                    // Don't compress binary streams
+                    if (req.headers['accept']?.includes('application/octet-stream')) {
+                        return false;
+                    }
+                    // Use default filter for other content types
+                    return compression.filter(req, res);
+                },
+                level: 6, // Balance between speed and compression ratio
+                threshold: 1024 // Only compress responses larger than 1KB
+            });
+            this.#app.use(compressionMiddleware);
+        }
         // Enable CORS for cross-origin requests from frontend applications
         this.#app.use(cors({
             origin: process.env.CORS_ORIGIN || true, // Allow all origins by default, configure in production
@@ -361,14 +425,219 @@ export class DigitalTwinEngine {
         return app.port ?? this.#options.server?.port;
     }
     /**
-     * Stops the Digital Twin Engine gracefully
+     * Registers a single component with automatic type detection.
+     *
+     * The engine automatically detects the component type based on its class
+     * and adds it to the appropriate internal collection.
+     *
+     * @param component - Component instance to register
+     * @returns The engine instance for method chaining
+     * @throws Error if component type cannot be determined or is already registered
+     *
+     * @example
+     * ```typescript
+     * const engine = new DigitalTwinEngine({ storage, database })
+     *
+     * engine
+     *   .register(new WeatherCollector())
+     *   .register(new TrafficAnalysisHarvester())
+     *   .register(new ApiHandler())
+     *   .register(new GLTFAssetsManager())
+     *
+     * await engine.start()
+     * ```
+     */
+    register(component) {
+        const type = detectComponentType(component);
+        const config = component.getConfiguration();
+        // Check for duplicate registration
+        if (this.#isComponentRegistered(config.name, type)) {
+            throw new Error(`Component "${config.name}" of type "${type}" is already registered. ` +
+                'Each component must have a unique name within its type.');
+        }
+        switch (type) {
+            case 'collector':
+                if (isCollector(component)) {
+                    this.#dynamicCollectors.push(component);
+                }
+                break;
+            case 'harvester':
+                if (isHarvester(component)) {
+                    this.#dynamicHarvesters.push(component);
+                }
+                break;
+            case 'handler':
+                if (isHandler(component)) {
+                    this.#dynamicHandlers.push(component);
+                }
+                break;
+            case 'assets_manager':
+                if (isAssetsManager(component)) {
+                    this.#dynamicAssetsManagers.push(component);
+                }
+                break;
+            case 'custom_table_manager':
+                if (isCustomTableManager(component)) {
+                    this.#dynamicCustomTableManagers.push(component);
+                }
+                break;
+        }
+        return this;
+    }
+    /**
+     * Registers multiple components at once with automatic type detection.
+     *
+     * Useful for registering all components from a module or when loading
+     * components dynamically.
+     *
+     * @param components - Array of component instances to register
+     * @returns The engine instance for method chaining
+     * @throws Error if any component type cannot be determined or is duplicate
+     *
+     * @example
+     * ```typescript
+     * const engine = new DigitalTwinEngine({ storage, database })
+     *
+     * engine.registerAll([
+     *   new WeatherCollector(),
+     *   new TrafficCollector(),
+     *   new AnalysisHarvester(),
+     *   new ApiHandler()
+     * ])
+     *
+     * await engine.start()
+     * ```
+     */
+    registerAll(components) {
+        for (const component of components) {
+            this.register(component);
+        }
+        return this;
+    }
+    /**
+     * Registers components with explicit type specification.
+     *
+     * Provides full type safety at compile time. Use this method when you
+     * have pre-sorted components from auto-discovery or want explicit control.
+     *
+     * @param components - Object with typed component arrays
+     * @returns The engine instance for method chaining
+     *
+     * @example
+     * ```typescript
+     * const loaded = await loadComponents('./src/components')
+     *
+     * engine.registerComponents({
+     *   collectors: loaded.collectors,
+     *   harvesters: loaded.harvesters,
+     *   handlers: loaded.handlers,
+     *   assetsManagers: loaded.assetsManagers,
+     *   customTableManagers: loaded.customTableManagers
+     * })
+     * ```
+     */
+    registerComponents(components) {
+        if (components.collectors) {
+            this.#dynamicCollectors.push(...components.collectors);
+        }
+        if (components.harvesters) {
+            this.#dynamicHarvesters.push(...components.harvesters);
+        }
+        if (components.handlers) {
+            this.#dynamicHandlers.push(...components.handlers);
+        }
+        if (components.assetsManagers) {
+            this.#dynamicAssetsManagers.push(...components.assetsManagers);
+        }
+        if (components.customTableManagers) {
+            this.#dynamicCustomTableManagers.push(...components.customTableManagers);
+        }
+        return this;
+    }
+    /**
+     * Checks if a component with the given name is already registered.
+     */
+    #isComponentRegistered(name, type) {
+        const allComponents = this.#getAllComponentsOfType(type);
+        return allComponents.some(c => c.getConfiguration().name === name);
+    }
+    /**
+     * Gets all components of a specific type (both from constructor and register()).
+     */
+    #getAllComponentsOfType(type) {
+        switch (type) {
+            case 'collector':
+                return this.#allCollectors;
+            case 'harvester':
+                return this.#allHarvesters;
+            case 'handler':
+                return this.#allHandlers;
+            case 'assets_manager':
+                return this.#allAssetsManagers;
+            case 'custom_table_manager':
+                return this.#allCustomTableManagers;
+        }
+    }
+    /**
+     * Configure the shutdown timeout (in ms)
+     * @param timeout Timeout in milliseconds (default: 30000)
+     */
+    setShutdownTimeout(timeout) {
+        this.#shutdownTimeout = timeout;
+    }
+    /**
+     * Check if the engine is currently shutting down
+     * @returns true if shutdown is in progress
+     */
+    isShuttingDown() {
+        return this.#isShuttingDown;
+    }
+    /**
+     * Register a custom health check
+     * @param name Unique name for the check
+     * @param checkFn Function that performs the check
+     *
+     * @example
+     * ```typescript
+     * engine.registerHealthCheck('external-api', async () => {
+     *     try {
+     *         const res = await fetch('https://api.example.com/health')
+     *         return { status: res.ok ? 'up' : 'down' }
+     *     } catch (error) {
+     *         return { status: 'down', error: error.message }
+     *     }
+     * })
+     * ```
+     */
+    registerHealthCheck(name, checkFn) {
+        this.#healthChecker.registerCheck(name, checkFn);
+    }
+    /**
+     * Remove a custom health check
+     * @param name Name of the check to remove
+     * @returns true if the check was removed, false if it didn't exist
+     */
+    removeHealthCheck(name) {
+        return this.#healthChecker.removeCheck(name);
+    }
+    /**
+     * Get list of registered health check names
+     */
+    getHealthCheckNames() {
+        return this.#healthChecker.getCheckNames();
+    }
+    /**
+     * Stops the Digital Twin Engine with graceful shutdown
      *
      * This method:
-     * 1. Closes HTTP server
-     * 2. Stops background workers
-     * 3. Closes all queue connections
-     * 4. Closes database connections
-     * 5. Clean up resources
+     * 1. Prevents new work from being accepted
+     * 2. Removes all event listeners
+     * 3. Closes HTTP server
+     * 4. Drains queues and waits for active jobs
+     * 5. Closes all queue workers
+     * 6. Stops upload processor
+     * 7. Closes queue manager
+     * 8. Closes database connections
      *
      * @async
      * @returns {Promise<void>}
@@ -380,66 +649,125 @@ export class DigitalTwinEngine {
      * ```
      */
     async stop() {
+        if (this.#isShuttingDown) {
+            if (process.env.NODE_ENV !== 'test') {
+                console.warn('[DigitalTwin] Shutdown already in progress');
+            }
+            return;
+        }
+        this.#isShuttingDown = true;
+        const startTime = Date.now();
+        if (process.env.NODE_ENV !== 'test') {
+            console.log('[DigitalTwin] Graceful shutdown initiated...');
+        }
         const errors = [];
-        // 1. Close HTTP server first (uWebSockets.js TemplatedApp.close() is synchronous)
+        // 1. Remove all event listeners to prevent new work
+        this.#cleanupEventListeners();
+        // 2. Close HTTP server (uWebSockets.js TemplatedApp.close() is synchronous)
         if (this.#server) {
             try {
                 this.#server.close();
             }
             catch (error) {
-                errors.push(new Error(`Server close error: ${error instanceof Error ? error.message : String(error)}`));
+                errors.push(this.#wrapError('Server close', error));
             }
             this.#server = undefined;
         }
-        // 2. Close all workers with extended timeout and force close
-        if (this.#workers.length > 0) {
-            await Promise.all(this.#workers.map(async (worker) => {
-                try {
-                    await Promise.race([
-                        worker.close(),
-                        new Promise((_, reject) => setTimeout(() => reject(new Error('Worker close timeout')), 5000))
-                    ]);
-                }
-                catch {
-                    // Force close if timeout or error
-                    try {
-                        await worker.disconnect();
-                    }
-                    catch (disconnectError) {
-                        errors.push(new Error(`Worker force close error: ${disconnectError instanceof Error ? disconnectError.message : String(disconnectError)}`));
-                    }
-                }
-            }));
+        // 3. Drain queues - wait for active jobs with timeout
+        if (this.#queueManager) {
+            try {
+                await this.#drainQueues();
+            }
+            catch (error) {
+                errors.push(this.#wrapError('Queue drain', error));
+            }
         }
-        // 3. Stop upload processor worker
+        // 4. Close all workers with extended timeout and force close
+        await this.#closeWorkers(errors);
+        // 5. Stop upload processor worker
         if (this.#uploadProcessor) {
             try {
                 await this.#uploadProcessor.stop();
             }
             catch (error) {
-                errors.push(new Error(`Upload processor close error: ${error instanceof Error ? error.message : String(error)}`));
+                errors.push(this.#wrapError('Upload processor', error));
             }
         }
-        // 4. Close queue connections (only if we have a queue manager)
+        // 6. Close queue connections (only if we have a queue manager)
         if (this.#queueManager) {
             try {
                 await this.#queueManager.close();
             }
             catch (error) {
-                errors.push(new Error(`Queue manager close error: ${error instanceof Error ? error.message : String(error)}`));
+                errors.push(this.#wrapError('Queue manager', error));
             }
         }
-        // 5. Close database connections
+        // 7. Close database connections
         try {
             await this.#database.close();
         }
         catch (error) {
-            errors.push(new Error(`Database close error: ${error instanceof Error ? error.message : String(error)}`));
+            errors.push(this.#wrapError('Database', error));
         }
-        if (errors.length > 0 && process.env.NODE_ENV !== 'test') {
-            console.warn('[DigitalTwin] Stopped with warnings:', errors.map(e => e.message).join(', '));
+        const duration = Date.now() - startTime;
+        if (process.env.NODE_ENV !== 'test') {
+            if (errors.length > 0) {
+                console.error(`[DigitalTwin] Shutdown completed with ${errors.length} errors in ${duration}ms:`, errors.map(e => e.message).join(', '));
+            }
+            else {
+                console.log(`[DigitalTwin] Shutdown completed successfully in ${duration}ms`);
+            }
+        }
+    }
+    #cleanupEventListeners() {
+        engineEventBus.removeAllListeners();
+    }
+    async #drainQueues() {
+        if (!this.#queueManager)
+            return;
+        const timeout = Math.min(this.#shutdownTimeout / 2, 15000);
+        const startTime = Date.now();
+        while (Date.now() - startTime < timeout) {
+            try {
+                const stats = await this.#queueManager.getQueueStats();
+                const totalActive = Object.values(stats).reduce((sum, q) => sum + (q.active || 0), 0);
+                if (totalActive === 0)
+                    break;
+                if (process.env.NODE_ENV !== 'test') {
+                    console.log(`[DigitalTwin] Waiting for ${totalActive} active jobs...`);
+                }
+                await new Promise(resolve => setTimeout(resolve, 1000));
+            }
+            catch {
+                await new Promise(resolve => setTimeout(resolve, 1000));
+                break;
+            }
         }
     }
+    async #closeWorkers(errors) {
+        const workerTimeout = Math.min(this.#shutdownTimeout / 3, 10000);
+        await Promise.all(this.#workers.map(async (worker) => {
+            try {
+                await Promise.race([
+                    worker.close(),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('Worker close timeout')), workerTimeout))
+                ]);
+            }
+            catch {
+                try {
+                    await worker.disconnect();
+                }
+                catch (disconnectError) {
+                    errors.push(this.#wrapError('Worker disconnect', disconnectError));
+                }
+            }
+        }));
+        this.#workers = [];
+    }
+    #wrapError(context, error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return new Error(`${context}: ${message}`);
+    }
     /**
      * Validate the engine configuration and all components
      *
@@ -459,24 +787,24 @@ export class DigitalTwinEngine {
     async validateConfiguration() {
         const componentResults = [];
         const engineErrors = [];
-        // Validate collectors
-        for (const collector of this.#collectors) {
+        // Validate collectors (includes dynamically registered)
+        for (const collector of this.#allCollectors) {
             componentResults.push(await this.#validateComponent(collector, 'collector'));
         }
-        // Validate harvesters
-        for (const harvester of this.#harvesters) {
+        // Validate harvesters (includes dynamically registered)
+        for (const harvester of this.#allHarvesters) {
             componentResults.push(await this.#validateComponent(harvester, 'harvester'));
         }
-        // Validate handlers
-        for (const handler of this.#handlers) {
+        // Validate handlers (includes dynamically registered)
+        for (const handler of this.#allHandlers) {
             componentResults.push(await this.#validateComponent(handler, 'handler'));
         }
-        // Validate assets managers
-        for (const assetsManager of this.#assetsManagers) {
+        // Validate assets managers (includes dynamically registered)
+        for (const assetsManager of this.#allAssetsManagers) {
             componentResults.push(await this.#validateComponent(assetsManager, 'assets_manager'));
         }
-        // Validate store managers
-        for (const customTableManager of this.#customTableManagers) {
+        // Validate custom table managers (includes dynamically registered)
+        for (const customTableManager of this.#allCustomTableManagers) {
             componentResults.push(await this.#validateComponent(customTableManager, 'custom_table_manager'));
         }
         // Validate engine-level configuration
@@ -536,23 +864,23 @@ export class DigitalTwinEngine {
      */
     async testComponents() {
         const results = [];
-        // Test collectors
-        for (const collector of this.#collectors) {
+        // Test collectors (includes dynamically registered)
+        for (const collector of this.#allCollectors) {
             const result = await this.#testCollector(collector);
             results.push(result);
         }
-        // Test harvesters
-        for (const harvester of this.#harvesters) {
+        // Test harvesters (includes dynamically registered)
+        for (const harvester of this.#allHarvesters) {
             const result = await this.#testHarvester(harvester);
             results.push(result);
         }
-        // Test handlers
-        for (const handler of this.#handlers) {
+        // Test handlers (includes dynamically registered)
+        for (const handler of this.#allHandlers) {
             const result = await this.#testHandler(handler);
             results.push(result);
         }
-        // Test assets managers
-        for (const assetsManager of this.#assetsManagers) {
+        // Test assets managers (includes dynamically registered)
+        for (const assetsManager of this.#allAssetsManagers) {
             const result = await this.#testAssetsManager(assetsManager);
             results.push(result);
         }