@cepseudo/engine 1.0.0

package/dist/digital_twin_engine.js

@@ -0,0 +1,1200 @@
+import express from 'ultimate-express';
+import multer from 'multer';
+import fs from 'fs/promises';
+import cors from 'cors';
+import compression from 'compression';
+import { initializeComponents, initializeAssetsManagers } from './initializer.js';
+import { detectComponentType, isCollector, isHarvester, isHandler, isAssetsManager, isCustomTableManager } from './component_types.js';
+import { UserService, AuthMiddleware } from '@cepseudo/auth';
+import { exposeEndpoints } from './endpoints.js';
+import { scheduleComponents } from './scheduler.js';
+import { LogLevel, engineEventBus } from '@cepseudo/shared';
+import { QueueManager } from './queue_manager.js';
+import { UploadProcessor, UploadReconciler } from '@cepseudo/assets';
+import { isAsyncUploadable } from '@cepseudo/assets';
+import { HealthChecker, createDatabaseCheck, createRedisCheck, createStorageCheck, livenessCheck } from './health.js';
+/**
+ * Digital Twin Engine - Core orchestrator for collectors, harvesters, and handlers
+ *
+ * The engine manages the lifecycle of all components, sets up queues for processing,
+ * exposes HTTP endpoints, and handles the overall coordination of the digital twin system.
+ *
+ * @class DigitalTwinEngine
+ * @example
+ * ```TypeScript
+ * import { DigitalTwinEngine } from './digital_twin_engine.js'
+ * import { StorageServiceFactory } from '../storage/storage_factory.js'
+ * import { KnexDatabaseAdapter } from '../database/adapters/knex_database_adapter.js'
+ *
+ * const storage = StorageServiceFactory.create()
+ * const database = new KnexDatabaseAdapter({ client: 'sqlite3', connection: ':memory:' }, storage)
+ *
+ * const engine = new DigitalTwinEngine({
+ *   storage,
+ *   database,
+ *   collectors: [myCollector],
+ *   server: { port: 3000 }
+ * })
+ *
+ * await engine.start()
+ * ```
+ */
+export class DigitalTwinEngine {
+    #collectors;
+    #harvesters;
+    #handlers;
+    #assetsManagers;
+    #customTableManagers;
+    #storage;
+    #database;
+    #app;
+    #router;
+    #options;
+    #queueManager;
+    #uploadProcessor;
+    #uploadReconciler;
+    /** uWebSockets.js TemplatedApp - has close() method to shut down all connections */
+    #server;
+    #workers = [];
+    #isShuttingDown = false;
+    #isStarted = false;
+    #shutdownTimeout = 30000;
+    #healthChecker = new HealthChecker();
+    // Mutable arrays for dynamically registered components
+    #dynamicCollectors = [];
+    #dynamicHarvesters = [];
+    #dynamicHandlers = [];
+    #dynamicAssetsManagers = [];
+    #dynamicCustomTableManagers = [];
+    /** Get all collectors (from constructor + register()) */
+    get #allCollectors() {
+        return [...this.#collectors, ...this.#dynamicCollectors];
+    }
+    /** Get all harvesters (from constructor + register()) */
+    get #allHarvesters() {
+        return [...this.#harvesters, ...this.#dynamicHarvesters];
+    }
+    /** Get all handlers (from constructor + register()) */
+    get #allHandlers() {
+        return [...this.#handlers, ...this.#dynamicHandlers];
+    }
+    /** Get all assets managers (from constructor + register()) */
+    get #allAssetsManagers() {
+        return [...this.#assetsManagers, ...this.#dynamicAssetsManagers];
+    }
+    /** Get all custom table managers (from constructor + register()) */
+    get #allCustomTableManagers() {
+        return [...this.#customTableManagers, ...this.#dynamicCustomTableManagers];
+    }
+    /** Get all active components (collectors and harvesters) */
+    get #activeComponents() {
+        return [...this.#allCollectors, ...this.#allHarvesters];
+    }
+    /** Get all components (collectors + harvesters + handlers + assetsManagers + customTableManagers) */
+    get #allComponents() {
+        return [
+            ...this.#allCollectors,
+            ...this.#allHarvesters,
+            ...this.#allHandlers,
+            ...this.#allAssetsManagers,
+            ...this.#allCustomTableManagers
+        ];
+    }
+    /** Check if multi-queue mode is enabled */
+    get #isMultiQueueEnabled() {
+        return this.#options.queues?.multiQueue ?? true;
+    }
+    /**
+     * Creates a new Digital Twin Engine instance
+     *
+     * @param {EngineOptions} options - Configuration options for the engine
+     * @throws {Error} If required options (storage, database) are missing
+     *
+     * @example
+     * ```TypeScript
+     * const engine = new DigitalTwinEngine({
+     *   storage: myStorageService,
+     *   database: myDatabaseAdapter,
+     *   collectors: [collector1, collector2],
+     *   server: { port: 4000, host: 'localhost' }
+     * })
+     * ```
+     */
+    constructor(options) {
+        this.#options = this.#applyDefaults(options);
+        this.#collectors = this.#options.collectors ?? [];
+        this.#harvesters = this.#options.harvesters ?? [];
+        this.#handlers = this.#options.handlers ?? [];
+        this.#assetsManagers = this.#options.assetsManagers ?? [];
+        this.#customTableManagers = this.#options.customTableManagers ?? [];
+        this.#storage = this.#options.storage;
+        this.#database = this.#options.database;
+        this.#app = express();
+        this.#router = express.Router();
+        this.#queueManager = this.#createQueueManager();
+        this.#uploadProcessor = this.#createUploadProcessor();
+        this.#uploadReconciler = new UploadReconciler(this.#database, this.#storage);
+    }
+    #createUploadProcessor() {
+        // Only create upload processor if we have a queue manager (which means Redis is available)
+        if (!this.#queueManager) {
+            return null;
+        }
+        return new UploadProcessor(this.#storage, this.#database);
+    }
+    #applyDefaults(options) {
+        return {
+            collectors: [],
+            harvesters: [],
+            handlers: [],
+            assetsManagers: [],
+            customTableManagers: [],
+            server: {
+                port: 3000,
+                host: '0.0.0.0',
+                ...options.server
+            },
+            queues: {
+                multiQueue: true,
+                workers: {
+                    collectors: 1,
+                    harvesters: 1,
+                    ...options.queues?.workers
+                },
+                ...options.queues
+            },
+            logging: {
+                level: LogLevel.INFO,
+                format: 'text',
+                ...options.logging
+            },
+            dryRun: false,
+            ...options
+        };
+    }
+    #createQueueManager() {
+        // Create queue manager if we have collectors, harvesters, OR assets managers that may need async uploads
+        // Note: At construction time, only constructor-provided components are available
+        // Dynamic components registered via register() will be handled at start() time
+        const hasActiveComponents = this.#collectors.length > 0 || this.#harvesters.length > 0;
+        const hasAssetsManagers = this.#assetsManagers.length > 0;
+        if (!hasActiveComponents && !hasAssetsManagers) {
+            return null;
+        }
+        return new QueueManager({
+            redis: this.#options.redis,
+            collectorWorkers: this.#options.queues?.workers?.collectors,
+            harvesterWorkers: this.#options.queues?.workers?.harvesters,
+            queueOptions: this.#options.queues?.options
+        });
+    }
+    /**
+     * Initialize store managers and create their database tables
+     * @private
+     */
+    async #initializeCustomTableManagers(authMiddleware) {
+        for (const customTableManager of this.#customTableManagers) {
+            // Inject dependencies
+            customTableManager.setDependencies(this.#database, authMiddleware);
+            // Initialize the table with custom columns
+            await customTableManager.initializeTable();
+        }
+    }
+    /**
+     * Ensure temporary upload directory exists
+     * @private
+     */
+    async #ensureTempUploadDir() {
+        const tempDir = process.env.TEMP_UPLOAD_DIR || '/tmp/digitaltwin-uploads';
+        try {
+            await fs.mkdir(tempDir, { recursive: true });
+        }
+        catch (error) {
+            throw new Error(`Failed to create temp upload directory ${tempDir}: ${error}`);
+        }
+    }
+    /**
+     * Setup monitoring endpoints for queue statistics and health checks
+     * @private
+     */
+    #setupMonitoringEndpoints() {
+        // Register default health checks
+        this.#healthChecker.registerCheck('database', createDatabaseCheck(this.#database));
+        if (this.#queueManager) {
+            this.#healthChecker.registerCheck('redis', createRedisCheck(this.#queueManager));
+        }
+        this.#healthChecker.registerCheck('storage', createStorageCheck(this.#storage));
+        // Set component counts (includes both constructor and dynamically registered components)
+        this.#healthChecker.setComponentCounts({
+            collectors: this.#allCollectors.length,
+            harvesters: this.#allHarvesters.length,
+            handlers: this.#allHandlers.length,
+            assetsManagers: this.#allAssetsManagers.length
+        });
+        // Liveness probe - shallow check, always returns ok if process is running
+        this.#router.get('/api/health/live', (req, res) => {
+            res.status(200).json(livenessCheck());
+        });
+        // Readiness probe - deep check with database and redis verification
+        this.#router.get('/api/health/ready', async (req, res) => {
+            const health = await this.#healthChecker.performCheck();
+            const statusCode = health.status === 'unhealthy' ? 503 : 200;
+            res.status(statusCode).json(health);
+        });
+        // Full health check endpoint (detailed)
+        this.#router.get('/api/health', async (req, res) => {
+            const health = await this.#healthChecker.performCheck();
+            res.json(health);
+        });
+        // Queue statistics endpoint
+        this.#router.get('/api/queues/stats', async (req, res) => {
+            if (this.#queueManager) {
+                const stats = await this.#queueManager.getQueueStats();
+                res.json(stats);
+            }
+            else {
+                res.json({
+                    collectors: { status: 'No collectors configured' },
+                    harvesters: { status: 'No harvesters configured' }
+                });
+            }
+        });
+    }
+    /**
+     * Starts the Digital Twin Engine
+     *
+     * This method:
+     * 1. Initializes all registered components (collectors, harvesters, handlers)
+     * 2. Set up HTTP endpoints for component access
+     * 3. Configures and starts background job queues
+     * 4. Starts the HTTP server
+     * 5. Exposes queue monitoring endpoints
+     *
+     * @async
+     * @returns {Promise<void>}
+     *
+     * @example
+     * ```TypeScript
+     * await engine.start()
+     * console.log('Engine is running!')
+     * ```
+     */
+    async start() {
+        const isDryRun = this.#options.dryRun ?? false;
+        if (isDryRun) {
+            // In dry run, just validate everything without creating tables
+            const validationResult = await this.validateConfiguration();
+            if (!validationResult.valid) {
+                throw new Error(`Validation failed:\n${validationResult.engineErrors.join('\n')}`);
+            }
+            return;
+        }
+        // Mark as started to prevent component registration
+        this.#isStarted = true;
+        // Recreate queue manager if we have new components registered after construction
+        if (!this.#queueManager && (this.#collectors.length > 0 || this.#harvesters.length > 0 || this.#assetsManagers.length > 0)) {
+            this.#queueManager = this.#createQueueManager();
+        }
+        // Normal startup - initialize user management tables and auth middleware
+        const userRepository = this.#database.getUserRepository();
+        const userService = new UserService(userRepository);
+        await userService.initializeTables();
+        const authMiddleware = new AuthMiddleware(userService);
+        // Get autoMigration setting (default: true)
+        const autoMigration = this.#options.autoMigration ?? true;
+        // Initialize components and create tables if needed
+        await initializeComponents(this.#activeComponents, this.#database, this.#storage, autoMigration);
+        // Initialize assets managers and create their tables if needed
+        await initializeAssetsManagers(this.#assetsManagers, this.#database, this.#storage, autoMigration, authMiddleware);
+        // Initialize store managers and create their tables if needed
+        await this.#initializeCustomTableManagers(authMiddleware);
+        // Initialize handlers (inject dependencies if needed)
+        for (const handler of this.#handlers) {
+            if ('setDependencies' in handler && typeof handler.setDependencies === 'function') {
+                handler.setDependencies(this.#database, this.#storage);
+            }
+            // If it's a GlobalAssetsHandler, inject the AssetsManager instances
+            if ('setAssetsManagers' in handler && typeof handler.setAssetsManagers === 'function') {
+                handler.setAssetsManagers(this.#assetsManagers);
+            }
+        }
+        // Inject upload queue to components that support async uploads
+        if (this.#queueManager) {
+            for (const manager of this.#allAssetsManagers) {
+                if (isAsyncUploadable(manager)) {
+                    manager.setUploadQueue(this.#queueManager.uploadQueue);
+                }
+            }
+        }
+        // Start upload processor worker (for async file processing)
+        // Uses same Redis config as QueueManager (defaults to localhost:6379 if not specified)
+        if (this.#uploadProcessor) {
+            const redisConfig = this.#options.redis || {
+                host: 'localhost',
+                port: 6379,
+                maxRetriesPerRequest: null
+            };
+            this.#uploadProcessor.start(redisConfig);
+        }
+        // Start upload reconciler for presigned uploads (registers all asset manager tables)
+        const assetTableNames = this.#allAssetsManagers.map(m => m.getConfiguration().name);
+        if (assetTableNames.length > 0) {
+            this.#uploadReconciler.registerTables(assetTableNames);
+            this.#uploadReconciler.start();
+        }
+        await exposeEndpoints(this.#router, this.#allComponents);
+        // Setup component scheduling with queue manager (only if we have active components)
+        if (this.#activeComponents.length > 0 && this.#queueManager) {
+            this.#workers = await scheduleComponents(this.#activeComponents, this.#queueManager, this.#isMultiQueueEnabled);
+        }
+        this.#setupMonitoringEndpoints();
+        // Ensure temporary upload directory exists
+        await this.#ensureTempUploadDir();
+        // HTTP compression - disabled by default as API gateways (APISIX, Kong, etc.) typically handle this
+        // Enable with DIGITALTWIN_ENABLE_COMPRESSION=true for standalone deployments without a gateway
+        const enableCompression = process.env.DIGITALTWIN_ENABLE_COMPRESSION === 'true';
+        if (enableCompression) {
+            const compressionMiddleware = compression({
+                filter: (req, res) => {
+                    // Don't compress binary streams
+                    if (req.headers['accept']?.includes('application/octet-stream')) {
+                        return false;
+                    }
+                    // Use default filter for other content types
+                    return compression.filter(req, res);
+                },
+                level: 6, // Balance between speed and compression ratio
+                threshold: 1024 // Only compress responses larger than 1KB
+            });
+            this.#app.use(compressionMiddleware);
+        }
+        // Enable CORS for cross-origin requests from frontend applications
+        this.#app.use(cors({
+            origin: process.env.CORS_ORIGIN || true, // Allow all origins by default, configure in production
+            methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
+            allowedHeaders: ['Content-Type', 'Authorization'],
+            credentials: true // Allow cookies/credentials
+        }));
+        // Configure Express middlewares for body parsing - no limits for large files
+        this.#app.use(express.json({ limit: '10gb' }));
+        this.#app.use(express.urlencoded({ extended: true, limit: '10gb' }));
+        // Add multipart/form-data support for file uploads with disk storage for large files
+        const upload = multer({
+            storage: multer.diskStorage({
+                destination: (req, file, cb) => {
+                    // Use temporary directory, will be cleaned up after processing
+                    const tempDir = process.env.TEMP_UPLOAD_DIR || '/tmp/digitaltwin-uploads';
+                    cb(null, tempDir);
+                },
+                filename: (req, file, cb) => {
+                    // Generate unique filename to avoid conflicts
+                    const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9);
+                    cb(null, file.fieldname + '-' + uniqueSuffix + '-' + file.originalname);
+                }
+            }),
+            limits: {
+                // Remove file size limit to allow large files (10GB+)
+                files: 1, // Only one file per request for safety
+                parts: 10, // Limit form parts
+                headerPairs: 2000 // Limit header pairs
+            }
+        });
+        this.#app.use(upload.single('file'));
+        this.#app.use(this.#router);
+        const { port, host = '0.0.0.0' } = this.#options.server ?? {
+            port: 3000,
+            host: '0.0.0.0'
+        };
+        // Wait for server to be ready
+        // Note: ultimate-express types say (host, port, callback) but implementation accepts (port, host, callback)
+        // The implementation internally reorders to (host, port) for uWebSockets.js
+        // Using type assertion to work around this types bug in ultimate-express
+        await new Promise(resolve => {
+            const app = this.#app;
+            this.#server = app.listen(port, host, () => {
+                resolve();
+            });
+        });
+        // Note: uWebSockets.js (used by ultimate-express) handles timeouts differently than Node.js http.Server:
+        // - HTTP idle timeout is 10 seconds (only when connection is inactive)
+        // - During active data transfer (file uploads), the connection stays open
+        // - Properties like timeout, keepAliveTimeout, headersTimeout don't exist on TemplatedApp
+        // For large file uploads, this should work fine as the connection remains active during transfer.
+        // Attempt to load optional packages (e.g. @cepseudo/ngsi-ld)
+        await this.#loadOptionalPackages();
+    }
+    /**
+     * Attempts to load optional plugin packages that extend the engine.
+     * Failures are silently ignored — the engine works without them.
+     * @private
+     */
+    async #loadOptionalPackages() {
+        try {
+            // Using a computed specifier prevents TypeScript from requiring the module at compile time.
+            // The engine works correctly whether or not this optional package is installed.
+            const ngsiLdPkg = '@cepseudo/ngsi-ld';
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const ngsiLd = await import(ngsiLdPkg);
+            const { Logger } = await import('@cepseudo/shared');
+            await ngsiLd.registerNgsiLd({
+                router: this.#router,
+                db: this.#database,
+                redis: this.getRedisConfig(),
+                components: this.getAllComponents(),
+                logger: new Logger('ngsi-ld'),
+            });
+        }
+        catch {
+            // Package not installed — skip silently
+        }
+    }
+    /**
+     * Get the server port
+     *
+     * @returns {number | undefined} The server port or undefined if not started
+     *
+     * @example
+     * ```TypeScript
+     * const port = engine.getPort()
+     * console.log(`Server running on port ${port}`)
+     * ```
+     */
+    getPort() {
+        if (!this.#server)
+            return undefined;
+        // ultimate-express stores port on the app instance
+        // Use type assertion to access the port property
+        const app = this.#app;
+        return app.port ?? this.#options.server?.port;
+    }
+    /**
+     * Returns the internal Express Router.
+     * Used by optional plugin packages to register additional routes.
+     */
+    getRouter() {
+        return this.#router;
+    }
+    /**
+     * Returns the DatabaseAdapter instance.
+     * Used by optional plugin packages for their own persistence needs.
+     */
+    getDatabase() {
+        return this.#database;
+    }
+    /**
+     * Returns the Redis connection configuration extracted from engine options.
+     * Falls back to localhost:6379 when no Redis config was provided.
+     */
+    getRedisConfig() {
+        const redis = this.#options.redis;
+        if (!redis) {
+            return { host: 'localhost', port: 6379 };
+        }
+        if ('host' in redis && 'port' in redis) {
+            return {
+                host: redis.host,
+                port: redis.port,
+                password: redis.password,
+            };
+        }
+        // Fallback for path-based connection strings
+        return { host: 'localhost', port: 6379 };
+    }
+    /**
+     * Returns a flat list of all components registered with the engine.
+     * Used by optional plugin packages to inspect component capabilities.
+     */
+    getAllComponents() {
+        return [...this.#allComponents];
+    }
+    /**
+     * Registers a single component with automatic type detection.
+     *
+     * The engine automatically detects the component type based on its class
+     * and adds it to the appropriate internal collection.
+     *
+     * @param component - Component instance to register
+     * @returns The engine instance for method chaining
+     * @throws Error if component type cannot be determined or is already registered
+     *
+     * @example
+     * ```typescript
+     * const engine = new DigitalTwinEngine({ storage, database })
+     *
+     * engine
+     *   .register(new WeatherCollector())
+     *   .register(new TrafficAnalysisHarvester())
+     *   .register(new ApiHandler())
+     *   .register(new GLTFAssetsManager())
+     *
+     * await engine.start()
+     * ```
+     */
+    register(component) {
+        if (this.#isStarted) {
+            throw new Error('Cannot register components after the engine has started');
+        }
+        const type = detectComponentType(component);
+        const config = component.getConfiguration();
+        // Check for duplicate registration
+        if (this.#isComponentRegistered(config.name, type)) {
+            throw new Error(`Component "${config.name}" of type "${type}" is already registered. ` +
+                'Each component must have a unique name within its type.');
+        }
+        switch (type) {
+            case 'collector':
+                if (isCollector(component)) {
+                    this.#dynamicCollectors.push(component);
+                }
+                break;
+            case 'harvester':
+                if (isHarvester(component)) {
+                    this.#dynamicHarvesters.push(component);
+                }
+                break;
+            case 'handler':
+                if (isHandler(component)) {
+                    this.#dynamicHandlers.push(component);
+                }
+                break;
+            case 'assets_manager':
+                if (isAssetsManager(component)) {
+                    this.#dynamicAssetsManagers.push(component);
+                }
+                break;
+            case 'custom_table_manager':
+                if (isCustomTableManager(component)) {
+                    this.#dynamicCustomTableManagers.push(component);
+                }
+                break;
+        }
+        return this;
+    }
+    /**
+     * Registers multiple components at once with automatic type detection.
+     *
+     * Useful for registering all components from a module or when loading
+     * components dynamically.
+     *
+     * @param components - Array of component instances to register
+     * @returns The engine instance for method chaining
+     * @throws Error if any component type cannot be determined or is duplicate
+     *
+     * @example
+     * ```typescript
+     * const engine = new DigitalTwinEngine({ storage, database })
+     *
+     * engine.registerAll([
+     *   new WeatherCollector(),
+     *   new TrafficCollector(),
+     *   new AnalysisHarvester(),
+     *   new ApiHandler()
+     * ])
+     *
+     * await engine.start()
+     * ```
+     */
+    registerAll(components) {
+        for (const component of components) {
+            this.register(component);
+        }
+        return this;
+    }
+    /**
+     * Registers components with explicit type specification.
+     *
+     * Provides full type safety at compile time. Use this method when you
+     * have pre-sorted components from auto-discovery or want explicit control.
+     *
+     * @param components - Object with typed component arrays
+     * @returns The engine instance for method chaining
+     *
+     * @example
+     * ```typescript
+     * const loaded = await loadComponents('./src/components')
+     *
+     * engine.registerComponents({
+     *   collectors: loaded.collectors,
+     *   harvesters: loaded.harvesters,
+     *   handlers: loaded.handlers,
+     *   assetsManagers: loaded.assetsManagers,
+     *   customTableManagers: loaded.customTableManagers
+     * })
+     * ```
+     */
+    registerComponents(components) {
+        if (components.collectors) {
+            this.#dynamicCollectors.push(...components.collectors);
+        }
+        if (components.harvesters) {
+            this.#dynamicHarvesters.push(...components.harvesters);
+        }
+        if (components.handlers) {
+            this.#dynamicHandlers.push(...components.handlers);
+        }
+        if (components.assetsManagers) {
+            this.#dynamicAssetsManagers.push(...components.assetsManagers);
+        }
+        if (components.customTableManagers) {
+            this.#dynamicCustomTableManagers.push(...components.customTableManagers);
+        }
+        return this;
+    }
+    /**
+     * Checks if a component with the given name is already registered.
+     */
+    #isComponentRegistered(name, type) {
+        const allComponents = this.#getAllComponentsOfType(type);
+        return allComponents.some(c => c.getConfiguration().name === name);
+    }
+    /**
+     * Gets all components of a specific type (both from constructor and register()).
+     */
+    #getAllComponentsOfType(type) {
+        switch (type) {
+            case 'collector':
+                return this.#allCollectors;
+            case 'harvester':
+                return this.#allHarvesters;
+            case 'handler':
+                return this.#allHandlers;
+            case 'assets_manager':
+                return this.#allAssetsManagers;
+            case 'custom_table_manager':
+                return this.#allCustomTableManagers;
+        }
+    }
+    /**
+     * Configure the shutdown timeout (in ms)
+     * @param timeout Timeout in milliseconds (default: 30000)
+     */
+    setShutdownTimeout(timeout) {
+        this.#shutdownTimeout = timeout;
+    }
+    /**
+     * Check if the engine is currently shutting down
+     * @returns true if shutdown is in progress
+     */
+    isShuttingDown() {
+        return this.#isShuttingDown;
+    }
+    /**
+     * Register a custom health check
+     * @param name Unique name for the check
+     * @param checkFn Function that performs the check
+     *
+     * @example
+     * ```typescript
+     * engine.registerHealthCheck('external-api', async () => {
+     *     try {
+     *         const res = await fetch('https://api.example.com/health')
+     *         return { status: res.ok ? 'up' : 'down' }
+     *     } catch (error) {
+     *         return { status: 'down', error: error.message }
+     *     }
+     * })
+     * ```
+     */
+    registerHealthCheck(name, checkFn) {
+        this.#healthChecker.registerCheck(name, checkFn);
+    }
+    /**
+     * Remove a custom health check
+     * @param name Name of the check to remove
+     * @returns true if the check was removed, false if it didn't exist
+     */
+    removeHealthCheck(name) {
+        return this.#healthChecker.removeCheck(name);
+    }
+    /**
+     * Get list of registered health check names
+     */
+    getHealthCheckNames() {
+        return this.#healthChecker.getCheckNames();
+    }
+    /**
+     * Stops the Digital Twin Engine with graceful shutdown
+     *
+     * This method:
+     * 1. Prevents new work from being accepted
+     * 2. Removes all event listeners
+     * 3. Closes HTTP server
+     * 4. Drains queues and waits for active jobs
+     * 5. Closes all queue workers
+     * 6. Stops upload processor
+     * 7. Closes queue manager
+     * 8. Closes database connections
+     *
+     * @async
+     * @returns {Promise<void>}
+     *
+     * @example
+     * ```TypeScript
+     * await engine.stop()
+     * console.log('Engine stopped gracefully')
+     * ```
+     */
+    async stop() {
+        if (this.#isShuttingDown) {
+            if (process.env.NODE_ENV !== 'test') {
+                console.warn('[DigitalTwin] Shutdown already in progress');
+            }
+            return;
+        }
+        this.#isShuttingDown = true;
+        const startTime = Date.now();
+        if (process.env.NODE_ENV !== 'test') {
+            console.log('[DigitalTwin] Graceful shutdown initiated...');
+        }
+        const errors = [];
+        // 1. Remove all event listeners to prevent new work
+        this.#cleanupEventListeners();
+        // 2. Close HTTP server (uWebSockets.js TemplatedApp.close() is synchronous)
+        if (this.#server) {
+            try {
+                this.#server.close();
+            }
+            catch (error) {
+                errors.push(this.#wrapError('Server close', error));
+            }
+            this.#server = undefined;
+        }
+        // 3. Drain queues - wait for active jobs with timeout
+        if (this.#queueManager) {
+            try {
+                await this.#drainQueues();
+            }
+            catch (error) {
+                errors.push(this.#wrapError('Queue drain', error));
+            }
+        }
+        // 4. Close all workers with extended timeout and force close
+        await this.#closeWorkers(errors);
+        // 5. Stop upload reconciler and processor
+        try {
+            this.#uploadReconciler.stop();
+        }
+        catch (error) {
+            errors.push(this.#wrapError('Upload reconciler', error));
+        }
+        if (this.#uploadProcessor) {
+            try {
+                await this.#uploadProcessor.stop();
+            }
+            catch (error) {
+                errors.push(this.#wrapError('Upload processor', error));
+            }
+        }
+        // 6. Close queue connections (only if we have a queue manager)
+        if (this.#queueManager) {
+            try {
+                await this.#queueManager.close();
+            }
+            catch (error) {
+                errors.push(this.#wrapError('Queue manager', error));
+            }
+        }
+        // 7. Close database connections
+        try {
+            await this.#database.close();
+        }
+        catch (error) {
+            errors.push(this.#wrapError('Database', error));
+        }
+        const duration = Date.now() - startTime;
+        if (process.env.NODE_ENV !== 'test') {
+            if (errors.length > 0) {
+                console.error(`[DigitalTwin] Shutdown completed with ${errors.length} errors in ${duration}ms:`, errors.map(e => e.message).join(', '));
+            }
+            else {
+                console.log(`[DigitalTwin] Shutdown completed successfully in ${duration}ms`);
+            }
+        }
+    }
+    #cleanupEventListeners() {
+        engineEventBus.removeAllListeners();
+    }
+    async #drainQueues() {
+        if (!this.#queueManager)
+            return;
+        const timeout = Math.min(this.#shutdownTimeout / 2, 15000);
+        const startTime = Date.now();
+        while (Date.now() - startTime < timeout) {
+            try {
+                const stats = await this.#queueManager.getQueueStats();
+                const totalActive = Object.values(stats).reduce((sum, q) => sum + (q.active || 0), 0);
+                if (totalActive === 0)
+                    break;
+                if (process.env.NODE_ENV !== 'test') {
+                    console.log(`[DigitalTwin] Waiting for ${totalActive} active jobs...`);
+                }
+                await new Promise(resolve => setTimeout(resolve, 1000));
+            }
+            catch {
+                await new Promise(resolve => setTimeout(resolve, 1000));
+                break;
+            }
+        }
+    }
+    async #closeWorkers(errors) {
+        const workerTimeout = Math.min(this.#shutdownTimeout / 3, 10000);
+        await Promise.all(this.#workers.map(async (worker) => {
+            try {
+                await Promise.race([
+                    worker.close(),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('Worker close timeout')), workerTimeout))
+                ]);
+            }
+            catch {
+                try {
+                    await worker.disconnect();
+                }
+                catch (disconnectError) {
+                    errors.push(this.#wrapError('Worker disconnect', disconnectError));
+                }
+            }
+        }));
+        this.#workers = [];
+    }
+    #wrapError(context, error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return new Error(`${context}: ${message}`);
+    }
+    /**
+     * Validate the engine configuration and all components
+     *
+     * This method checks that all components are properly configured and can be initialized
+     * without actually creating tables or starting the server.
+     *
+     * @returns {Promise<ValidationResult>} Comprehensive validation results
+     *
+     * @example
+     * ```typescript
+     * const result = await engine.validateConfiguration()
+     * if (!result.valid) {
+     *   console.error('Validation errors:', result.engineErrors)
+     * }
+     * ```
+     */
+    async validateConfiguration() {
+        const componentResults = [];
+        const engineErrors = [];
+        // Validate collectors (includes dynamically registered)
+        for (const collector of this.#allCollectors) {
+            componentResults.push(await this.#validateComponent(collector, 'collector'));
+        }
+        // Validate harvesters (includes dynamically registered)
+        for (const harvester of this.#allHarvesters) {
+            componentResults.push(await this.#validateComponent(harvester, 'harvester'));
+        }
+        // Validate handlers (includes dynamically registered)
+        for (const handler of this.#allHandlers) {
+            componentResults.push(await this.#validateComponent(handler, 'handler'));
+        }
+        // Validate assets managers (includes dynamically registered)
+        for (const assetsManager of this.#allAssetsManagers) {
+            componentResults.push(await this.#validateComponent(assetsManager, 'assets_manager'));
+        }
+        // Validate custom table managers (includes dynamically registered)
+        for (const customTableManager of this.#allCustomTableManagers) {
+            componentResults.push(await this.#validateComponent(customTableManager, 'custom_table_manager'));
+        }
+        // Validate engine-level configuration
+        try {
+            if (!this.#storage) {
+                engineErrors.push('Storage service is required');
+            }
+            if (!this.#database) {
+                engineErrors.push('Database adapter is required');
+            }
+            // Test storage connection
+            if (this.#storage && typeof this.#storage.save === 'function') {
+                // Storage validation passed
+            }
+            else {
+                engineErrors.push('Storage service does not implement required methods');
+            }
+            // Test database connection
+            if (this.#database && typeof this.#database.save === 'function') {
+                // Database validation passed
+            }
+            else {
+                engineErrors.push('Database adapter does not implement required methods');
+            }
+        }
+        catch (error) {
+            engineErrors.push(`Engine configuration error: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        // Calculate summary
+        const validComponents = componentResults.filter(c => c.valid).length;
+        const totalWarnings = componentResults.reduce((acc, c) => acc + c.warnings.length, 0);
+        const result = {
+            valid: componentResults.every(c => c.valid) && engineErrors.length === 0,
+            components: componentResults,
+            engineErrors,
+            summary: {
+                total: componentResults.length,
+                valid: validComponents,
+                invalid: componentResults.length - validComponents,
+                warnings: totalWarnings
+            }
+        };
+        return result;
+    }
+    /**
+     * Test all components by running their core methods without persistence
+     *
+     * @returns {Promise<ComponentValidationResult[]>} Test results for each component
+     *
+     * @example
+     * ```typescript
+     * const results = await engine.testComponents()
+     * results.forEach(result => {
+     *   console.log(`${result.name}: ${result.valid ? '✅' : '❌'}`)
+     * })
+     * ```
+     */
+    async testComponents() {
+        const results = [];
+        // Test collectors (includes dynamically registered)
+        for (const collector of this.#allCollectors) {
+            const result = await this.#testCollector(collector);
+            results.push(result);
+        }
+        // Test harvesters (includes dynamically registered)
+        for (const harvester of this.#allHarvesters) {
+            const result = await this.#testHarvester(harvester);
+            results.push(result);
+        }
+        // Test handlers (includes dynamically registered)
+        for (const handler of this.#allHandlers) {
+            const result = await this.#testHandler(handler);
+            results.push(result);
+        }
+        // Test assets managers (includes dynamically registered)
+        for (const assetsManager of this.#allAssetsManagers) {
+            const result = await this.#testAssetsManager(assetsManager);
+            results.push(result);
+        }
+        return results;
+    }
+    /**
+     * Validate a single component
+     */
+    async #validateComponent(component, type) {
+        const errors = [];
+        const warnings = [];
+        try {
+            // Check if component has required methods
+            if (typeof component.getConfiguration !== 'function') {
+                errors.push('Component must implement getConfiguration() method');
+            }
+            const config = component.getConfiguration();
+            // Validate configuration
+            if (!config.name) {
+                errors.push('Component configuration must have a name');
+            }
+            if (!config.description) {
+                warnings.push('Component configuration should have a description');
+            }
+            // Type-specific validation
+            if (type === 'collector' || type === 'harvester') {
+                const activeComponent = component;
+                if (typeof activeComponent.setDependencies !== 'function') {
+                    errors.push('Active components must implement setDependencies() method');
+                }
+            }
+            if (type === 'collector') {
+                const collector = component;
+                if (typeof collector.collect !== 'function') {
+                    errors.push('Collector must implement collect() method');
+                }
+                if (typeof collector.getSchedule !== 'function') {
+                    errors.push('Collector must implement getSchedule() method');
+                }
+            }
+            if (type === 'harvester') {
+                const harvester = component;
+                if (typeof harvester.harvest !== 'function') {
+                    errors.push('Harvester must implement harvest() method');
+                }
+            }
+            if (type === 'assets_manager') {
+                const assetsManager = component;
+                if (typeof assetsManager.uploadAsset !== 'function') {
+                    errors.push('AssetsManager must implement uploadAsset() method');
+                }
+                if (typeof assetsManager.getAllAssets !== 'function') {
+                    errors.push('AssetsManager must implement getAllAssets() method');
+                }
+            }
+            if (type === 'custom_table_manager') {
+                const customTableManager = component;
+                if (typeof customTableManager.setDependencies !== 'function') {
+                    errors.push('CustomTableManager must implement setDependencies() method');
+                }
+                if (typeof customTableManager.initializeTable !== 'function') {
+                    errors.push('CustomTableManager must implement initializeTable() method');
+                }
+                // Validate store configuration
+                const config = customTableManager.getConfiguration();
+                if (typeof config !== 'object' || config === null) {
+                    errors.push('CustomTableManager must return a valid configuration object');
+                }
+                else {
+                    if (!config.columns || typeof config.columns !== 'object') {
+                        errors.push('CustomTableManager configuration must define columns');
+                    }
+                    else {
+                        // Validate columns definition
+                        const columnCount = Object.keys(config.columns).length;
+                        if (columnCount === 0) {
+                            warnings.push('CustomTableManager has no custom columns defined');
+                        }
+                        // Validate column names and types
+                        for (const [columnName, columnType] of Object.entries(config.columns)) {
+                            if (!columnName || typeof columnName !== 'string') {
+                                errors.push('Column names must be non-empty strings');
+                            }
+                            if (!columnType || typeof columnType !== 'string') {
+                                errors.push(`Column '${columnName}' must have a valid SQL type`);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            errors.push(`Validation error: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        return {
+            name: component.getConfiguration?.()?.name || 'unknown',
+            type,
+            valid: errors.length === 0,
+            errors,
+            warnings
+        };
+    }
+    /**
+     * Test a collector by running its collect method
+     */
+    async #testCollector(collector) {
+        const errors = [];
+        const warnings = [];
+        const config = collector.getConfiguration();
+        try {
+            // Test the collect method
+            const result = await collector.collect();
+            if (!Buffer.isBuffer(result)) {
+                errors.push('collect() method must return a Buffer');
+            }
+            if (result.length === 0) {
+                warnings.push('collect() method returned empty buffer');
+            }
+        }
+        catch (error) {
+            errors.push(`collect() method failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        return {
+            name: config.name,
+            type: 'collector',
+            valid: errors.length === 0,
+            errors,
+            warnings
+        };
+    }
+    /**
+     * Test a harvester (more complex as it needs mock data)
+     */
+    async #testHarvester(harvester) {
+        const errors = [];
+        const warnings = [];
+        const config = harvester.getConfiguration();
+        try {
+            // Create mock data for testing
+            const mockData = {
+                id: 1,
+                name: 'test',
+                date: new Date(),
+                contentType: 'application/json',
+                url: 'test://url',
+                data: async () => Buffer.from('{"test": true}')
+            };
+            // Test the harvest method
+            const result = await harvester.harvest(mockData, {});
+            if (!Buffer.isBuffer(result)) {
+                errors.push('harvest() method must return a Buffer');
+            }
+        }
+        catch (error) {
+            errors.push(`harvest() method failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        return {
+            name: config.name,
+            type: 'harvester',
+            valid: errors.length === 0,
+            errors,
+            warnings
+        };
+    }
+    /**
+     * Test a handler
+     */
+    async #testHandler(handler) {
+        const errors = [];
+        const warnings = [];
+        const config = handler.getConfiguration();
+        try {
+            // Handlers are mostly validated through their endpoint configuration
+            if (typeof handler.getEndpoints === 'function') {
+                const endpoints = handler.getEndpoints();
+                if (!Array.isArray(endpoints)) {
+                    errors.push('getEndpoints() must return an array');
+                }
+            }
+        }
+        catch (error) {
+            errors.push(`Handler test failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        return {
+            name: config.name,
+            type: 'handler',
+            valid: errors.length === 0,
+            errors,
+            warnings
+        };
+    }
+    /**
+     * Test an assets manager
+     */
+    async #testAssetsManager(assetsManager) {
+        const errors = [];
+        const warnings = [];
+        const config = assetsManager.getConfiguration();
+        try {
+            // Test configuration
+            if (!config.contentType) {
+                errors.push('AssetsManager configuration must have a contentType');
+            }
+            // In dry run mode, we can't test actual upload/download without dependencies
+            // Just validate that the methods exist and are callable
+            if (typeof assetsManager.getEndpoints === 'function') {
+                const endpoints = assetsManager.getEndpoints();
+                if (!Array.isArray(endpoints)) {
+                    errors.push('getEndpoints() must return an array');
+                }
+            }
+        }
+        catch (error) {
+            errors.push(`AssetsManager test failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        return {
+            name: config.name,
+            type: 'assets_manager',
+            valid: errors.length === 0,
+            errors,
+            warnings
+        };
+    }
+}
+//# sourceMappingURL=digital_twin_engine.js.map