@cepseudo/engine 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Axel Hoffmann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,209 @@
+# @cepseudo/engine
+
+[![Node.js](https://img.shields.io/badge/node-%3E%3D20.0.0-brightgreen)](https://nodejs.org/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](./LICENSE)
+
+Central orchestrator for the Digital Twin Framework. Wires together components, databases, storage, authentication, queues, and HTTP serving into a single runnable application.
+
+## Installation
+
+```bash
+pnpm add @cepseudo/engine
+```
+
+This package depends on the full stack of `@cepseudo/*` packages (shared, database, storage, auth, assets, components) and requires Redis for queue management.
+
+## Quick Start
+
+```typescript
+import { DigitalTwinEngine } from '@cepseudo/engine'
+import { KnexDatabaseAdapter } from '@cepseudo/database'
+import { StorageServiceFactory } from '@cepseudo/storage'
+import { MyCollector } from './collectors/my_collector.js'
+
+const database = new KnexDatabaseAdapter({
+    client: 'pg',
+    connection: process.env.DATABASE_URL
+})
+
+const storage = StorageServiceFactory.create()
+
+const engine = new DigitalTwinEngine({
+    storage,
+    database,
+    collectors: [new MyCollector()],
+    server: { port: 3000 }
+})
+
+await engine.start()
+```
+
+## Features
+
+- **Component lifecycle management** -- registers, validates, and initializes collectors, harvesters, handlers, assets managers, and custom table managers
+- **Dependency injection** -- automatically injects database, storage, and auth middleware into components
+- **BullMQ scheduling** -- cron-based and event-driven scheduling across 4 queues (collectors, harvesters, priority, uploads)
+- **HTTP server** -- Express-compatible server (via ultimate-express) with automatic endpoint registration from components
+- **Health checks** -- aggregated health status for Kubernetes readiness/liveness probes (database, Redis, storage)
+- **OpenAPI generation** -- auto-generates OpenAPI 3.0.3 specs from registered components
+- **Graceful shutdown** -- handles SIGTERM/SIGINT with configurable timeout and ordered resource cleanup
+- **Dynamic component loading** -- loads components from user project directories at runtime
+
+## Configuration
+
+The `EngineOptions` interface controls engine behavior:
+
+```typescript
+interface EngineOptions {
+    // Required
+    storage: StorageService
+    database: DatabaseAdapter
+
+    // Components (register via constructor or engine.register())
+    collectors?: Collector[]
+    harvesters?: Harvester[]
+    handlers?: Handler[]
+    assetsManagers?: AssetsManager[]
+    customTableManagers?: CustomTableManager[]
+
+    // Redis & queues
+    redis?: ConnectionOptions
+    queues?: {
+        multiQueue?: boolean                    // default: true
+        workers?: {
+            collectors?: number                 // default: 1
+            harvesters?: number                 // default: 1
+        }
+        options?: QueueConfig['queueOptions']
+    }
+
+    // HTTP server
+    server?: {
+        port: number                            // default: 3000
+        host?: string                           // default: '0.0.0.0'
+    }
+
+    // Logging
+    logging?: {
+        level: LogLevel                         // default: LogLevel.INFO
+        format?: 'json' | 'text'               // default: 'text'
+    }
+
+    dryRun?: boolean                            // default: false
+    autoMigration?: boolean                     // default: true
+}
+```
+
+## Usage Examples
+
+### Registering components dynamically
+
+Components can be registered after construction, before calling `start()`:
+
+```typescript
+const engine = new DigitalTwinEngine({ storage, database })
+
+engine.register(new WeatherCollector())
+engine.register(new AirQualityHandler())
+engine.registerAll([new TilesetManager(), new MapManager()])
+
+await engine.start()
+```
+
+### Health checks
+
+The engine includes built-in health checks for database, Redis, and storage. Custom checks can be added:
+
+```typescript
+import {
+    HealthChecker,
+    createDatabaseCheck,
+    createRedisCheck,
+    createStorageCheck
+} from '@cepseudo/engine'
+
+const checker = new HealthChecker()
+checker.register('database', createDatabaseCheck(database))
+checker.register('redis', createRedisCheck(queueManager))
+checker.register('storage', createStorageCheck(storage))
+
+// Returns { status: 'healthy' | 'degraded' | 'unhealthy', checks: {...} }
+const status = await checker.check()
+```
+
+The engine automatically registers health endpoints:
+- `GET /health` -- full health status (readiness probe)
+- `GET /health/live` -- lightweight liveness probe
+
+### OpenAPI spec generation
+
+```typescript
+import { OpenAPIGenerator } from '@cepseudo/engine'
+
+const generator = new OpenAPIGenerator({
+    title: 'My Digital Twin API',
+    version: '1.0.0',
+    description: 'Digital twin for city infrastructure'
+})
+
+// Components expose their endpoint schemas via getConfiguration()
+const spec = generator.generate(components)
+// Returns an OpenAPI 3.0.3 JSON object
+```
+
+### Graceful shutdown
+
+```typescript
+import { setupGracefulShutdown } from '@cepseudo/engine'
+
+// Automatically handles SIGTERM and SIGINT
+setupGracefulShutdown({
+    engine,
+    timeout: 30000 // ms before forced exit
+})
+```
+
+### Dynamic component loading
+
+Load components from a user project directory at runtime:
+
+```typescript
+import { loadComponents } from '@cepseudo/engine'
+
+const result = await loadComponents({
+    directory: './src/components',
+    recursive: true
+})
+
+engine.registerAll([
+    ...result.collectors,
+    ...result.harvesters,
+    ...result.handlers
+])
+```
+
+## Architecture
+
+`@cepseudo/engine` is the LAYER 3 (top layer) package in the Digital Twin Framework. It depends on all lower layers and acts as the composition root that ties everything together.
+
+```
+LAYER 3:  engine          -- orchestration, HTTP, scheduling, health
+LAYER 2:  assets, components  -- business logic, file management
+LAYER 1:  database, storage, auth  -- infrastructure adapters
+LAYER 0:  shared          -- types, errors, utilities, validation
+```
+
+On `engine.start()`, the following sequence executes:
+
+1. **Database initialization** -- runs migrations, creates component tables
+2. **Component initialization** -- injects dependencies (database, storage, auth middleware) into active components
+3. **Endpoint registration** -- maps component HTTP endpoints to Express routes
+4. **Queue setup** -- creates BullMQ queues and workers backed by Redis
+5. **Scheduling** -- registers cron schedules and event triggers for collectors/harvesters
+6. **Server start** -- binds the HTTP server and begins accepting requests
+
+On `engine.stop()`, resources are cleaned up in reverse order with a configurable timeout to allow in-flight requests and queue jobs to complete.
+
+## License
+
+MIT

package/dist/component_types.d.ts

@@ -0,0 +1,92 @@
+/**
+ * @fileoverview Type definitions and guards for dynamic component registration
+ *
+ * Provides type-safe utilities for runtime component type detection and
+ * dynamic registration with the DigitalTwinEngine.
+ */
+import type { Collector, Harvester, Handler, CustomTableManager } from '@cepseudo/components';
+import type { AssetsManager } from '@cepseudo/assets';
+/**
+ * String literal union of all component type names.
+ * Used for discriminated unions and type narrowing.
+ */
+export type ComponentTypeName = 'collector' | 'harvester' | 'handler' | 'assets_manager' | 'custom_table_manager';
+/**
+ * Union type of all component class instances that can be registered.
+ */
+export type AnyComponent = Collector | Harvester | Handler | AssetsManager | CustomTableManager;
+/**
+ * Union type of components that can be scheduled (have run() and getSchedule()).
+ */
+export type SchedulableComponent = Collector | Harvester;
+/**
+ * Union type of components that require setDependencies(db, storage).
+ */
+export type ActiveComponent = Collector | Harvester;
+/**
+ * Map of component type names to their corresponding class types.
+ * Enables type-safe lookups and factory patterns.
+ */
+export interface ComponentTypeMap {
+    collector: Collector;
+    harvester: Harvester;
+    handler: Handler;
+    assets_manager: AssetsManager;
+    custom_table_manager: CustomTableManager;
+}
+/**
+ * Result of loading components from a directory.
+ */
+export interface LoadedComponents {
+    collectors: Collector[];
+    harvesters: Harvester[];
+    handlers: Handler[];
+    assetsManagers: AssetsManager[];
+    customTableManagers: CustomTableManager[];
+}
+/**
+ * Type guard: Check if component is a Collector.
+ * Collectors have collect() method and getSchedule().
+ */
+export declare function isCollector(component: AnyComponent): component is Collector;
+/**
+ * Type guard: Check if component is a Harvester.
+ * Harvesters have harvest() method and getUserConfiguration().
+ */
+export declare function isHarvester(component: AnyComponent): component is Harvester;
+/**
+ * Type guard: Check if component is a Handler.
+ * Handlers have getEndpoints() but NOT setDependencies() by default,
+ * and don't have collect() or harvest().
+ */
+export declare function isHandler(component: AnyComponent): component is Handler;
+/**
+ * Type guard: Check if component is an AssetsManager.
+ * AssetsManager has uploadAsset() and getAllAssets().
+ */
+export declare function isAssetsManager(component: AnyComponent): component is AssetsManager;
+/**
+ * Type guard: Check if component is a CustomTableManager.
+ * CustomTableManager has initializeTable() and findAll().
+ */
+export declare function isCustomTableManager(component: AnyComponent): component is CustomTableManager;
+/**
+ * Type guard: Check if component is an active (schedulable) component.
+ */
+export declare function isActiveComponent(component: AnyComponent): component is ActiveComponent;
+/**
+ * Detect the component type from an instance.
+ * Uses type guards for accurate detection.
+ *
+ * @param component - The component instance to detect
+ * @returns The detected component type name
+ * @throws Error if component type cannot be determined
+ *
+ * @example
+ * ```typescript
+ * const type = detectComponentType(myComponent)
+ * // type: 'collector' | 'harvester' | 'handler' | 'assets_manager' | 'custom_table_manager'
+ * ```
+ */
+export declare function detectComponentType(component: AnyComponent): ComponentTypeName;
+//# sourceMappingURL=component_types.d.ts.map

package/dist/component_types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"component_types.d.ts","sourceRoot":"","sources":["../src/component_types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AAC7F,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErD;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,WAAW,GAAG,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,sBAAsB,CAAA;AAEjH;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,GAAG,aAAa,GAAG,kBAAkB,CAAA;AAE/F;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,SAAS,GAAG,SAAS,CAAA;AAExD;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,SAAS,CAAA;AAEnD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B,SAAS,EAAE,SAAS,CAAA;IACpB,SAAS,EAAE,SAAS,CAAA;IACpB,OAAO,EAAE,OAAO,CAAA;IAChB,cAAc,EAAE,aAAa,CAAA;IAC7B,oBAAoB,EAAE,kBAAkB,CAAA;CAC3C;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC7B,UAAU,EAAE,SAAS,EAAE,CAAA;IACvB,UAAU,EAAE,SAAS,EAAE,CAAA;IACvB,QAAQ,EAAE,OAAO,EAAE,CAAA;IACnB,cAAc,EAAE,aAAa,EAAE,CAAA;IAC/B,mBAAmB,EAAE,kBAAkB,EAAE,CAAA;CAC5C;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,YAAY,GAAG,SAAS,IAAI,SAAS,CAO3E;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,YAAY,GAAG,SAAS,IAAI,SAAS,CAO3E;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,YAAY,GAAG,SAAS,IAAI,OAAO,CASvE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,YAAY,GAAG,SAAS,IAAI,aAAa,CAMnF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,YAAY,GAAG,SAAS,IAAI,kBAAkB,CAO7F;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,YAAY,GAAG,SAAS,IAAI,eAAe,CAEvF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,YAAY,GAAG,iBAAiB,CAW9E"}

package/dist/component_types.js

@@ -0,0 +1,93 @@
+/**
+ * @fileoverview Type definitions and guards for dynamic component registration
+ *
+ * Provides type-safe utilities for runtime component type detection and
+ * dynamic registration with the DigitalTwinEngine.
+ */
+/**
+ * Type guard: Check if component is a Collector.
+ * Collectors have collect() method and getSchedule().
+ */
+export function isCollector(component) {
+    return (typeof component.collect === 'function' &&
+        typeof component.getSchedule === 'function' &&
+        typeof component.setDependencies === 'function' &&
+        typeof component.run === 'function');
+}
+/**
+ * Type guard: Check if component is a Harvester.
+ * Harvesters have harvest() method and getUserConfiguration().
+ */
+export function isHarvester(component) {
+    return (typeof component.harvest === 'function' &&
+        typeof component.getUserConfiguration === 'function' &&
+        typeof component.setDependencies === 'function' &&
+        typeof component.run === 'function');
+}
+/**
+ * Type guard: Check if component is a Handler.
+ * Handlers have getEndpoints() but NOT setDependencies() by default,
+ * and don't have collect() or harvest().
+ */
+export function isHandler(component) {
+    return (typeof component.getEndpoints === 'function' &&
+        typeof component.getConfiguration === 'function' &&
+        !('collect' in component && typeof component.collect === 'function') &&
+        !('harvest' in component && typeof component.harvest === 'function') &&
+        !('uploadAsset' in component && typeof component.uploadAsset === 'function') &&
+        !('initializeTable' in component && typeof component.initializeTable === 'function'));
+}
+/**
+ * Type guard: Check if component is an AssetsManager.
+ * AssetsManager has uploadAsset() and getAllAssets().
+ */
+export function isAssetsManager(component) {
+    return (typeof component.uploadAsset === 'function' &&
+        typeof component.getAllAssets === 'function' &&
+        typeof component.setDependencies === 'function');
+}
+/**
+ * Type guard: Check if component is a CustomTableManager.
+ * CustomTableManager has initializeTable() and findAll().
+ */
+export function isCustomTableManager(component) {
+    return (typeof component.initializeTable === 'function' &&
+        typeof component.findAll === 'function' &&
+        typeof component.create === 'function' &&
+        typeof component.setDependencies === 'function');
+}
+/**
+ * Type guard: Check if component is an active (schedulable) component.
+ */
+export function isActiveComponent(component) {
+    return isCollector(component) || isHarvester(component);
+}
+/**
+ * Detect the component type from an instance.
+ * Uses type guards for accurate detection.
+ *
+ * @param component - The component instance to detect
+ * @returns The detected component type name
+ * @throws Error if component type cannot be determined
+ *
+ * @example
+ * ```typescript
+ * const type = detectComponentType(myComponent)
+ * // type: 'collector' | 'harvester' | 'handler' | 'assets_manager' | 'custom_table_manager'
+ * ```
+ */
+export function detectComponentType(component) {
+    // Order matters: check more specific types first
+    if (isCollector(component))
+        return 'collector';
+    if (isHarvester(component))
+        return 'harvester';
+    if (isCustomTableManager(component))
+        return 'custom_table_manager';
+    if (isAssetsManager(component))
+        return 'assets_manager';
+    if (isHandler(component))
+        return 'handler';
+    throw new Error('Unable to detect component type. Component must extend Collector, Harvester, Handler, AssetsManager, or CustomTableManager.');
+}
+//# sourceMappingURL=component_types.js.map

package/dist/component_types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"component_types.js","sourceRoot":"","sources":["../src/component_types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiDH;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,SAAuB;IAC/C,OAAO,CACH,OAAQ,SAAuB,CAAC,OAAO,KAAK,UAAU;QACtD,OAAQ,SAAuB,CAAC,WAAW,KAAK,UAAU;QAC1D,OAAQ,SAAuB,CAAC,eAAe,KAAK,UAAU;QAC9D,OAAQ,SAAuB,CAAC,GAAG,KAAK,UAAU,CACrD,CAAA;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,SAAuB;IAC/C,OAAO,CACH,OAAQ,SAAuB,CAAC,OAAO,KAAK,UAAU;QACtD,OAAQ,SAAuB,CAAC,oBAAoB,KAAK,UAAU;QACnE,OAAQ,SAAuB,CAAC,eAAe,KAAK,UAAU;QAC9D,OAAQ,SAAuB,CAAC,GAAG,KAAK,UAAU,CACrD,CAAA;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,SAAuB;IAC7C,OAAO,CACH,OAAQ,SAAqB,CAAC,YAAY,KAAK,UAAU;QACzD,OAAQ,SAAqB,CAAC,gBAAgB,KAAK,UAAU;QAC7D,CAAC,CAAC,SAAS,IAAI,SAAS,IAAI,OAAQ,SAAiB,CAAC,OAAO,KAAK,UAAU,CAAC;QAC7E,CAAC,CAAC,SAAS,IAAI,SAAS,IAAI,OAAQ,SAAiB,CAAC,OAAO,KAAK,UAAU,CAAC;QAC7E,CAAC,CAAC,aAAa,IAAI,SAAS,IAAI,OAAQ,SAAiB,CAAC,WAAW,KAAK,UAAU,CAAC;QACrF,CAAC,CAAC,iBAAiB,IAAI,SAAS,IAAI,OAAQ,SAAiB,CAAC,eAAe,KAAK,UAAU,CAAC,CAChG,CAAA;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAuB;IACnD,OAAO,CACH,OAAQ,SAA2B,CAAC,WAAW,KAAK,UAAU;QAC9D,OAAQ,SAA2B,CAAC,YAAY,KAAK,UAAU;QAC/D,OAAQ,SAA2B,CAAC,eAAe,KAAK,UAAU,CACrE,CAAA;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAuB;IACxD,OAAO,CACH,OAAQ,SAAgC,CAAC,eAAe,KAAK,UAAU;QACvE,OAAQ,SAAgC,CAAC,OAAO,KAAK,UAAU;QAC/D,OAAQ,SAAgC,CAAC,MAAM,KAAK,UAAU;QAC9D,OAAQ,SAAgC,CAAC,eAAe,KAAK,UAAU,CAC1E,CAAA;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAuB;IACrD,OAAO,WAAW,CAAC,SAAS,CAAC,IAAI,WAAW,CAAC,SAAS,CAAC,CAAA;AAC3D,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAAuB;IACvD,iDAAiD;IACjD,IAAI,WAAW,CAAC,SAAS,CAAC;QAAE,OAAO,WAAW,CAAA;IAC9C,IAAI,WAAW,CAAC,SAAS,CAAC;QAAE,OAAO,WAAW,CAAA;IAC9C,IAAI,oBAAoB,CAAC,SAAS,CAAC;QAAE,OAAO,sBAAsB,CAAA;IAClE,IAAI,eAAe,CAAC,SAAS,CAAC;QAAE,OAAO,gBAAgB,CAAA;IACvD,IAAI,SAAS,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAA;IAE1C,MAAM,IAAI,KAAK,CACX,6HAA6H,CAChI,CAAA;AACL,CAAC"}