digitaltwin-core 0.14.3 → 1.0.1

package/README.md

@@ -12,6 +12,7 @@ Digital Twin Core is a minimalist TypeScript framework used to collect and proce
 - **Storage adapters** – currently local filesystem and OVH Object Storage via S3 API.
 - **Database adapter** – implemented with [Knex](https://knexjs.org/) to index metadata.
 - **Engine** – orchestrates components, schedules jobs with BullMQ and exposes endpoints via Express.
+- **Authentication** – pluggable authentication system supporting API gateway headers, JWT tokens, or no-auth mode.
 
 ## Installation
 
@@ -65,6 +66,70 @@ const engine = new DigitalTwinEngine({ storage, database });
 engine.start();
 ```
 
+## Developer Experience
+
+### Auto-Discovery
+
+The framework provides automatic component discovery to reduce boilerplate:
+
+```typescript
+import { DigitalTwinEngine, loadComponents } from 'digitaltwin-core'
+
+// Before: manual imports for each component
+// import { WeatherCollector } from './components/weather_collector.js'
+// import { SensorCollector } from './components/sensor_collector.js'
+// ...
+
+// After: automatic discovery
+const result = await loadComponents('./dist/components')
+
+const engine = new DigitalTwinEngine({ storage, database })
+engine.registerComponents(result)
+await engine.start()
+```
+
+The loader discovers components based on file naming conventions:
+- `*_collector.js` → Collectors
+- `*_harvester.js` → Harvesters
+- `*_handler.js` → Handlers
+- `*_assets_manager.js` → Assets Managers
+- `*_custom_table.js` → Custom Table Managers
+
+### Loading Options
+
+```typescript
+const result = await loadComponents('./dist/components', {
+  recursive: true,        // Scan subdirectories (default)
+  verbose: true,          // Log discovered components
+  extensions: ['.js'],    // File extensions
+  exclude: ['*.test.*']   // Exclusion patterns
+})
+
+// Result includes metadata
+console.log(result.summary)
+// { total: 5, collectors: 2, harvesters: 1, handlers: 2, errors: 0 }
+
+// Check for loading errors
+if (result.errors.length > 0) {
+  console.warn('Some components failed to load:', result.errors)
+}
+```
+
+### Type Guards
+
+Runtime type detection utilities:
+
+```typescript
+import { isCollector, isHandler, detectComponentType } from 'digitaltwin-core'
+
+if (isCollector(component)) {
+  // TypeScript narrowing works here
+  await component.collect()
+}
+
+const type = detectComponentType(component) // 'collector' | 'handler' | ...
+```
+
 ## Components
 
 ### Collectors
@@ -326,6 +391,153 @@ class WMSLayersManager extends CustomTableManager {
 - `update(id, data)` - Update existing record
 - `delete(id)` - Delete record
 
+## Request Validation
+
+The framework includes VineJS integration for type-safe request validation:
+
+```typescript
+import { validate, AssetUploadSchema } from 'digitaltwin-core'
+
+// In your component
+async handleUpload(req: any) {
+  const data = await validate(AssetUploadSchema, req.body)
+  // data is now typed and validated
+}
+```
+
+Validation errors return HTTP 422 with detailed error messages:
+
+```json
+{
+  "error": "Validation failed",
+  "details": [
+    { "field": "description", "message": "The description field must be a string" }
+  ]
+}
+```
+
+## Error Handling
+
+The framework provides custom error classes for structured error handling:
+
+```typescript
+import { CollectorError, ValidationError, StorageError } from 'digitaltwin-core'
+
+// Errors include context
+throw new CollectorError('Failed to fetch data', 'weather-collector', originalError)
+
+// Validation errors return 422
+throw new ValidationError('Invalid input', details)
+```
+
+All component errors are caught and logged with context (component name, stack trace). Non-critical operations use `safeAsync` to log errors without crashing:
+
+```typescript
+import { safeAsync } from 'digitaltwin-core'
+
+// Won't throw, just logs on failure
+await safeAsync(() => cleanup(), 'cleanup temporary files', logger)
+```
+
+## Production Features
+
+### Graceful Shutdown
+
+The engine supports graceful shutdown with configurable timeout:
+
+```typescript
+const engine = new DigitalTwinEngine({ database, storage })
+
+// Configure shutdown timeout (default: 30s)
+engine.setShutdownTimeout(60000)
+
+// Check if shutting down
+if (engine.isShuttingDown()) {
+  // Don't accept new work
+}
+
+// Graceful stop
+await engine.stop()
+```
+
+### Health Checks
+
+Register custom health checks for monitoring:
+
+```typescript
+engine.registerHealthCheck('external-api', async () => {
+  const response = await fetch('https://api.example.com/health')
+  return { status: response.ok ? 'up' : 'down' }
+})
+
+// Built-in checks: database, redis (if configured)
+const names = engine.getHealthCheckNames() // ['database', 'redis', 'external-api']
+
+// Remove check
+engine.removeHealthCheck('external-api')
+```
+
+### OpenAPI Specification
+
+Generate OpenAPI 3.0 specs from your components:
+
+```typescript
+import { OpenAPIGenerator } from 'digitaltwin-core'
+
+const spec = OpenAPIGenerator.generate({
+  info: { title: 'My API', version: '1.0.0' },
+  components: [collector, assetsManager, handler]
+})
+
+// Output as JSON or YAML
+const json = OpenAPIGenerator.toJSON(spec)
+const yaml = OpenAPIGenerator.toYAML(spec)
+```
+
+## Server Configuration
+
+### HTTP Compression
+
+HTTP compression is **disabled by default** because API gateways (Apache APISIX, Kong, Nginx, etc.) typically handle compression at the gateway level.
+
+For standalone deployments without a gateway, enable compression via environment variable:
+
+```bash
+export DIGITALTWIN_ENABLE_COMPRESSION=true
+```
+
+When enabled, the server uses gzip compression for JSON responses larger than 1KB, reducing bandwidth by 60-80%.
+
+## Authentication
+
+The framework supports multiple authentication modes:
+
+- **Gateway** (default): Uses headers from API gateways (Apache APISIX, KrakenD)
+- **JWT**: Direct JWT token validation
+- **None**: Disabled for development/testing
+
+### Gateway Mode (Default)
+
+No configuration needed. The framework reads `x-user-id` and `x-user-roles` headers set by your API gateway.
+
+### JWT Mode
+
+```bash
+export AUTH_MODE=jwt
+export JWT_SECRET=your-secret-key
+# Or for RSA: JWT_PUBLIC_KEY or JWT_PUBLIC_KEY_FILE
+```
+
+### Disable Authentication
+
+```bash
+export DIGITALTWIN_DISABLE_AUTH=true
+# Or
+export AUTH_MODE=none
+```
+
+For detailed configuration options, see [src/auth/README.md](src/auth/README.md).
+
 ## Project Scaffolding
 
 Use [create-digitaltwin](https://github.com/CePseudoBE/create-digitaltwin) to quickly bootstrap new projects:
@@ -348,11 +560,16 @@ node dt make:harvester DataProcessor --source weather-collector
 ## Folder structure
 
 - `src/` – framework sources
+    - `auth/` – authentication providers and user management
     - `components/` – base classes for collectors, harvesters, handlers and assets manager
-    - `engine/` – orchestration logic
+    - `engine/` – orchestration logic (includes component type guards)
+    - `loader/` – component auto-discovery utilities
     - `storage/` – storage service abstractions and adapters
     - `database/` – metadata database adapter
     - `env/` – environment configuration helper
+    - `errors/` – custom error classes
+    - `validation/` – request validation with VineJS
+    - `utils/` – utility functions (graceful shutdown, HTTP responses, etc.)
 - `tests/` – unit tests
 
 ---

package/dist/auth/apisix_parser.d.ts

@@ -1,16 +1,23 @@
 import type { AuthenticatedUser } from './types.js';
+import type { AuthProvider } from './auth_provider.js';
+/**
+ * Headers type that accepts both Express IncomingHttpHeaders and Record<string, string>
+ */
+export type HeadersLike = Record<string, string | string[] | undefined>;
 /**
  * Parses authentication information from Apache APISIX headers set after Keycloak authentication.
  *
- * This class handles the parsing of authentication headers forwarded by Apache APISIX
- * after successful Keycloak authentication. APISIX acts as a gateway that:
- * 1. Validates JWT tokens with Keycloak
- * 2. Extracts user information from the token
- * 3. Forwards user data as HTTP headers to downstream services
+ * This class provides a static API for backward compatibility while internally using
+ * the AuthProvider system. It automatically handles:
+ * - Gateway mode (x-user-id, x-user-roles headers)
+ * - JWT mode (Authorization: Bearer token)
+ * - No-auth mode (DIGITALTWIN_DISABLE_AUTH=true)
  *
- * Authentication can be disabled via environment variables for development/testing:
- * - Set DIGITALTWIN_DISABLE_AUTH=true to bypass authentication checks
- * - Set DIGITALTWIN_ANONYMOUS_USER_ID=custom-id to use a custom anonymous user ID
+ * For new code, consider using AuthProviderFactory directly:
+ * ```typescript
+ * const authProvider = AuthProviderFactory.fromEnv()
+ * const user = authProvider.parseRequest(req)
+ * ```
  *
  * @example
  * ```typescript
@@ -24,18 +31,39 @@ import type { AuthenticatedUser } from './types.js';
  * ```
  */
 export declare class ApisixAuthParser {
+    private static _provider;
+    /**
+     * Get the authentication provider instance.
+     * Creates it on first use based on environment configuration.
+     */
+    private static getProvider;
+    /**
+     * Reset the provider instance (useful for testing).
+     * @internal
+     */
+    static _resetProvider(): void;
+    /**
+     * Set a custom provider (useful for testing).
+     * @internal
+     */
+    static _setProvider(provider: AuthProvider): void;
+    /**
+     * Create a request-like object from headers for the AuthProvider.
+     * Normalizes headers by taking only the first value for array headers.
+     */
+    private static toAuthRequest;
     /**
-     * Extracts user information from APISIX headers.
+     * Extracts user information from authentication headers.
      *
-     * Parses the authentication headers forwarded by APISIX:
-     * - `x-user-id`: Keycloak user UUID (required)
-     * - `x-user-roles`: Comma-separated list of user roles (optional)
+     * Parses the authentication headers (gateway mode) or JWT token (jwt mode):
+     * - Gateway: `x-user-id` and `x-user-roles` headers
+     * - JWT: `Authorization: Bearer <token>` header
      *
      * When authentication is disabled (DIGITALTWIN_DISABLE_AUTH=true),
-     * returns a default anonymous user instead of requiring headers.
+     * returns a default anonymous user.
      *
-     * @param headers - HTTP request headers from APISIX
-     * @returns Parsed user authentication data, or null if x-user-id is missing and auth is enabled
+     * @param headers - HTTP request headers
+     * @returns Parsed user authentication data, or null if not authenticated
      *
      * @example
      * ```typescript
@@ -46,48 +74,35 @@ export declare class ApisixAuthParser {
      *
      * const authUser = ApisixAuthParser.parseAuthHeaders(headers)
      * // Returns: { id: '6e06a527...', roles: ['default-roles-master', 'offline_access'] }
-     *
-     * // With DIGITALTWIN_DISABLE_AUTH=true
-     * const authUser = ApisixAuthParser.parseAuthHeaders({})
-     * // Returns: { id: 'anonymous', roles: ['anonymous'] }
      * ```
      */
-    static parseAuthHeaders(headers: Record<string, string>): AuthenticatedUser | null;
+    static parseAuthHeaders(headers: HeadersLike): AuthenticatedUser | null;
     /**
-     * Checks if a request has valid authentication headers.
+     * Checks if a request has valid authentication.
      *
      * Performs a quick validation to determine if the request contains
-     * the minimum required authentication information (x-user-id header).
-     * Use this for early authentication checks before parsing.
+     * valid authentication credentials (gateway headers or JWT token).
      *
-     * When authentication is disabled (DIGITALTWIN_DISABLE_AUTH=true),
-     * this always returns true to allow all requests through.
+     * When authentication is disabled, this always returns true.
      *
      * @param headers - HTTP request headers
-     * @returns true if x-user-id header is present or auth is disabled, false otherwise
+     * @returns true if authentication is valid or disabled, false otherwise
      *
      * @example
      * ```typescript
-     * // Early authentication check in handler
      * if (!ApisixAuthParser.hasValidAuth(req.headers)) {
      *   return { status: 401, content: 'Authentication required' }
      * }
-     *
-     * // Now safe to proceed with parsing
-     * const authUser = ApisixAuthParser.parseAuthHeaders(req.headers)
      * ```
      */
-    static hasValidAuth(headers: Record<string, string>): boolean;
+    static hasValidAuth(headers: HeadersLike): boolean;
     /**
      * Extracts just the user ID from headers.
      *
-     * Convenience method for cases where you only need the user ID
-     * without parsing the full authentication context.
-     *
-     * When authentication is disabled, returns the configured anonymous user ID.
+     * Convenience method for cases where you only need the user ID.
      *
      * @param headers - HTTP request headers
-     * @returns Keycloak user ID, anonymous user ID if auth disabled, or null if not present
+     * @returns User ID, or null if not authenticated
      *
      * @example
      * ```typescript
@@ -97,17 +112,12 @@ export declare class ApisixAuthParser {
      * }
      * ```
      */
-    static getUserId(headers: Record<string, string>): string | null;
+    static getUserId(headers: HeadersLike): string | null;
     /**
      * Extracts just the user roles from headers.
      *
-     * Convenience method for cases where you only need the user roles
-     * without parsing the full authentication context.
-     *
-     * When authentication is disabled, returns the anonymous user roles.
-     *
      * @param headers - HTTP request headers
-     * @returns Array of role names, anonymous roles if auth disabled, empty array if no roles header present
+     * @returns Array of role names, empty array if not authenticated
      *
      * @example
      * ```typescript
@@ -117,30 +127,20 @@ export declare class ApisixAuthParser {
      * }
      * ```
      */
-    static getUserRoles(headers: Record<string, string>): string[];
+    static getUserRoles(headers: HeadersLike): string[];
     /**
      * Checks if a user has the admin role.
      *
-     * Determines if the authenticated user has administrative privileges by checking
-     * if their roles include the configured admin role name (default: "admin").
-     *
-     * The admin role name can be configured via DIGITALTWIN_ADMIN_ROLE_NAME environment variable.
-     *
      * @param headers - HTTP request headers
      * @returns true if user has admin role, false otherwise
      *
      * @example
      * ```typescript
      * if (ApisixAuthParser.isAdmin(req.headers)) {
-     *   // User has full administrative access
-     *   // Can view all assets including private assets owned by others
-     *   console.log('Admin user detected')
+     *   // Admin-only logic
      * }
-     *
-     * // With custom admin role name (DIGITALTWIN_ADMIN_ROLE_NAME=administrator)
-     * const isAdmin = ApisixAuthParser.isAdmin(req.headers)
      * ```
      */
-    static isAdmin(headers: Record<string, string>): boolean;
+    static isAdmin(headers: HeadersLike): boolean;
 }
 //# sourceMappingURL=apisix_parser.d.ts.map

package/dist/auth/apisix_parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"apisix_parser.d.ts","sourceRoot":"","sources":["../../src/auth/apisix_parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAGnD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBAAa,gBAAgB;IACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,iBAAiB,GAAG,IAAI;IAqBlF;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO;IAS7D;;;;;;;;;;;;;;;;;;OAkBG;IACH,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,GAAG,IAAI;IAShE;;;;;;;;;;;;;;;;;;OAkBG;IACH,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,EAAE;IAU9D;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO;CAK3D"}
+{"version":3,"file":"apisix_parser.d.ts","sourceRoot":"","sources":["../../src/auth/apisix_parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,KAAK,EAAE,YAAY,EAAe,MAAM,oBAAoB,CAAA;AAGnE;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAA;AAEvE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,qBAAa,gBAAgB;IACzB,OAAO,CAAC,MAAM,CAAC,SAAS,CAA4B;IAEpD;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,WAAW;IAO1B;;;OAGG;IACH,MAAM,CAAC,cAAc,IAAI,IAAI;IAI7B;;;OAGG;IACH,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI;IAIjD;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,aAAa;IAY5B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,WAAW,GAAG,iBAAiB,GAAG,IAAI;IAIvE;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO;IAIlD;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI;IAIrD;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,EAAE;IAInD;;;;;;;;;;;;OAYG;IACH,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO;CAGhD"}