digitaltwin-core 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Axel Hoffmann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,125 @@
+# Digital Twin Core
+
+Digital Twin Core is a minimalist TypeScript framework used to collect and process data for Digital Twin projects. It provides building blocks to create scheduled collectors, harvesters and HTTP handlers while abstracting storage and database access.
+
+## Features
+
+- **Collectors** – fetch regular data from APIs (typically JSON) based on a Buffer schedule, store it and expose it via GET endpoints.
+- **Harvesters** – transform data collected by collectors, store the results and expose them via GET endpoints.
+- **Handlers** – expose GET endpoints that directly return the result of the method defined in the decorator.
+- **Assets Manager** – upload, store and manage file assets with metadata, providing RESTful endpoints for CRUD operations.
+- **Storage adapters** – currently local filesystem and OVH Object Storage via S3 API.
+- **Database adapter** – implemented with [Knex](https://knexjs.org/) to index metadata.
+- **Engine** – orchestrates components, schedules jobs with BullMQ and exposes endpoints via Express.
+
+## Installation
+
+```bash
+npm install
+```
+
+The project requires Node.js 18 or later.
+
+## Building
+
+Compile the TypeScript sources to `dist/`:
+
+```bash
+npm run build
+```
+
+During development you can use the watcher:
+
+```bash
+npm run dev
+```
+
+## Running tests
+
+The test suite uses [Japa](https://github.com/japa/runner). Run all tests with:
+
+```bash
+npm test
+```
+
+## Example usage
+
+Below is a very small example showing how the engine may be instantiated. Storage and database implementations are selected through the provided factories.
+
+```ts
+import { DigitalTwinEngine } from './src/engine/digital_twin_engine.js';
+import { StorageServiceFactory } from './src/storage/storage_factory.js';
+import { KnexDatabaseAdapter } from './src/database/adapters/knex_database_adapter.js';
+import { Env } from './src/env/env.js';
+
+// Validate environment variables and bootstrap services
+const env = Env.validate({
+  STORAGE_CONFIG: Env.schema.enum(['local', 'ovh'])
+});
+
+const storage = StorageServiceFactory.create();
+const database = new KnexDatabaseAdapter({ client: 'sqlite3', connection: ':memory:' }, storage);
+
+const engine = new DigitalTwinEngine({ storage, database });
+engine.start();
+```
+
+## Components
+
+### Collectors
+
+Collectors are scheduled components that fetch data from external sources at regular intervals. They implement a `collect()` method that returns a Buffer, which is then stored and exposed via HTTP endpoints.
+
+**Key features:**
+- Cron-based scheduling
+- Automatic storage and metadata indexing
+- HTTP GET endpoint for retrieving latest data
+- Event emission on successful collection
+
+### Assets Manager
+
+The Assets Manager provides a complete solution for file asset management with metadata support. It's an abstract base class that can be extended for specific asset types.
+
+**Key features:**
+- File upload with metadata (description, source URL, owner, filename)
+- RESTful CRUD operations via HTTP endpoints
+- Content-type aware storage and retrieval
+- Separate display and download endpoints
+- Source URL validation for data provenance
+- Component isolation (each manager handles its own asset type)
+
+**Available endpoints:**
+- `GET /{assetType}` - List all assets with metadata
+- `POST /{assetType}/upload` - Upload new asset with metadata
+- `GET /{assetType}/{id}` - Retrieve asset content for display
+- `GET /{assetType}/{id}/download` - Download asset with attachment headers
+- `PUT /{assetType}/{id}` - Update asset metadata
+- `DELETE /{assetType}/{id}` - Delete asset
+
+**Example usage:**
+```typescript
+class GLTFAssetsManager extends AssetsManager {
+  getConfiguration() {
+    return {
+      name: 'gltf',
+      description: 'GLTF 3D models manager',
+      contentType: 'model/gltf-binary',
+      tags: ['assets', '3d', 'gltf']
+    }
+  }
+}
+```
+
+## Folder structure
+
+- `src/` – framework sources
+    - `components/` – base classes for collectors, harvesters, handlers and assets manager
+    - `engine/` – orchestration logic
+    - `storage/` – storage service abstractions and adapters
+    - `database/` – metadata database adapter
+    - `env/` – environment configuration helper
+- `tests/` – unit tests
+
+---
+
+This project is licensed under the MIT License.

package/dist/components/assets_manager.d.ts

@@ -0,0 +1,384 @@
+import { Component, Servable } from './interfaces.js';
+import { ComponentConfiguration, DataResponse } from './types.js';
+import { HttpMethod } from '../engine/endpoints.js';
+import { StorageService } from '../storage/storage_service.js';
+import { DatabaseAdapter, MetadataRow } from '../database/database_adapter.js';
+import { DataRecord } from "../types/data_record.js";
+/**
+ * Extended metadata row for assets with additional fields.
+ * This will be stored as separate columns in the database table.
+ *
+ * @interface AssetMetadataRow
+ * @extends {MetadataRow}
+ *
+ * @example
+ * ```typescript
+ * const assetMeta: AssetMetadataRow = {
+ *   name: 'gltf',
+ *   type: 'model/gltf-binary',
+ *   url: '/storage/gltf/model.glb',
+ *   date: new Date(),
+ *   description: '3D building model',
+ *   source: 'https://example.com/data-source',
+ *   owner_id: 'user123',
+ *   filename: 'building.glb'
+ * }
+ * ```
+ */
+export interface AssetMetadataRow extends MetadataRow {
+    /** Human-readable description of the asset */
+    description: string;
+    /** Source URL for data provenance and licensing compliance (must be valid URL) */
+    source: string;
+    /** ID of the user who owns this asset (for access control) */
+    owner_id: string | null;
+    /** Original filename provided by the user */
+    filename: string;
+}
+/**
+ * Request payload for creating a new asset.
+ *
+ * @interface CreateAssetRequest
+ *
+ * @example
+ * ```typescript
+ * const request: CreateAssetRequest = {
+ *   description: '3D model of building',
+ *   source: 'https://city-data.example.com/buildings',
+ *   owner_id: 'user123',
+ *   filename: 'building.glb',
+ *   file: fileBuffer
+ * }
+ * ```
+ */
+export interface CreateAssetRequest {
+    /** Human-readable description of the asset */
+    description: string;
+    /** Source URL for data provenance (validated as proper URL) */
+    source: string;
+    /** Owner user ID for access control (can be null) */
+    owner_id: string | null;
+    /** Original filename */
+    filename: string;
+    /** File content as Buffer */
+    file: Buffer;
+}
+/**
+ * Request payload for updating asset metadata.
+ *
+ * @interface UpdateAssetRequest
+ *
+ * @example
+ * ```typescript
+ * const updates: UpdateAssetRequest = {
+ *   description: 'Updated building model with new textures',
+ *   source: 'https://updated-source.example.com'
+ * }
+ * ```
+ */
+export interface UpdateAssetRequest {
+    /** Updated description (optional) */
+    description?: string;
+    /** Updated source URL (optional, validated if provided) */
+    source?: string;
+}
+/**
+ * Abstract base class for Assets Manager components.
+ *
+ * Provides file upload, storage, and retrieval capabilities following the Digital Twin framework patterns.
+ * Each concrete implementation manages a specific type of asset and creates its own database table.
+ *
+ * @abstract
+ * @class AssetsManager
+ * @implements {Component}
+ * @implements {Servable}
+ *
+ * @example
+ * ```typescript
+ * // Create concrete implementations for different asset types
+ * class GLTFAssetsManager extends AssetsManager {
+ *   getConfiguration() {
+ *     return { name: 'gltf', description: 'GLTF 3D models manager', ... }
+ *   }
+ * }
+ *
+ * class PointCloudAssetsManager extends AssetsManager {
+ *   getConfiguration() {
+ *     return { name: 'pointcloud', description: 'Point cloud data manager', ... }
+ *   }
+ * }
+ *
+ * // Usage in engine
+ * const gltfManager = new GLTFAssetsManager()
+ * gltfManager.setDependencies(database, storage)
+ *
+ * // Each creates its own table and endpoints:
+ * // - GLTFAssetsManager → table 'gltf', endpoints /gltf/*
+ * // - PointCloudAssetsManager → table 'pointcloud', endpoints /pointcloud/*
+ * ```
+ *
+ * @remarks
+ * Asset metadata is stored as dedicated columns in the database table:
+ * - id, name, url, date (standard columns)
+ * - description, source, owner_id, filename (asset-specific columns)
+ *
+ * Each concrete AssetsManager creates its own table based on the configuration name.
+ */
+export declare abstract class AssetsManager implements Component, Servable {
+    protected db: DatabaseAdapter;
+    protected storage: StorageService;
+    /**
+     * Injects dependencies into the assets manager.
+     *
+     * Called by the framework during component initialization.
+     *
+     * @param {DatabaseAdapter} db - The database adapter for metadata storage
+     * @param {StorageService} storage - The storage service for file persistence
+     *
+     * @example
+     * ```typescript
+     * const assetsManager = new MyAssetsManager()
+     * assetsManager.setDependencies(databaseAdapter, storageService)
+     * ```
+     */
+    setDependencies(db: DatabaseAdapter, storage: StorageService): void;
+    /**
+     * Returns the configuration of the assets manager.
+     *
+     * Must be implemented by subclasses to define the asset type,
+     * table name, and content types.
+     *
+     * @abstract
+     * @returns {ComponentConfiguration} The component configuration
+     *
+     * @example
+     * ```typescript
+     * class GLTFAssetsManager extends AssetsManager {
+     *   getConfiguration(): ComponentConfiguration {
+     *     return {
+     *       name: 'gltf',
+     *       description: 'GLTF 3D models manager',
+     *       contentType: 'model/gltf-binary',
+     *       tags: ['assets', '3d', 'gltf']
+     *     }
+     *   }
+     * }
+     * ```
+     */
+    abstract getConfiguration(): ComponentConfiguration;
+    /**
+     * Validates that a source string is a valid URL.
+     *
+     * Used internally to ensure data provenance URLs are properly formatted.
+     *
+     * @private
+     * @param {string} source - The source URL to validate
+     * @returns {boolean} True if the source is a valid URL, false otherwise
+     *
+     * @example
+     * ```typescript
+     * this.validateSourceURL('https://example.com/data') // returns true
+     * this.validateSourceURL('not-a-url') // returns false
+     * ```
+     */
+    private validateSourceURL;
+    /**
+     * Upload a new asset file with metadata.
+     *
+     * Stores the file using the storage service and saves metadata to the database.
+     * Asset metadata is stored as dedicated columns in the database table.
+     *
+     * @param {CreateAssetRequest} request - The asset upload request
+     * @throws {Error} If source URL is invalid
+     *
+     * @example
+     * ```typescript
+     * await assetsManager.uploadAsset({
+     *   description: '3D building model',
+     *   source: 'https://city-data.example.com/buildings',
+     *   owner_id: 'user123',
+     *   filename: 'building.glb',
+     *   file: fileBuffer
+     * })
+     * ```
+     */
+    uploadAsset(request: CreateAssetRequest): Promise<void>;
+    /**
+     * Retrieve all assets for this component (like other components).
+     *
+     * Returns a JSON list of all assets with their metadata, following the
+     * framework pattern but adapted for assets management.
+     *
+     * @returns {Promise<DataResponse>} JSON response with all assets
+     */
+    retrieve(): Promise<DataResponse>;
+    /**
+     * Get all assets for this component type.
+     *
+     * Retrieves all assets managed by this component, with their metadata.
+     * Uses a very old start date to get all records.
+     *
+     * @returns {Promise<DataRecord[]>} Array of all asset records
+     *
+     * @example
+     * ```typescript
+     * const allAssets = await assetsManager.getAllAssets()
+     * // Returns: [{ id, name, type, url, date, contentType }, ...]
+     * ```
+     */
+    getAllAssets(): Promise<DataRecord[]>;
+    /**
+     * Get asset by specific ID.
+     *
+     * @param {string} id - The asset ID to retrieve
+     * @returns {Promise<DataRecord | undefined>} The asset record or undefined if not found
+     *
+     * @example
+     * ```typescript
+     * const asset = await assetsManager.getAssetById('123')
+     * if (asset) {
+     *   const fileData = await asset.data()
+     * }
+     * ```
+     */
+    getAssetById(id: string): Promise<DataRecord | undefined>;
+    /**
+     * Update asset metadata by ID.
+     *
+     * Updates the metadata (description and/or source) of a specific asset.
+     * Asset metadata is stored as dedicated columns in the database.
+     *
+     * @param {string} id - The ID of the asset to update
+     * @param {UpdateAssetRequest} updates - The metadata updates to apply
+     * @throws {Error} If source URL is invalid or asset not found
+     *
+     * @example
+     * ```typescript
+     * await assetsManager.updateAssetMetadata('123', {
+     *   description: 'Updated building model with new textures',
+     *   source: 'https://updated-source.example.com'
+     * })
+     * ```
+     */
+    updateAssetMetadata(id: string, updates: UpdateAssetRequest): Promise<void>;
+    /**
+     * Delete asset by ID.
+     *
+     * Removes a specific asset.
+     *
+     * @param {string} id - The ID of the asset to delete
+     * @throws {Error} If asset not found or doesn't belong to this component
+     *
+     * @example
+     * ```typescript
+     * await assetsManager.deleteAssetById('123')
+     * ```
+     */
+    deleteAssetById(id: string): Promise<void>;
+    /**
+     * Delete latest asset (simplified)
+     *
+     * Removes the most recently uploaded asset for this component type.
+     *
+     * @throws {Error} If no assets exist to delete
+     *
+     * @example
+     * ```typescript
+     * await assetsManager.deleteLatestAsset()
+     * ```
+     */
+    deleteLatestAsset(): Promise<void>;
+    /**
+     * Get endpoints following the framework pattern
+     */
+    /**
+     * Get HTTP endpoints exposed by this assets manager.
+     *
+     * Returns the standard CRUD endpoints following the framework pattern.
+     *
+     * @returns {Array} Array of endpoint descriptors with methods, paths, and handlers
+     *
+     * @example
+     * ```typescript
+     * // For a manager with assetType: 'gltf', provides:
+     * GET    /gltf         - Get all assets
+     * POST   /gltf/upload  - Upload new asset
+     * GET    /gltf/123     - Get specific asset
+     * PUT    /gltf/123     - Update asset metadata
+     * GET    /gltf/123/download - Download asset
+     * DELETE /gltf/123     - Delete asset
+     * ```
+     */
+    getEndpoints(): Array<{
+        method: HttpMethod;
+        path: string;
+        handler: (...args: any[]) => any;
+        responseType?: string;
+    }>;
+    /**
+     * Handle upload endpoint
+     */
+    handleUpload(req: any): Promise<DataResponse>;
+    /**
+     * Handle update endpoint (PUT).
+     *
+     * Updates metadata for a specific asset by ID.
+     *
+     * @param {any} req - HTTP request object with params.id and body containing updates
+     * @returns {Promise<DataResponse>} HTTP response
+     *
+     * @example
+     * ```typescript
+     * // PUT /gltf/123
+     * // Body: { "description": "Updated model", "source": "https://new-source.com" }
+     * ```
+     */
+    handleUpdate(req: any): Promise<DataResponse>;
+    /**
+     * Handle get asset endpoint (GET).
+     *
+     * Returns the file content of a specific asset by ID for display/use in front-end.
+     * No download headers - just the raw file content.
+     *
+     * @param {any} req - HTTP request object with params.id
+     * @returns {Promise<DataResponse>} HTTP response with file content
+     *
+     * @example
+     * ```typescript
+     * // GET /gltf/123
+     * // Returns the .glb file content for display in 3D viewer
+     * ```
+     */
+    handleGetAsset(req: any): Promise<DataResponse>;
+    /**
+     * Handle download endpoint (GET).
+     *
+     * Downloads the file content of a specific asset by ID with download headers.
+     * Forces browser to download the file rather than display it.
+     *
+     * @param {any} req - HTTP request object with params.id
+     * @returns {Promise<DataResponse>} HTTP response with file content and download headers
+     *
+     * @example
+     * ```typescript
+     * // GET /gltf/123/download
+     * // Returns the .glb file with download headers - browser will save it
+     * ```
+     */
+    handleDownload(req: any): Promise<DataResponse>;
+    /**
+     * Handle delete endpoint (DELETE).
+     *
+     * Deletes a specific asset by ID.
+     *
+     * @param {any} req - HTTP request object with params.id
+     * @returns {Promise<DataResponse>} HTTP response
+     *
+     * @example
+     * ```typescript
+     * // DELETE /gltf/123
+     * ```
+     */
+    handleDelete(req: any): Promise<DataResponse>;
+}
+//# sourceMappingURL=assets_manager.d.ts.map

package/dist/components/assets_manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assets_manager.d.ts","sourceRoot":"","sources":["../../src/components/assets_manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,EAAE,sBAAsB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAA;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAA;AAC9E,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAC;AAsBnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACjD,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAA;IACnB,kFAAkF;IAClF,MAAM,EAAE,MAAM,CAAA;IACd,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAA;CACnB;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,kBAAkB;IAC/B,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAA;IACnB,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAA;IACd,qDAAqD;IACrD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAA;IAChB,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,kBAAkB;IAC/B,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,8BAAsB,aAAc,YAAW,SAAS,EAAE,QAAQ;IAC9D,SAAS,CAAC,EAAE,EAAG,eAAe,CAAA;IAC9B,SAAS,CAAC,OAAO,EAAG,cAAc,CAAA;IAElC;;;;;;;;;;;;;OAaG;IACH,eAAe,CAAC,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI;IAKnE;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,QAAQ,CAAC,gBAAgB,IAAI,sBAAsB;IAEnD;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,iBAAiB;IASzB;;;;;;;;;;;;;;;;;;;OAmBG;IACG,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA0B7D;;;;;;;OAOG;IACG,QAAQ,IAAI,OAAO,CAAC,YAAY,CAAC;IAyCvC;;;;;;;;;;;;;OAaG;IACG,YAAY,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;IAQ3C;;;;;;;;;;;;;OAaG;IACG,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAI/D;;;;;;;;;;;;;;;;;OAiBG;IACG,mBAAmB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmCjF;;;;;;;;;;;;OAYG;IACG,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBhD;;;;;;;;;;;OAWG;IACG,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAUxC;;OAEG;IACH;;;;;;;;;;;;;;;;;OAiBG;IACH,YAAY,IAAI,KAAK,CAAC;QAClB,MAAM,EAAE,UAAU,CAAA;QAClB,IAAI,EAAE,MAAM,CAAA;QACZ,OAAO,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAA;QAChC,YAAY,CAAC,EAAE,MAAM,CAAA;KACxB,CAAC;IA0CF;;OAEG;IACG,YAAY,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC;IAuCnD;;;;;;;;;;;;;OAaG;IACG,YAAY,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC;IA+CnD;;;;;;;;;;;;;;OAcG;IACG,cAAc,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC;IAwDrD;;;;;;;;;;;;;;OAcG;IACG,cAAc,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC;IA2DrD;;;;;;;;;;;;OAYG;IACG,YAAY,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,YAAY,CAAC;CA8BtD"}