digitaltwin-core 0.14.2 → 1.0.0

package/dist/storage/adapters/ovh_storage_service.js

@@ -0,0 +1,205 @@
+/**
+ * OVH Object Storage implementation of StorageService
+ * via S3-compatible API using @aws-sdk/client-s3
+ */
+import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, ListObjectsV2Command, PutBucketCorsCommand, ObjectCannedACL } from '@aws-sdk/client-s3';
+import { StorageService } from '../storage_service.js';
+import { safeAsync } from '../../utils/safe_async.js';
+import { Logger } from '../../utils/logger.js';
+const logger = new Logger('OvhS3Storage');
+export class OvhS3StorageService extends StorageService {
+    #s3;
+    #bucket;
+    #endpoint;
+    constructor(config) {
+        super();
+        this.#bucket = config.bucket;
+        this.#endpoint = config.endpoint;
+        this.#s3 = new S3Client({
+            endpoint: config.endpoint,
+            region: config.region ?? 'gra',
+            credentials: {
+                accessKeyId: config.accessKey,
+                secretAccessKey: config.secretKey
+            },
+            forcePathStyle: false,
+            // Match Python boto3 config for OVH compatibility
+            requestChecksumCalculation: 'WHEN_REQUIRED',
+            responseChecksumValidation: 'WHEN_REQUIRED'
+        });
+    }
+    /**
+     * Uploads a file to the OVH S3-compatible object storage.
+     * @param buffer - File contents to upload
+     * @param collectorName - Folder/prefix to store under
+     * @param extension - Optional file extension (e.g. 'json')
+     * @returns The relative path (key) of the stored object
+     */
+    async save(buffer, collectorName, extension) {
+        const now = new Date();
+        const timestamp = now.toISOString().replace(/[:.]/g, '-');
+        const key = `${collectorName || 'default'}/${timestamp}${extension ? '.' + extension : ''}`;
+        await this.#s3.send(new PutObjectCommand({
+            Bucket: this.#bucket,
+            Key: key,
+            Body: buffer,
+            ACL: ObjectCannedACL.private
+        }));
+        return key;
+    }
+    /**
+     * Downloads and returns a stored object as a Buffer.
+     * @param relativePath - The key/path of the object to retrieve
+     * @returns The object contents as a Buffer
+     */
+    async retrieve(relativePath) {
+        const res = await this.#s3.send(new GetObjectCommand({
+            Bucket: this.#bucket,
+            Key: relativePath
+        }));
+        const chunks = [];
+        const stream = res.Body;
+        for await (const chunk of stream) {
+            chunks.push(Buffer.from(chunk));
+        }
+        return Buffer.concat(chunks);
+    }
+    /**
+     * Deletes an object from the storage bucket.
+     * @param relativePath - The key/path of the object to delete
+     */
+    async delete(relativePath) {
+        await this.#s3.send(new DeleteObjectCommand({
+            Bucket: this.#bucket,
+            Key: relativePath
+        }));
+    }
+    /**
+     * Uploads a file to OVH S3 at a specific path (preserves filename).
+     * Unlike save(), this method does not auto-generate a timestamp filename.
+     * Files are uploaded with public-read ACL for direct access (e.g., Cesium tilesets).
+     * @param buffer - File contents to upload
+     * @param relativePath - Full relative path including filename (e.g., 'tilesets/123/tileset.json')
+     * @returns The same relative path that was provided
+     */
+    async saveWithPath(buffer, relativePath) {
+        await this.#s3.send(new PutObjectCommand({
+            Bucket: this.#bucket,
+            Key: relativePath,
+            Body: buffer,
+            ACL: ObjectCannedACL.public_read
+        }));
+        return relativePath;
+    }
+    /**
+     * Deletes multiple objects in batch using S3 DeleteObjects API.
+     * Much faster than individual deletes - can delete up to 1000 objects per request.
+     * @param paths - Array of object keys to delete
+     */
+    async deleteBatch(paths) {
+        if (paths.length === 0)
+            return;
+        // S3 DeleteObjects supports max 1000 objects per request
+        const BATCH_SIZE = 1000;
+        const batches = [];
+        for (let i = 0; i < paths.length; i += BATCH_SIZE) {
+            batches.push(paths.slice(i, i + BATCH_SIZE));
+        }
+        // Process batches in parallel (but limit concurrency to avoid overwhelming the API)
+        const MAX_CONCURRENT = 5;
+        for (let i = 0; i < batches.length; i += MAX_CONCURRENT) {
+            const concurrentBatches = batches.slice(i, i + MAX_CONCURRENT);
+            await Promise.all(concurrentBatches.map((batch, index) => safeAsync(() => this.#s3.send(new DeleteObjectsCommand({
+                Bucket: this.#bucket,
+                Delete: {
+                    Objects: batch.map(key => ({ Key: key })),
+                    Quiet: true // Don't return info about each deleted object
+                }
+            })), `delete batch ${i + index + 1}/${batches.length}`, logger)));
+        }
+    }
+    /**
+     * Returns the public URL for a stored file.
+     * Constructs the OVH S3 public URL format: https://{bucket}.{endpoint_host}/{key}
+     * @param relativePath - The storage path/key of the file
+     * @returns The public URL to access the file directly
+     */
+    getPublicUrl(relativePath) {
+        // Extract host from endpoint (e.g., 'https://s3.gra.io.cloud.ovh.net' -> 's3.gra.io.cloud.ovh.net')
+        const endpointHost = this.#endpoint.replace(/^https?:\/\//, '');
+        // OVH S3 URL format: https://{bucket}.{endpoint_host}/{key}
+        return `https://${this.#bucket}.${endpointHost}/${relativePath}`;
+    }
+    /**
+     * Deletes all objects under a given prefix (folder).
+     * Lists objects by prefix and deletes them in batches for performance.
+     * @param prefix - The folder/prefix to delete (e.g., 'tilesets/123')
+     * @returns Number of files deleted
+     */
+    async deleteByPrefix(prefix) {
+        let totalDeleted = 0;
+        let continuationToken;
+        // Ensure prefix ends with '/' to avoid partial matches
+        const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/`;
+        do {
+            // List objects with prefix (max 1000 per request)
+            const listResponse = await this.#s3.send(new ListObjectsV2Command({
+                Bucket: this.#bucket,
+                Prefix: normalizedPrefix,
+                ContinuationToken: continuationToken
+            }));
+            const objects = listResponse.Contents || [];
+            if (objects.length === 0)
+                break;
+            // Delete objects in batch
+            const keys = objects.map(obj => obj.Key).filter((key) => !!key);
+            if (keys.length > 0) {
+                await this.#s3.send(new DeleteObjectsCommand({
+                    Bucket: this.#bucket,
+                    Delete: {
+                        Objects: keys.map(key => ({ Key: key })),
+                        Quiet: true
+                    }
+                }));
+                totalDeleted += keys.length;
+            }
+            continuationToken = listResponse.NextContinuationToken;
+        } while (continuationToken);
+        return totalDeleted;
+    }
+    /**
+     * Configure CORS settings for the bucket.
+     * Required for browser-based access to public files (e.g., Cesium loading tilesets).
+     * Should be called once during application startup.
+     *
+     * @param allowedOrigins - List of allowed origins (default: ['*'])
+     * @param allowedMethods - List of allowed HTTP methods (default: ['GET', 'HEAD'])
+     * @param allowedHeaders - List of allowed headers (default: ['*', 'Authorization'])
+     * @returns true if successful, false otherwise
+     */
+    async configureCors(allowedOrigins = ['*'], allowedMethods = ['GET', 'HEAD'], allowedHeaders = ['*', 'Authorization']) {
+        try {
+            await this.#s3.send(new PutBucketCorsCommand({
+                Bucket: this.#bucket,
+                CORSConfiguration: {
+                    CORSRules: [
+                        {
+                            AllowedOrigins: allowedOrigins,
+                            AllowedMethods: allowedMethods,
+                            AllowedHeaders: allowedHeaders,
+                            ExposeHeaders: ['ETag', 'Content-Length'],
+                            MaxAgeSeconds: 3000
+                        }
+                    ]
+                }
+            }));
+            console.log('[OvhS3StorageService] CORS configured successfully');
+            return true;
+        }
+        catch (error) {
+            console.error('[OvhS3StorageService] Error configuring CORS:', error);
+            return false;
+        }
+    }
+}
+//# sourceMappingURL=ovh_storage_service.js.map

package/dist/storage/adapters/ovh_storage_service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ovh_storage_service.js","sourceRoot":"","sources":["../../../src/storage/adapters/ovh_storage_service.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EACH,QAAQ,EACR,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,oBAAoB,EACpB,oBAAoB,EACpB,oBAAoB,EACpB,eAAe,EAClB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAA;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAG9C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,cAAc,CAAC,CAAA;AAUzC,MAAM,OAAO,mBAAoB,SAAQ,cAAc;IACnD,GAAG,CAAU;IACJ,OAAO,CAAQ;IACf,SAAS,CAAQ;IAE1B,YAAY,MAAmB;QAC3B,KAAK,EAAE,CAAA;QACP,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAA;QAC5B,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAA;QAChC,IAAI,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC;YACpB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,KAAK;YAC9B,WAAW,EAAE;gBACT,WAAW,EAAE,MAAM,CAAC,SAAS;gBAC7B,eAAe,EAAE,MAAM,CAAC,SAAS;aACpC;YACD,cAAc,EAAE,KAAK;YACrB,kDAAkD;YAClD,0BAA0B,EAAE,eAAe;YAC3C,0BAA0B,EAAE,eAAe;SAC9C,CAAC,CAAA;IACN,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,CAAC,MAAc,EAAE,aAAqB,EAAE,SAAkB;QAChE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAA;QACtB,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;QACzD,MAAM,GAAG,GAAG,GAAG,aAAa,IAAI,SAAS,IAAI,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;QAE3F,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CACf,IAAI,gBAAgB,CAAC;YACjB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,GAAG,EAAE,GAAG;YACR,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,eAAe,CAAC,OAAO;SAC/B,CAAC,CACL,CAAA;QAED,OAAO,GAAG,CAAA;IACd,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CAAC,YAAoB;QAC/B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAC3B,IAAI,gBAAgB,CAAC;YACjB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,GAAG,EAAE,YAAY;SACpB,CAAC,CACL,CAAA;QAED,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,IAAgB,CAAA;QAEnC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QACnC,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAChC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,YAAoB;QAC7B,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CACf,IAAI,mBAAmB,CAAC;YACpB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,GAAG,EAAE,YAAY;SACpB,CAAC,CACL,CAAA;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,YAAoB;QACnD,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CACf,IAAI,gBAAgB,CAAC;YACjB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,GAAG,EAAE,YAAY;YACjB,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,eAAe,CAAC,WAAW;SACnC,CAAC,CACL,CAAA;QAED,OAAO,YAAY,CAAA;IACvB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,KAAe;QAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE9B,yDAAyD;QACzD,MAAM,UAAU,GAAG,IAAI,CAAA;QACvB,MAAM,OAAO,GAAe,EAAE,CAAA;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,CAAA;QAChD,CAAC;QAED,oFAAoF;QACpF,MAAM,cAAc,GAAG,CAAC,CAAA;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,iBAAiB,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAA;YAC9D,MAAM,OAAO,CAAC,GAAG,CACb,iBAAiB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CACnC,SAAS,CACL,GAAG,EAAE,CACD,IAAI,CAAC,GAAG,CAAC,IAAI,CACT,IAAI,oBAAoB,CAAC;gBACrB,MAAM,EAAE,IAAI,CAAC,OAAO;gBACpB,MAAM,EAAE;oBACJ,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;oBACzC,KAAK,EAAE,IAAI,CAAC,8CAA8C;iBAC7D;aACJ,CAAC,CACL,EACL,gBAAgB,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,EACjD,MAAM,CACT,CACJ,CACJ,CAAA;QACL,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,YAAoB;QAC7B,oGAAoG;QACpG,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;QAC/D,4DAA4D;QAC5D,OAAO,WAAW,IAAI,CAAC,OAAO,IAAI,YAAY,IAAI,YAAY,EAAE,CAAA;IACpE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAAC,MAAc;QAC/B,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,iBAAqC,CAAA;QAEzC,uDAAuD;QACvD,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,CAAA;QAErE,GAAG,CAAC;YACA,kDAAkD;YAClD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CACpC,IAAI,oBAAoB,CAAC;gBACrB,MAAM,EAAE,IAAI,CAAC,OAAO;gBACpB,MAAM,EAAE,gBAAgB;gBACxB,iBAAiB,EAAE,iBAAiB;aACvC,CAAC,CACL,CAAA;YAED,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,IAAI,EAAE,CAAA;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAK;YAE/B,0BAA0B;YAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAiB,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YAE9E,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CACf,IAAI,oBAAoB,CAAC;oBACrB,MAAM,EAAE,IAAI,CAAC,OAAO;oBACpB,MAAM,EAAE;wBACJ,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;wBACxC,KAAK,EAAE,IAAI;qBACd;iBACJ,CAAC,CACL,CAAA;gBACD,YAAY,IAAI,IAAI,CAAC,MAAM,CAAA;YAC/B,CAAC;YAED,iBAAiB,GAAG,YAAY,CAAC,qBAAqB,CAAA;QAC1D,CAAC,QAAQ,iBAAiB,EAAC;QAE3B,OAAO,YAAY,CAAA;IACvB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,aAAa,CACf,iBAA2B,CAAC,GAAG,CAAC,EAChC,iBAA2B,CAAC,KAAK,EAAE,MAAM,CAAC,EAC1C,iBAA2B,CAAC,GAAG,EAAE,eAAe,CAAC;QAEjD,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CACf,IAAI,oBAAoB,CAAC;gBACrB,MAAM,EAAE,IAAI,CAAC,OAAO;gBACpB,iBAAiB,EAAE;oBACf,SAAS,EAAE;wBACP;4BACI,cAAc,EAAE,cAAc;4BAC9B,cAAc,EAAE,cAAc;4BAC9B,cAAc,EAAE,cAAc;4BAC9B,aAAa,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC;4BACzC,aAAa,EAAE,IAAI;yBACtB;qBACJ;iBACJ;aACJ,CAAC,CACL,CAAA;YACD,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAA;YACjE,OAAO,IAAI,CAAA;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,KAAK,CAAC,CAAA;YACrE,OAAO,KAAK,CAAA;QAChB,CAAC;IACL,CAAC;CACJ"}

package/dist/storage/storage_factory.d.ts

@@ -0,0 +1,14 @@
+import type { StorageService } from './storage_service.js';
+export declare class StorageServiceFactory {
+    /**
+     * Creates and returns an instance of StorageService
+     * based on the STORAGE_CONFIG environment variable.
+     *
+     * - 'local': returns a LocalStorageService
+     * - 'ovh': returns an OvhS3StorageService
+     *
+     * @throws Error if STORAGE_CONFIG is not supported
+     */
+    static create(): StorageService;
+}
+//# sourceMappingURL=storage_factory.d.ts.map

package/dist/storage/storage_factory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage_factory.d.ts","sourceRoot":"","sources":["../../src/storage/storage_factory.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AAM1D,qBAAa,qBAAqB;IAC9B;;;;;;;;OAQG;IACH,MAAM,CAAC,MAAM,IAAI,cAAc;CAwBlC"}

package/dist/storage/storage_factory.js

@@ -0,0 +1,43 @@
+/**
+ * Factory class for creating the appropriate StorageService
+ * implementation based on environment configuration.
+ */
+import { Env } from '../env/env.js';
+import { OvhS3StorageService } from './adapters/ovh_storage_service.js';
+import { LocalStorageService } from './adapters/local_storage_service.js';
+import { safeAsync } from '../utils/safe_async.js';
+import { Logger } from '../utils/logger.js';
+const logger = new Logger('StorageFactory');
+export class StorageServiceFactory {
+    /**
+     * Creates and returns an instance of StorageService
+     * based on the STORAGE_CONFIG environment variable.
+     *
+     * - 'local': returns a LocalStorageService
+     * - 'ovh': returns an OvhS3StorageService
+     *
+     * @throws Error if STORAGE_CONFIG is not supported
+     */
+    static create() {
+        const env = Env.config;
+        switch (env.STORAGE_CONFIG) {
+            case 'local':
+                return new LocalStorageService(env.LOCAL_STORAGE_DIR || 'data');
+            case 'ovh': {
+                const ovhStorage = new OvhS3StorageService({
+                    accessKey: env.OVH_ACCESS_KEY,
+                    secretKey: env.OVH_SECRET_KEY,
+                    endpoint: env.OVH_ENDPOINT,
+                    bucket: env.OVH_BUCKET,
+                    region: env.OVH_REGION ?? 'gra'
+                });
+                // Configure CORS for browser access (non-blocking)
+                safeAsync(() => ovhStorage.configureCors(), 'configure OVH CORS', logger);
+                return ovhStorage;
+            }
+            default:
+                throw new Error(`Unsupported STORAGE_CONFIG: ${env.STORAGE_CONFIG}`);
+        }
+    }
+}
+//# sourceMappingURL=storage_factory.js.map

package/dist/storage/storage_factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage_factory.js","sourceRoot":"","sources":["../../src/storage/storage_factory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,OAAO,EAAE,GAAG,EAAE,MAAM,eAAe,CAAA;AACnC,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAA;AACvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAA;AAEzE,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAA;AAE3C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAAA;AAE3C,MAAM,OAAO,qBAAqB;IAC9B;;;;;;;;OAQG;IACH,MAAM,CAAC,MAAM;QACT,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAA;QAEtB,QAAQ,GAAG,CAAC,cAAc,EAAE,CAAC;YACzB,KAAK,OAAO;gBACR,OAAO,IAAI,mBAAmB,CAAC,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC,CAAA;YAEnE,KAAK,KAAK,CAAC,CAAC,CAAC;gBACT,MAAM,UAAU,GAAG,IAAI,mBAAmB,CAAC;oBACvC,SAAS,EAAE,GAAG,CAAC,cAAc;oBAC7B,SAAS,EAAE,GAAG,CAAC,cAAc;oBAC7B,QAAQ,EAAE,GAAG,CAAC,YAAY;oBAC1B,MAAM,EAAE,GAAG,CAAC,UAAU;oBACtB,MAAM,EAAE,GAAG,CAAC,UAAU,IAAI,KAAK;iBAClC,CAAC,CAAA;gBACF,mDAAmD;gBACnD,SAAS,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,EAAE,EAAE,oBAAoB,EAAE,MAAM,CAAC,CAAA;gBACzE,OAAO,UAAU,CAAA;YACrB,CAAC;YAED;gBACI,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,CAAC,cAAc,EAAE,CAAC,CAAA;QAC5E,CAAC;IACL,CAAC;CACJ"}

package/dist/storage/storage_service.d.ts

@@ -0,0 +1,163 @@
+/**
+ * Abstract base class for storage service implementations.
+ *
+ * Defines the contract for persisting and retrieving binary data in the Digital Twin framework.
+ * Concrete implementations provide storage backends like local filesystem, AWS S3, Azure Blob, etc.
+ *
+ * @abstract
+ * @class StorageService
+ *
+ * @example
+ * ```typescript
+ * // Implement for specific storage backend
+ * class S3StorageService extends StorageService {
+ *   async save(buffer: Buffer, collectorName: string, extension?: string): Promise<string> {
+ *     // Upload to S3 bucket
+ *     return 's3://bucket/path/to/file'
+ *   }
+ *
+ *   async retrieve(path: string): Promise<Buffer> {
+ *     // Download from S3
+ *     return buffer
+ *   }
+ *
+ *   async delete(path: string): Promise<void> {
+ *     // Delete from S3
+ *   }
+ * }
+ * ```
+ */
+export declare abstract class StorageService {
+    /**
+     * Persists binary data and returns a unique identifier for retrieval.
+     *
+     * The storage implementation should ensure the returned path/URL is unique
+     * and can be used later to retrieve the exact same data.
+     *
+     * @abstract
+     * @param {Buffer} buffer - Binary data to store
+     * @param {string} collectorName - Component name for organizing storage (used as folder/prefix)
+     * @param {string} extension - Optional file extension for proper content handling
+     * @returns {Promise<string>} Unique storage identifier (path, URL, or key)
+     * @throws {Error} When storage operation fails
+     *
+     * @example
+     * ```typescript
+     * const buffer = Buffer.from('{"temperature": 23.5}')
+     * const path = await storage.save(buffer, 'weather-sensor', 'json')
+     * // Returns: '/storage/weather-sensor/2024-01-15_14-30-00.json'
+     * ```
+     */
+    abstract save(buffer: Buffer, collectorName: string, extension?: string): Promise<string>;
+    /**
+     * Retrieves previously stored binary data.
+     *
+     * Uses the identifier returned by save() to fetch the original data.
+     *
+     * @abstract
+     * @param {string} path - Storage identifier from save() operation
+     * @returns {Promise<Buffer>} The original binary data
+     * @throws {Error} When file doesn't exist or retrieval fails
+     *
+     * @example
+     * ```typescript
+     * const path = '/storage/weather-sensor/2024-01-15_14-30-00.json'
+     * const data = await storage.retrieve(path)
+     * const json = JSON.parse(data.toString())
+     * ```
+     */
+    abstract retrieve(path: string): Promise<Buffer>;
+    /**
+     * Removes stored data permanently.
+     *
+     * Deletes the data associated with the given storage identifier.
+     *
+     * @abstract
+     * @param {string} path - Storage identifier from save() operation
+     * @returns {Promise<void>}
+     * @throws {Error} When deletion fails or path doesn't exist
+     *
+     * @example
+     * ```typescript
+     * const path = '/storage/weather-sensor/old-data.json'
+     * await storage.delete(path)
+     * ```
+     */
+    abstract delete(path: string): Promise<void>;
+    /**
+     * Persists binary data at a specific path (no auto-generated filename).
+     *
+     * Unlike save(), this method stores the file at the exact path specified,
+     * preserving the original filename and directory structure.
+     * Useful for extracting archives where file paths must be preserved.
+     *
+     * @param {Buffer} buffer - Binary data to store
+     * @param {string} relativePath - Full relative path including filename (e.g., 'tilesets/123/tileset.json')
+     * @returns {Promise<string>} The same path that was provided (for consistency)
+     * @throws {Error} When storage operation fails
+     *
+     * @example
+     * ```typescript
+     * const buffer = Buffer.from('{"asset": {"version": "1.0"}}')
+     * const path = await storage.saveWithPath(buffer, 'tilesets/123/tileset.json')
+     * // Returns: 'tilesets/123/tileset.json'
+     * ```
+     */
+    abstract saveWithPath(buffer: Buffer, relativePath: string): Promise<string>;
+    /**
+     * Deletes multiple files in batch for better performance.
+     *
+     * Default implementation calls delete() sequentially, but storage backends
+     * can override this with optimized bulk delete operations (e.g., S3 DeleteObjects).
+     *
+     * @param {string[]} paths - Array of storage identifiers to delete
+     * @returns {Promise<void>}
+     *
+     * @example
+     * ```typescript
+     * await storage.deleteBatch([
+     *   'tilesets/123/tileset.json',
+     *   'tilesets/123/tile_0.b3dm',
+     *   'tilesets/123/tile_1.b3dm'
+     * ])
+     * ```
+     */
+    deleteBatch(paths: string[]): Promise<void>;
+    /**
+     * Returns the public URL for a stored file.
+     *
+     * For cloud storage (S3, OVH, Azure), this returns the direct HTTP URL.
+     * For local storage, this may return a relative path or throw an error.
+     *
+     * @abstract
+     * @param {string} relativePath - The storage path/key of the file
+     * @returns {string} The public URL to access the file directly
+     *
+     * @example
+     * ```typescript
+     * const url = storage.getPublicUrl('tilesets/123/tileset.json')
+     * // Returns: 'https://bucket.s3.region.cloud.ovh.net/tilesets/123/tileset.json'
+     * ```
+     */
+    abstract getPublicUrl(relativePath: string): string;
+    /**
+     * Deletes all files under a given prefix/folder.
+     *
+     * This is more efficient than deleteBatch() when you don't know all file paths,
+     * as it lists objects by prefix and deletes them in bulk.
+     * Useful for deleting entire tilesets or component data.
+     *
+     * @abstract
+     * @param {string} prefix - The folder/prefix to delete (e.g., 'tilesets/123')
+     * @returns {Promise<number>} Number of files deleted
+     *
+     * @example
+     * ```typescript
+     * const count = await storage.deleteByPrefix('tilesets/123')
+     * // Deletes all files starting with 'tilesets/123/'
+     * console.log(`Deleted ${count} files`)
+     * ```
+     */
+    abstract deleteByPrefix(prefix: string): Promise<number>;
+}
+//# sourceMappingURL=storage_service.d.ts.map

package/dist/storage/storage_service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage_service.d.ts","sourceRoot":"","sources":["../../src/storage/storage_service.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,8BAAsB,cAAc;IAChC;;;;;;;;;;;;;;;;;;;OAmBG;IACH,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAEzF;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAEhD;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAE5C;;;;;;;;;;;;;;;;;;OAkBG;IACH,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAE5E;;;;;;;;;;;;;;;;;OAiBG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAMjD;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM;IAEnD;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAC3D"}

package/dist/storage/storage_service.js

@@ -0,0 +1,58 @@
+import { safeAsync } from '../utils/safe_async.js';
+import { Logger } from '../utils/logger.js';
+const logger = new Logger('StorageService');
+/**
+ * Abstract base class for storage service implementations.
+ *
+ * Defines the contract for persisting and retrieving binary data in the Digital Twin framework.
+ * Concrete implementations provide storage backends like local filesystem, AWS S3, Azure Blob, etc.
+ *
+ * @abstract
+ * @class StorageService
+ *
+ * @example
+ * ```typescript
+ * // Implement for specific storage backend
+ * class S3StorageService extends StorageService {
+ *   async save(buffer: Buffer, collectorName: string, extension?: string): Promise<string> {
+ *     // Upload to S3 bucket
+ *     return 's3://bucket/path/to/file'
+ *   }
+ *
+ *   async retrieve(path: string): Promise<Buffer> {
+ *     // Download from S3
+ *     return buffer
+ *   }
+ *
+ *   async delete(path: string): Promise<void> {
+ *     // Delete from S3
+ *   }
+ * }
+ * ```
+ */
+export class StorageService {
+    /**
+     * Deletes multiple files in batch for better performance.
+     *
+     * Default implementation calls delete() sequentially, but storage backends
+     * can override this with optimized bulk delete operations (e.g., S3 DeleteObjects).
+     *
+     * @param {string[]} paths - Array of storage identifiers to delete
+     * @returns {Promise<void>}
+     *
+     * @example
+     * ```typescript
+     * await storage.deleteBatch([
+     *   'tilesets/123/tileset.json',
+     *   'tilesets/123/tile_0.b3dm',
+     *   'tilesets/123/tile_1.b3dm'
+     * ])
+     * ```
+     */
+    async deleteBatch(paths) {
+        // Default parallel implementation - subclasses can override with bulk operations
+        // Individual failures are logged but don't prevent other deletions
+        await Promise.all(paths.map(path => safeAsync(() => this.delete(path), `delete file ${path}`, logger)));
+    }
+}
+//# sourceMappingURL=storage_service.js.map

package/dist/storage/storage_service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage_service.js","sourceRoot":"","sources":["../../src/storage/storage_service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAA;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAA;AAE3C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAAA;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAgB,cAAc;IAiFhC;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,WAAW,CAAC,KAAe;QAC7B,iFAAiF;QACjF,mEAAmE;QACnE,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,eAAe,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IAC3G,CAAC;CAuCJ"}

package/dist/types/data_record.d.ts

@@ -0,0 +1,123 @@
+/**
+ * @fileoverview Data record type definition for digital twin data storage
+ *
+ * This module defines the standardized interface for data records used
+ * throughout the digital twin system for both streaming data and asset storage.
+ */
+/**
+ * Standard data record interface for digital twin data storage.
+ *
+ * DataRecord provides a unified interface for all data stored in the digital twin system,
+ * from streaming sensor data to asset files. It supports both database metadata
+ * and file storage through a consistent API.
+ *
+ * @example
+ * ```typescript
+ * const record: DataRecord = {
+ *   id: 12345,
+ *   name: 'weather-station-001',
+ *   date: new Date(),
+ *   contentType: 'application/json',
+ *   url: '/api/data/weather/12345',
+ *   data: async () => Buffer.from(JSON.stringify(weatherData)),
+ *   description: 'Weather data from station 001',
+ *   source: 'https://api.weather.com/v1/current',
+ *   owner_id: 123
+ * };
+ * ```
+ */
+export interface DataRecord {
+    /** Unique identifier for this data record */
+    id: number;
+    /** Human-readable name or identifier for this record */
+    name: string;
+    /** Timestamp when this record was created or last updated */
+    date: Date;
+    /** MIME type of the data content (e.g., 'application/json', 'image/png') */
+    contentType: string;
+    /** URL or path where this record can be accessed */
+    url: string;
+    /**
+     * Function that returns the actual data content as a Buffer.
+     *
+     * This lazy-loading approach allows for efficient memory usage
+     * when working with large datasets or files.
+     *
+     * @returns Promise resolving to the data content as a Buffer
+     */
+    data: () => Promise<Buffer>;
+    /**
+     * Human-readable description of the asset.
+     *
+     * Used primarily by AssetsManager components to provide
+     * additional context about stored files and resources.
+     */
+    description?: string;
+    /**
+     * Source URL for data provenance.
+     *
+     * Records the original source of this data for traceability
+     * and debugging purposes. Used by AssetsManager components.
+     */
+    source?: string;
+    /**
+     * ID of the user who owns this asset.
+     *
+     * Enables access control and ownership tracking for
+     * asset management. Used by AssetsManager components.
+     */
+    owner_id?: number | null;
+    /**
+     * Original filename provided by the user.
+     *
+     * Preserves the original filename when assets are uploaded
+     * or imported. Used by AssetsManager components.
+     */
+    filename?: string;
+    /**
+     * Whether the asset is publicly accessible.
+     *
+     * Controls visibility of assets. If true, asset can be accessed
+     * by anyone. If false, only the owner can access it.
+     * Used by AssetsManager components for access control.
+     */
+    is_public?: boolean;
+    /**
+     * Public URL to tileset.json for Cesium loading.
+     * Cesium accesses files directly via this URL.
+     */
+    tileset_url?: string;
+    /**
+     * @deprecated No longer used. Kept for backward compatibility.
+     */
+    file_index?: {
+        /** List of files with their storage paths */
+        files: Array<{
+            /** Storage path for this file */
+            path: string;
+            /** Original filename */
+            name: string;
+            /** File size in bytes */
+            size?: number;
+        }>;
+        /** Main entry point file (e.g., 'tileset.json') */
+        root_file?: string;
+    };
+    /**
+     * Status of async upload processing.
+     * - 'pending': Job queued, waiting to start
+     * - 'processing': Job is running
+     * - 'completed': Job finished successfully
+     * - 'failed': Job failed with error
+     */
+    upload_status?: 'pending' | 'processing' | 'completed' | 'failed' | null;
+    /**
+     * Error message if upload failed.
+     */
+    upload_error?: string | null;
+    /**
+     * BullMQ job ID for tracking.
+     */
+    upload_job_id?: string | null;
+}
+//# sourceMappingURL=data_record.d.ts.map

package/dist/types/data_record.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data_record.d.ts","sourceRoot":"","sources":["../../src/types/data_record.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,UAAU;IACvB,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAA;IAEV,wDAAwD;IACxD,IAAI,EAAE,MAAM,CAAA;IAEZ,6DAA6D;IAC7D,IAAI,EAAE,IAAI,CAAA;IAEV,4EAA4E;IAC5E,WAAW,EAAE,MAAM,CAAA;IAEnB,oDAAoD;IACpD,GAAG,EAAE,MAAM,CAAA;IAEX;;;;;;;OAOG;IACH,IAAI,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAA;IAI3B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAExB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IAInB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAIpB;;OAEG;IACH,UAAU,CAAC,EAAE;QACT,6CAA6C;QAC7C,KAAK,EAAE,KAAK,CAAC;YACT,iCAAiC;YACjC,IAAI,EAAE,MAAM,CAAA;YACZ,wBAAwB;YACxB,IAAI,EAAE,MAAM,CAAA;YACZ,yBAAyB;YACzB,IAAI,CAAC,EAAE,MAAM,CAAA;SAChB,CAAC,CAAA;QACF,mDAAmD;QACnD,SAAS,CAAC,EAAE,MAAM,CAAA;KACrB,CAAA;IAID;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,SAAS,GAAG,YAAY,GAAG,WAAW,GAAG,QAAQ,GAAG,IAAI,CAAA;IAExE;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAE5B;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAChC"}

package/dist/types/data_record.js

@@ -0,0 +1,8 @@
+/**
+ * @fileoverview Data record type definition for digital twin data storage
+ *
+ * This module defines the standardized interface for data records used
+ * throughout the digital twin system for both streaming data and asset storage.
+ */
+export {};
+//# sourceMappingURL=data_record.js.map

package/dist/types/data_record.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"data_record.js","sourceRoot":"","sources":["../../src/types/data_record.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}