@cepseudo/assets 1.0.0

package/dist/tileset_manager.js

@@ -0,0 +1,705 @@
+import { AssetsManager } from './assets_manager.js';
+import { successResponse, errorResponse, badRequestResponse, notFoundResponse, forbiddenResponse, safeAsync, safeCleanup, Logger } from '@cepseudo/shared';
+import { ApisixAuthParser } from '@cepseudo/auth';
+import { extractAndStoreArchive } from './utils/zip_utils.js';
+import fs from 'fs/promises';
+const logger = new Logger('TilesetManager');
+/** Threshold for async upload (50MB) */
+const ASYNC_UPLOAD_THRESHOLD = 50 * 1024 * 1024;
+/**
+ * Specialized Assets Manager for handling 3D Tiles tilesets.
+ *
+ * This manager extracts uploaded ZIP files and stores each file in cloud storage (OVH S3),
+ * allowing Cesium and other 3D viewers to load tilesets directly via public URLs.
+ *
+ * ## How it works
+ *
+ * 1. User uploads a ZIP containing a 3D Tiles tileset
+ * 2. ZIP is extracted and all files are stored in OVH with public-read ACL
+ * 3. Database stores only the tileset.json URL and base path
+ * 4. Cesium loads tileset.json directly from OVH
+ * 5. Cesium fetches tiles using relative paths in tileset.json (directly from OVH)
+ *
+ * ## Endpoints
+ *
+ * - GET /{endpoint} - List all tilesets with their public URLs
+ * - POST /{endpoint} - Upload tileset ZIP (sync < 50MB, async >= 50MB)
+ * - GET /{endpoint}/:id/status - Poll async upload status
+ * - PUT /{endpoint}/:id - Update tileset metadata
+ * - DELETE /{endpoint}/:id - Delete tileset and all files from storage
+ *
+ * @example
+ * ```typescript
+ * class MyTilesetManager extends TilesetManager {
+ *   getConfiguration() {
+ *     return {
+ *       name: 'tilesets',
+ *       description: 'Manage 3D Tiles tilesets',
+ *       contentType: 'application/json',
+ *       endpoint: 'api/tilesets',
+ *       extension: '.zip'
+ *     }
+ *   }
+ * }
+ *
+ * // After upload, response contains:
+ * // { tileset_url: 'https://bucket.s3.../tilesets/123/tileset.json' }
+ * //
+ * // Cesium loads directly:
+ * // Cesium.Cesium3DTileset.fromUrl(tileset_url)
+ * ```
+ */
+export class TilesetManager extends AssetsManager {
+    constructor() {
+        super(...arguments);
+        /** Upload queue for async processing (injected by engine) */
+        this.uploadQueue = null;
+    }
+    /**
+     * Set the upload queue for async job processing.
+     * Called by DigitalTwinEngine during initialization.
+     */
+    setUploadQueue(queue) {
+        this.uploadQueue = queue;
+    }
+    /**
+     * Handle tileset upload.
+     *
+     * - Files < 50MB: Synchronous extraction and upload
+     * - Files >= 50MB: Queued for async processing (returns 202)
+     */
+    async handleUpload(req) {
+        try {
+            if (!req?.body) {
+                return badRequestResponse('Invalid request: missing request body');
+            }
+            // Authenticate user
+            const userId = await this.authenticateUser(req);
+            if (typeof userId !== 'number') {
+                return userId; // Returns error response
+            }
+            // Validate request
+            const body = req.body;
+            const description = body.description;
+            const filePath = req.file?.path;
+            const fileBuffer = req.file?.buffer;
+            const filename = (req.file?.originalname || body.filename);
+            const fileSize = req.file?.size || fileBuffer?.length || 0;
+            if (!filePath && !fileBuffer) {
+                return badRequestResponse('Missing required field: ZIP file');
+            }
+            if (!description) {
+                if (filePath)
+                    await safeAsync(() => fs.unlink(filePath), 'cleanup temp file on validation error', logger);
+                return badRequestResponse('Missing required field: description');
+            }
+            if (!filename) {
+                if (filePath)
+                    await safeAsync(() => fs.unlink(filePath), 'cleanup temp file on validation error', logger);
+                return badRequestResponse('Filename could not be determined from uploaded file');
+            }
+            if (!filename.toLowerCase().endsWith('.zip')) {
+                if (filePath)
+                    await safeAsync(() => fs.unlink(filePath), 'cleanup temp file on validation error', logger);
+                return badRequestResponse('Invalid file extension. Expected: .zip');
+            }
+            const config = this.getConfiguration();
+            const isPublic = body.is_public !== undefined ? Boolean(body.is_public) : true;
+            // Route to async or sync based on file size and queue availability
+            if (this.uploadQueue && filePath && fileSize >= ASYNC_UPLOAD_THRESHOLD) {
+                return this.handleAsyncUpload(userId, filePath, filename, description, isPublic, config);
+            }
+            return this.handleSyncUpload(userId, filePath, fileBuffer, filename, description, isPublic, config);
+        }
+        catch (error) {
+            if (req.file?.path)
+                await safeAsync(() => fs.unlink(req.file.path), 'cleanup temp file on error', logger);
+            return errorResponse(error);
+        }
+    }
+    /**
+     * Authenticate user from request headers.
+     * Returns user ID on success, or error response on failure.
+     */
+    async authenticateUser(req) {
+        const authResult = await this.authMiddleware.authenticate(req);
+        if (!authResult.success) {
+            return authResult.response;
+        }
+        return authResult.userRecord.id;
+    }
+    /**
+     * Queue upload for background processing. Returns HTTP 202 immediately.
+     */
+    async handleAsyncUpload(userId, filePath, filename, description, isPublic, config) {
+        let recordId = null;
+        try {
+            // Create pending record (url will be updated after extraction)
+            const metadata = {
+                name: config.name,
+                type: 'application/json',
+                url: '',
+                tileset_url: '',
+                date: new Date(),
+                description,
+                filename,
+                owner_id: userId,
+                is_public: isPublic,
+                upload_status: 'pending'
+            };
+            const savedRecord = await this.db.save(metadata);
+            recordId = savedRecord.id;
+            const jobData = {
+                type: 'tileset',
+                recordId,
+                tempFilePath: filePath,
+                componentName: config.name,
+                userId,
+                filename,
+                description
+            };
+            const job = await this.uploadQueue?.add(`tileset-${recordId}`, jobData, {
+                jobId: `tileset-upload-${recordId}`
+            });
+            if (!job)
+                throw new Error('Failed to queue upload job');
+            await this.db.updateById(config.name, recordId, { upload_job_id: job.id });
+            return {
+                status: 202,
+                content: JSON.stringify({
+                    message: 'Tileset upload accepted, processing in background',
+                    id: recordId,
+                    job_id: job.id,
+                    status: 'pending',
+                    status_url: `/${config.endpoint}/${recordId}/status`
+                }),
+                headers: { 'Content-Type': 'application/json' }
+            };
+        }
+        catch (error) {
+            await safeCleanup([
+                {
+                    operation: () => recordId !== null ? this.db.delete(String(recordId), config.name) : Promise.resolve(),
+                    context: 'delete record on async upload error'
+                },
+                { operation: () => fs.unlink(filePath), context: 'cleanup temp file on async upload error' }
+            ], logger);
+            throw error;
+        }
+    }
+    /**
+     * Process upload synchronously.
+     */
+    async handleSyncUpload(userId, filePath, fileBuffer, filename, description, isPublic, config) {
+        let zipBuffer;
+        try {
+            const readBuffer = fileBuffer || (filePath ? await fs.readFile(filePath) : null);
+            if (!readBuffer)
+                throw new Error('No file data available');
+            zipBuffer = readBuffer;
+        }
+        catch (error) {
+            return errorResponse(`Failed to read uploaded file: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        }
+        try {
+            // Generate unique base path using timestamp
+            const basePath = `${config.name}/${Date.now()}`;
+            // Extract ZIP and upload all files to storage
+            const extractResult = await extractAndStoreArchive(zipBuffer, this.storage, basePath);
+            if (!extractResult.root_file) {
+                // Clean up uploaded files
+                await safeAsync(() => this.storage.deleteByPrefix(basePath), 'cleanup storage on invalid tileset', logger);
+                return badRequestResponse('Invalid tileset: no tileset.json found in the ZIP archive');
+            }
+            // Build the public URL for tileset.json
+            const tilesetPath = `${basePath}/${extractResult.root_file}`;
+            const tilesetUrl = this.storage.getPublicUrl(tilesetPath);
+            // Save metadata to database (url = basePath for deletion)
+            const metadata = {
+                name: config.name,
+                type: 'application/json',
+                url: basePath,
+                tileset_url: tilesetUrl,
+                date: new Date(),
+                description,
+                filename,
+                owner_id: userId,
+                is_public: isPublic,
+                upload_status: 'completed'
+            };
+            const savedRecord = await this.db.save(metadata);
+            // Clean up temp file
+            if (filePath)
+                await safeAsync(() => fs.unlink(filePath), 'cleanup temp file after sync upload', logger);
+            return successResponse({
+                message: 'Tileset uploaded successfully',
+                id: savedRecord.id,
+                tileset_url: tilesetUrl,
+                file_count: extractResult.file_count
+            });
+        }
+        catch (error) {
+            if (filePath)
+                await safeAsync(() => fs.unlink(filePath), 'cleanup temp file on sync upload error', logger);
+            throw error;
+        }
+    }
+    /**
+     * Get upload status for async uploads.
+     */
+    async handleGetStatus(req) {
+        try {
+            const { id } = req.params || {};
+            if (!id) {
+                return badRequestResponse('Asset ID is required');
+            }
+            const asset = await this.getAssetById(id);
+            if (!asset) {
+                return notFoundResponse('Tileset not found');
+            }
+            const record = asset;
+            if (record.upload_status === 'completed') {
+                return successResponse({
+                    id: record.id,
+                    status: 'completed',
+                    tileset_url: record.tileset_url
+                });
+            }
+            if (record.upload_status === 'failed') {
+                return successResponse({
+                    id: record.id,
+                    status: 'failed',
+                    error: record.upload_error || 'Upload failed'
+                });
+            }
+            return successResponse({
+                id: record.id,
+                status: record.upload_status || 'unknown',
+                job_id: record.upload_job_id
+            });
+        }
+        catch (error) {
+            return errorResponse(error);
+        }
+    }
+    /**
+     * Override presigned upload confirmation for tilesets.
+     * Instead of marking as completed, queue a BullMQ job for ZIP extraction.
+     */
+    async handleUploadConfirm(req) {
+        try {
+            // Authenticate user
+            const authResult = await this.authMiddleware.authenticate(req);
+            if (!authResult.success) {
+                return authResult.response;
+            }
+            const userId = authResult.userRecord.id;
+            if (!userId) {
+                return errorResponse('Failed to retrieve user information');
+            }
+            const fileId = req.params?.fileId;
+            if (!fileId) {
+                return badRequestResponse('File ID is required');
+            }
+            const config = this.getConfiguration();
+            const asset = await this.getAssetById(fileId);
+            if (!asset) {
+                return notFoundResponse('Tileset not found');
+            }
+            // Check ownership
+            const ownershipError = this.validateOwnership(asset, userId, req.headers);
+            if (ownershipError) {
+                return ownershipError;
+            }
+            // Check status
+            if (asset.upload_status !== 'pending') {
+                return {
+                    status: 409,
+                    content: JSON.stringify({ error: `Upload is not pending (current status: ${asset.upload_status || 'completed'})` }),
+                    headers: { 'Content-Type': 'application/json' }
+                };
+            }
+            // Verify file exists on storage
+            if (!asset.presigned_key) {
+                return badRequestResponse('No presigned key found for this record');
+            }
+            const existsResult = await this.storage.objectExists(asset.presigned_key);
+            if (!existsResult.exists) {
+                return badRequestResponse('File not found on storage. Please upload the file using the presigned URL first.');
+            }
+            // Queue extraction job instead of marking completed
+            if (!this.uploadQueue) {
+                return errorResponse('Upload processing queue is not available');
+            }
+            const jobData = {
+                type: 'tileset',
+                recordId: asset.id,
+                tempFilePath: '', // Not used for presigned uploads
+                componentName: config.name,
+                userId,
+                filename: asset.filename || 'tileset.zip',
+                description: asset.description || '',
+                presignedKey: asset.presigned_key
+            };
+            const job = await this.uploadQueue.add(`tileset-presigned-${asset.id}`, jobData, {
+                jobId: `tileset-presigned-${asset.id}`
+            });
+            if (!job) {
+                return errorResponse('Failed to queue extraction job');
+            }
+            await this.db.updateById(config.name, asset.id, {
+                upload_status: 'processing',
+                upload_job_id: job.id
+            });
+            return {
+                status: 202,
+                content: JSON.stringify({
+                    message: 'Upload confirmed, extraction queued',
+                    id: asset.id,
+                    job_id: job.id,
+                    status: 'processing',
+                    status_url: `/${config.endpoint}/${asset.id}/status`
+                }),
+                headers: { 'Content-Type': 'application/json' }
+            };
+        }
+        catch (error) {
+            return errorResponse(error);
+        }
+    }
+    /**
+     * List all tilesets with their public URLs.
+     */
+    async retrieve(req) {
+        try {
+            const assets = await this.getAllAssets();
+            const isAdmin = req && ApisixAuthParser.isAdmin(req.headers || {});
+            // Get authenticated user ID if available
+            let authenticatedUserId = null;
+            if (req) {
+                const authResult = await this.authMiddleware.authenticate(req);
+                if (authResult.success) {
+                    authenticatedUserId = authResult.userRecord.id || null;
+                }
+            }
+            // Filter to visible assets only (unless admin)
+            const visibleAssets = isAdmin
+                ? assets
+                : assets.filter(asset => asset.is_public || (authenticatedUserId !== null && asset.owner_id === authenticatedUserId));
+            // Transform to response format
+            const response = visibleAssets.map(asset => ({
+                id: asset.id,
+                description: asset.description || '',
+                filename: asset.filename || '',
+                date: asset.date,
+                owner_id: asset.owner_id || null,
+                is_public: asset.is_public ?? true,
+                tileset_url: asset.tileset_url || '',
+                upload_status: asset.upload_status || 'completed'
+            }));
+            return successResponse(response);
+        }
+        catch (error) {
+            return errorResponse(error);
+        }
+    }
+    /**
+     * Delete tileset and all files from storage.
+     */
+    async handleDelete(req) {
+        try {
+            // Authenticate user
+            const userId = await this.authenticateUser(req);
+            if (typeof userId !== 'number') {
+                return userId;
+            }
+            const { id } = req.params || {};
+            if (!id) {
+                return badRequestResponse('Asset ID is required');
+            }
+            const asset = await this.getAssetById(id);
+            if (!asset) {
+                return notFoundResponse('Tileset not found');
+            }
+            // Check ownership (admins can delete any)
+            const isAdmin = ApisixAuthParser.isAdmin(req.headers || {});
+            if (!isAdmin && asset.owner_id !== null && asset.owner_id !== userId) {
+                return forbiddenResponse('You can only delete your own assets');
+            }
+            // Block deletion while upload in progress
+            if (asset.upload_status === 'pending' || asset.upload_status === 'processing') {
+                return {
+                    status: 409,
+                    content: JSON.stringify({ error: 'Cannot delete tileset while upload is in progress' }),
+                    headers: { 'Content-Type': 'application/json' }
+                };
+            }
+            // Delete all files from storage
+            // Support both new format (url = basePath) and legacy format (file_index.files)
+            const legacyFileIndex = asset.file_index;
+            if (legacyFileIndex?.files && legacyFileIndex.files.length > 0) {
+                // Legacy format: delete individual files from file_index
+                logger.info(`Deleting ${legacyFileIndex.files.length} files (legacy format)`);
+                for (const file of legacyFileIndex.files) {
+                    await safeAsync(() => this.storage.delete(file.path), `delete legacy file ${file.path}`, logger);
+                }
+            }
+            else if (asset.url) {
+                // New format: url contains basePath, use deleteByPrefix
+                const deletedCount = await this.storage.deleteByPrefix(asset.url);
+                logger.info(`Deleted ${deletedCount} files from ${asset.url}`);
+            }
+            // Delete database record
+            await this.deleteAssetById(id);
+            return successResponse({ message: 'Tileset deleted successfully' });
+        }
+        catch (error) {
+            return errorResponse(error);
+        }
+    }
+    /**
+     * Get HTTP endpoints for this manager.
+     */
+    getEndpoints() {
+        const config = this.getConfiguration();
+        return [
+            // Status endpoint (for async upload polling)
+            {
+                method: 'get',
+                path: `/${config.endpoint}/:id/status`,
+                handler: this.handleGetStatus.bind(this),
+                responseType: 'application/json'
+            },
+            // List tilesets
+            {
+                method: 'get',
+                path: `/${config.endpoint}`,
+                handler: this.retrieve.bind(this),
+                responseType: 'application/json'
+            },
+            // Upload tileset (multipart)
+            {
+                method: 'post',
+                path: `/${config.endpoint}`,
+                handler: this.handleUpload.bind(this),
+                responseType: 'application/json'
+            },
+            // Presigned upload request
+            {
+                method: 'post',
+                path: `/${config.endpoint}/upload-request`,
+                handler: this.handlePresignedUploadRequest.bind(this),
+                responseType: 'application/json'
+            },
+            // Presigned upload confirm (triggers extraction)
+            {
+                method: 'post',
+                path: `/${config.endpoint}/confirm/:fileId`,
+                handler: this.handleUploadConfirm.bind(this),
+                responseType: 'application/json'
+            },
+            // Update metadata
+            {
+                method: 'put',
+                path: `/${config.endpoint}/:id`,
+                handler: this.handleUpdate.bind(this),
+                responseType: 'application/json'
+            },
+            // Delete tileset
+            {
+                method: 'delete',
+                path: `/${config.endpoint}/:id`,
+                handler: this.handleDelete.bind(this),
+                responseType: 'application/json'
+            }
+        ];
+    }
+    /**
+     * Generate OpenAPI specification.
+     */
+    getOpenAPISpec() {
+        const config = this.getConfiguration();
+        const basePath = `/${config.endpoint}`;
+        const tagName = config.tags?.[0] || config.name;
+        return {
+            paths: {
+                [basePath]: {
+                    get: {
+                        summary: 'List all tilesets',
+                        description: 'Returns all tilesets with their public URLs for Cesium loading',
+                        tags: [tagName],
+                        responses: {
+                            '200': {
+                                description: 'List of tilesets',
+                                content: {
+                                    'application/json': {
+                                        schema: {
+                                            type: 'array',
+                                            items: { $ref: '#/components/schemas/TilesetResponse' }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    post: {
+                        summary: 'Upload a tileset',
+                        description: 'Upload a ZIP file containing a 3D Tiles tileset. Files < 50MB are processed synchronously, larger files are queued.',
+                        tags: [tagName],
+                        security: [{ ApiKeyAuth: [] }],
+                        requestBody: {
+                            required: true,
+                            content: {
+                                'multipart/form-data': {
+                                    schema: {
+                                        type: 'object',
+                                        required: ['file', 'description'],
+                                        properties: {
+                                            file: {
+                                                type: 'string',
+                                                format: 'binary',
+                                                description: 'ZIP file containing tileset'
+                                            },
+                                            description: { type: 'string', description: 'Tileset description' },
+                                            is_public: {
+                                                type: 'boolean',
+                                                description: 'Whether tileset is public (default: true)'
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        },
+                        responses: {
+                            '200': {
+                                description: 'Tileset uploaded successfully (sync)',
+                                content: {
+                                    'application/json': {
+                                        schema: {
+                                            type: 'object',
+                                            properties: {
+                                                message: { type: 'string' },
+                                                id: { type: 'integer' },
+                                                tileset_url: {
+                                                    type: 'string',
+                                                    description: 'Public URL to load in Cesium'
+                                                },
+                                                file_count: { type: 'integer' }
+                                            }
+                                        }
+                                    }
+                                }
+                            },
+                            '202': {
+                                description: 'Upload accepted for async processing',
+                                content: {
+                                    'application/json': {
+                                        schema: {
+                                            type: 'object',
+                                            properties: {
+                                                message: { type: 'string' },
+                                                id: { type: 'integer' },
+                                                status: { type: 'string' },
+                                                status_url: { type: 'string' }
+                                            }
+                                        }
+                                    }
+                                }
+                            },
+                            '400': { description: 'Bad request - missing fields or invalid file' },
+                            '401': { description: 'Unauthorized' }
+                        }
+                    }
+                },
+                [`${basePath}/{id}/status`]: {
+                    get: {
+                        summary: 'Get upload status',
+                        description: 'Poll the status of an async upload',
+                        tags: [tagName],
+                        parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'string' } }],
+                        responses: {
+                            '200': {
+                                description: 'Upload status',
+                                content: {
+                                    'application/json': {
+                                        schema: {
+                                            type: 'object',
+                                            properties: {
+                                                id: { type: 'integer' },
+                                                status: {
+                                                    type: 'string',
+                                                    enum: ['pending', 'processing', 'completed', 'failed']
+                                                },
+                                                tileset_url: { type: 'string' },
+                                                error: { type: 'string' }
+                                            }
+                                        }
+                                    }
+                                }
+                            },
+                            '404': { description: 'Tileset not found' }
+                        }
+                    }
+                },
+                [`${basePath}/{id}`]: {
+                    put: {
+                        summary: 'Update tileset metadata',
+                        tags: [tagName],
+                        security: [{ ApiKeyAuth: [] }],
+                        parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'string' } }],
+                        requestBody: {
+                            content: {
+                                'application/json': {
+                                    schema: {
+                                        type: 'object',
+                                        properties: {
+                                            description: { type: 'string' },
+                                            is_public: { type: 'boolean' }
+                                        }
+                                    }
+                                }
+                            }
+                        },
+                        responses: {
+                            '200': { description: 'Updated successfully' },
+                            '401': { description: 'Unauthorized' },
+                            '403': { description: 'Forbidden' },
+                            '404': { description: 'Not found' }
+                        }
+                    },
+                    delete: {
+                        summary: 'Delete tileset',
+                        description: 'Delete tileset and all files from storage',
+                        tags: [tagName],
+                        security: [{ ApiKeyAuth: [] }],
+                        parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'string' } }],
+                        responses: {
+                            '200': { description: 'Deleted successfully' },
+                            '401': { description: 'Unauthorized' },
+                            '403': { description: 'Forbidden' },
+                            '404': { description: 'Not found' },
+                            '409': { description: 'Upload in progress' }
+                        }
+                    }
+                }
+            },
+            tags: [{ name: tagName, description: config.description }],
+            schemas: {
+                TilesetResponse: {
+                    type: 'object',
+                    properties: {
+                        id: { type: 'integer' },
+                        description: { type: 'string' },
+                        filename: { type: 'string' },
+                        date: { type: 'string', format: 'date-time' },
+                        owner_id: { type: 'integer', nullable: true },
+                        is_public: { type: 'boolean' },
+                        tileset_url: { type: 'string', description: 'Public URL to load in Cesium' },
+                        upload_status: { type: 'string', enum: ['pending', 'processing', 'completed', 'failed'] }
+                    }
+                }
+            }
+        };
+    }
+}
+//# sourceMappingURL=tileset_manager.js.map