@cepseudo/assets 1.0.0

package/dist/utils/zip_utils.js

@@ -0,0 +1,169 @@
+import JSZip from 'jszip';
+/**
+ * Extracts the content of a zip file as a stream (for large files)
+ * @param zipBuffer - The content of the zip file as Buffer
+ * @returns A generator yielding tuples containing the name and content of each file in the zip file
+ */
+export async function* extractZipContentStream(zipBuffer) {
+    const zip = new JSZip();
+    const zipContent = await zip.loadAsync(zipBuffer);
+    for (const [name, zipObject] of Object.entries(zipContent.files)) {
+        if (!zipObject.dir) {
+            // Skip directories
+            const content = await zipObject.async('nodebuffer');
+            // Try to decode as UTF-8, fallback to Buffer for binary files
+            try {
+                const textContent = content.toString('utf-8');
+                // Check if it's valid UTF-8 by trying to encode it back
+                Buffer.from(textContent, 'utf-8');
+                yield [name, textContent];
+            }
+            catch {
+                // If UTF-8 decoding fails, return as Buffer
+                yield [name, content];
+            }
+        }
+    }
+}
+/**
+ * Converts a zip file to a dictionary containing all files and their contents
+ * @param zipBuffer - The content of the zip file as Buffer
+ * @returns A dictionary containing the content of the zip file
+ */
+export async function zipToDict(zipBuffer) {
+    const output = {};
+    for await (const [name, content] of extractZipContentStream(zipBuffer)) {
+        output[name] = content;
+    }
+    return output;
+}
+/**
+ * Detects the root file for 3D Tiles tilesets
+ * Looks for tileset.json or similar entry point files
+ * @param files - List of file paths in the archive
+ * @returns The path to the root file, or undefined if not found
+ */
+export function detectTilesetRootFile(files) {
+    // Priority order for 3D Tiles root files
+    const rootFilePatterns = [
+        /^tileset\.json$/i,
+        /\/tileset\.json$/i,
+        /^[^/]+\/tileset\.json$/i, // One level deep
+        /tileset\.json$/i // Any tileset.json as fallback
+    ];
+    for (const pattern of rootFilePatterns) {
+        const match = files.find(f => pattern.test(f));
+        if (match)
+            return match;
+    }
+    return undefined;
+}
+/**
+ * Normalizes file paths from ZIP archives
+ * Removes leading directory if all files share the same root folder
+ * @param files - Original file paths from the archive
+ * @returns Normalized file paths (original -> normalized)
+ */
+export function normalizeArchivePaths(files) {
+    const pathMap = new Map();
+    // Check if all files share a common root directory
+    const firstParts = files.map(f => f.split('/')[0]);
+    const commonRoot = firstParts.every(p => p === firstParts[0]) && firstParts[0] !== '' ? firstParts[0] : null;
+    for (const file of files) {
+        if (commonRoot && file.startsWith(commonRoot + '/')) {
+            // Remove the common root prefix
+            pathMap.set(file, file.substring(commonRoot.length + 1));
+        }
+        else {
+            pathMap.set(file, file);
+        }
+    }
+    return pathMap;
+}
+/**
+ * Extracts a ZIP archive and stores each file individually using the storage service.
+ *
+ * This function:
+ * 1. Extracts all files from the ZIP
+ * 2. Normalizes paths (removes common root directory if present)
+ * 3. Stores each file using the storage service with a unique base path
+ * 4. Returns the root file path and file count
+ *
+ * Files are uploaded in parallel batches for performance.
+ *
+ * @param zipBuffer - The ZIP file content as a Buffer
+ * @param storage - The storage service to use for saving files
+ * @param basePath - Base path/folder for storing extracted files (e.g., 'tilesets/1234567890')
+ * @returns ExtractedArchiveResult with root file and file count
+ *
+ * @example
+ * ```typescript
+ * const result = await extractAndStoreArchive(zipBuffer, storage, 'tilesets/1234567890')
+ * // result.root_file = 'tileset.json'
+ * // result.file_count = 42
+ *
+ * // Files are stored at:
+ * // tilesets/1234567890/tileset.json
+ * // tilesets/1234567890/tiles/tile_0.b3dm
+ * // etc.
+ * ```
+ */
+export async function extractAndStoreArchive(zipBuffer, storage, basePath) {
+    const zip = new JSZip();
+    const zipContent = await zip.loadAsync(zipBuffer);
+    // Get all file paths (excluding directories)
+    const filePaths = Object.entries(zipContent.files)
+        .filter(([_, zipObject]) => !zipObject.dir)
+        .map(([name]) => name);
+    // Normalize paths (remove common root if present)
+    const normalizedPaths = normalizeArchivePaths(filePaths);
+    // Detect root file before normalization, then get normalized path
+    const rootFileOriginal = detectTilesetRootFile(filePaths);
+    const rootFileNormalized = rootFileOriginal ? normalizedPaths.get(rootFileOriginal) : undefined;
+    // Extract and store files in parallel (batched to avoid overwhelming storage)
+    const PARALLEL_UPLOADS = 10;
+    const entries = Array.from(normalizedPaths.entries());
+    const totalFiles = entries.length;
+    let uploadedCount = 0;
+    const uploadedPaths = [];
+    console.log(`[ZipUtils] Extracting ${totalFiles} files to ${basePath}`);
+    try {
+        const totalBatches = Math.ceil(totalFiles / PARALLEL_UPLOADS);
+        // Log progress every 10% or at least every 10 batches
+        const logInterval = Math.max(1, Math.floor(totalBatches / 10));
+        for (let i = 0; i < entries.length; i += PARALLEL_UPLOADS) {
+            const batch = entries.slice(i, i + PARALLEL_UPLOADS);
+            const batchNum = Math.floor(i / PARALLEL_UPLOADS) + 1;
+            await Promise.all(batch.map(async ([originalPath, normalizedPath]) => {
+                const zipObject = zipContent.files[originalPath];
+                const content = await zipObject.async('nodebuffer');
+                // Build storage path: basePath/normalizedPath
+                const storagePath = `${basePath}/${normalizedPath}`;
+                // Save the file using saveWithPath which preserves the exact path
+                await storage.saveWithPath(content, storagePath);
+                uploadedPaths.push(storagePath);
+            }));
+            uploadedCount += batch.length;
+            // Log progress periodically (every ~10%) or on last batch
+            if (batchNum % logInterval === 0 || batchNum === totalBatches) {
+                const percent = Math.round((uploadedCount / totalFiles) * 100);
+                console.log(`[ZipUtils] Progress: ${percent}% (${uploadedCount}/${totalFiles} files)`);
+            }
+        }
+    }
+    catch (error) {
+        // Clean up any files that were already uploaded before the error
+        if (uploadedPaths.length > 0) {
+            console.log(`[ZipUtils] Error during extraction, cleaning up ${uploadedPaths.length} uploaded files...`);
+            await storage.deleteBatch(uploadedPaths).catch(cleanupErr => {
+                console.error(`[ZipUtils] Failed to clean up files after error:`, cleanupErr);
+            });
+        }
+        throw error;
+    }
+    return {
+        root_file: rootFileNormalized,
+        file_count: uploadedCount
+    };
+}
+//# sourceMappingURL=zip_utils.js.map

package/dist/utils/zip_utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip_utils.js","sourceRoot":"","sources":["../../src/utils/zip_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAA;AAazB;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAAC,SAAiB;IAC5D,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAA;IACvB,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;IAEjD,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;YACjB,mBAAmB;YACnB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAEnD,8DAA8D;YAC9D,IAAI,CAAC;gBACD,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;gBAC7C,wDAAwD;gBACxD,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;gBACjC,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACL,4CAA4C;gBAC5C,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YACzB,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAiB;IAC7C,MAAM,MAAM,GAAoC,EAAE,CAAA;IAElD,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAA;IAC1B,CAAC;IAED,OAAO,MAAM,CAAA;AACjB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAe;IACjD,yCAAyC;IACzC,MAAM,gBAAgB,GAAG;QACrB,kBAAkB;QAClB,mBAAmB;QACnB,yBAAyB,EAAE,iBAAiB;QAC5C,iBAAiB,CAAC,+BAA+B;KACpD,CAAA;IAED,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;QAC9C,IAAI,KAAK;YAAE,OAAO,KAAK,CAAA;IAC3B,CAAC;IAED,OAAO,SAAS,CAAA;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAe;IACjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEzC,mDAAmD;IACnD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAClD,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAE5G,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,EAAE,CAAC;YAClD,gCAAgC;YAChC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAC5D,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QAC3B,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAA;AAClB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CACxC,SAAiB,EACjB,OAAuB,EACvB,QAAgB;IAEhB,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAA;IACvB,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;IAEjD,6CAA6C;IAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;SAC7C,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC;SAC1C,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;IAE1B,kDAAkD;IAClD,MAAM,eAAe,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAA;IAExD,kEAAkE;IAClE,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAA;IACzD,MAAM,kBAAkB,GAAG,gBAAgB,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAE/F,8EAA8E;IAC9E,MAAM,gBAAgB,GAAG,EAAE,CAAA;IAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAA;IACrD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAA;IACjC,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,MAAM,aAAa,GAAa,EAAE,CAAA;IAElC,OAAO,CAAC,GAAG,CAAC,yBAAyB,UAAU,aAAa,QAAQ,EAAE,CAAC,CAAA;IAEvE,IAAI,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,gBAAgB,CAAC,CAAA;QAC7D,sDAAsD;QACtD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAA;QAE9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAA;YACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAA;YAErD,MAAM,OAAO,CAAC,GAAG,CACb,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,EAAE,EAAE;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;gBAChD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;gBAEnD,8CAA8C;gBAC9C,MAAM,WAAW,GAAG,GAAG,QAAQ,IAAI,cAAc,EAAE,CAAA;gBAEnD,kEAAkE;gBAClE,MAAM,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;gBAChD,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YACnC,CAAC,CAAC,CACL,CAAA;YAED,aAAa,IAAI,KAAK,CAAC,MAAM,CAAA;YAE7B,0DAA0D;YAC1D,IAAI,QAAQ,GAAG,WAAW,KAAK,CAAC,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;gBAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAA;gBAC9D,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,MAAM,aAAa,IAAI,UAAU,SAAS,CAAC,CAAA;YAC1F,CAAC;QACL,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,iEAAiE;QACjE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,mDAAmD,aAAa,CAAC,MAAM,oBAAoB,CAAC,CAAA;YACxG,MAAM,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE;gBACxD,OAAO,CAAC,KAAK,CAAC,kDAAkD,EAAE,UAAU,CAAC,CAAA;YACjF,CAAC,CAAC,CAAA;QACN,CAAC;QACD,MAAM,KAAK,CAAA;IACf,CAAC;IAED,OAAO;QACH,SAAS,EAAE,kBAAkB;QAC7B,UAAU,EAAE,aAAa;KAC5B,CAAA;AACL,CAAC"}

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@cepseudo/assets",
+  "version": "1.0.0",
+  "description": "Asset management for Digital Twin framework (AssetsManager, TilesetManager, MapManager)",
+  "license": "MIT",
+  "author": "Axel Hoffmann",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist/",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "clean": "rimraf dist tsconfig.tsbuildinfo",
+    "test": "node -r ./bin/set-test-env.cjs --import ts-node-maintained/register/esm --enable-source-maps bin/test.ts",
+    "lint": "eslint src tests --no-error-on-unmatched-pattern",
+    "lint:fix": "eslint src tests --fix --no-error-on-unmatched-pattern"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "dependencies": {
+    "@cepseudo/shared": "workspace:*",
+    "@cepseudo/database": "workspace:*",
+    "@cepseudo/storage": "workspace:*",
+    "@cepseudo/auth": "workspace:*",
+    "jszip": "^3.10.1"
+  },
+  "peerDependencies": {
+    "bullmq": ">=5.0.0"
+  },
+  "peerDependenciesMeta": {
+    "bullmq": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@japa/assert": "^4.1.0",
+    "@japa/runner": "^4.3.0",
+    "ts-node-maintained": "^10.9.5"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/CePseudoBE/digitaltwin.git",
+    "directory": "packages/assets"
+  },
+  "keywords": [
+    "digitaltwin",
+    "assets",
+    "upload",
+    "tileset",
+    "3d",
+    "presigned-url"
+  ],
+  "bugs": {
+    "url": "https://github.com/CePseudoBE/digitaltwin/issues"
+  },
+  "homepage": "https://github.com/CePseudoBE/digitaltwin#readme",
+  "publishConfig": {
+    "access": "public"
+  }
+}