digitaltwin-core 0.13.0 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/components/tileset_manager.d.ts +46 -65
- package/dist/components/tileset_manager.d.ts.map +1 -1
- package/dist/components/tileset_manager.js +415 -652
- package/dist/components/tileset_manager.js.map +1 -1
- package/dist/database/adapters/knex_database_adapter.d.ts.map +1 -1
- package/dist/database/adapters/knex_database_adapter.js +9 -12
- package/dist/database/adapters/knex_database_adapter.js.map +1 -1
- package/dist/engine/upload_processor.d.ts +7 -37
- package/dist/engine/upload_processor.d.ts.map +1 -1
- package/dist/engine/upload_processor.js +45 -92
- package/dist/engine/upload_processor.js.map +1 -1
- package/dist/storage/adapters/local_storage_service.d.ts +14 -0
- package/dist/storage/adapters/local_storage_service.d.ts.map +1 -1
- package/dist/storage/adapters/local_storage_service.js +46 -0
- package/dist/storage/adapters/local_storage_service.js.map +1 -1
- package/dist/storage/adapters/ovh_storage_service.d.ts +15 -0
- package/dist/storage/adapters/ovh_storage_service.d.ts.map +1 -1
- package/dist/storage/adapters/ovh_storage_service.js +54 -2
- package/dist/storage/adapters/ovh_storage_service.js.map +1 -1
- package/dist/storage/storage_service.d.ts +36 -0
- package/dist/storage/storage_service.d.ts.map +1 -1
- package/dist/storage/storage_service.js.map +1 -1
- package/dist/types/data_record.d.ts +22 -15
- package/dist/types/data_record.d.ts.map +1 -1
- package/dist/utils/index.d.ts +1 -1
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +1 -1
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/map_to_data_record.d.ts.map +1 -1
- package/dist/utils/map_to_data_record.js +8 -2
- package/dist/utils/map_to_data_record.js.map +1 -1
- package/dist/utils/zip_utils.d.ts +14 -30
- package/dist/utils/zip_utils.d.ts.map +1 -1
- package/dist/utils/zip_utils.js +50 -69
- package/dist/utils/zip_utils.js.map +1 -1
- package/package.json +1 -1
package/dist/utils/zip_utils.js
CHANGED
|
@@ -37,43 +37,6 @@ export async function zipToDict(zipBuffer) {
|
|
|
37
37
|
}
|
|
38
38
|
return output;
|
|
39
39
|
}
|
|
40
|
-
/**
|
|
41
|
-
* Analyzes a tileset ZIP content and extracts metadata
|
|
42
|
-
* @param zipDict - Dictionary of files from zipToDict()
|
|
43
|
-
* @returns Tileset metadata including name and file structure
|
|
44
|
-
*/
|
|
45
|
-
export function analyzeTilesetContent(zipDict) {
|
|
46
|
-
const files = Object.keys(zipDict);
|
|
47
|
-
const mainFiles = files.filter(f => f.endsWith('.json') || f.endsWith('.xml') || f.endsWith('.txt') || f.includes('metadata'));
|
|
48
|
-
// Try to extract tileset name from common patterns
|
|
49
|
-
let name = 'tileset';
|
|
50
|
-
const metadataFile = files.find(f => f.toLowerCase().includes('metadata'));
|
|
51
|
-
if (metadataFile && typeof zipDict[metadataFile] === 'string') {
|
|
52
|
-
try {
|
|
53
|
-
const metadata = JSON.parse(zipDict[metadataFile]);
|
|
54
|
-
name = metadata.name || metadata.title || name;
|
|
55
|
-
}
|
|
56
|
-
catch {
|
|
57
|
-
// Ignore parsing errors
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
// Fallback: use first directory name or first file prefix
|
|
61
|
-
if (name === 'tileset') {
|
|
62
|
-
const firstFile = files[0];
|
|
63
|
-
if (firstFile && firstFile.includes('/')) {
|
|
64
|
-
name = firstFile.split('/')[0];
|
|
65
|
-
}
|
|
66
|
-
else if (firstFile && firstFile.includes('.')) {
|
|
67
|
-
name = firstFile.split('.')[0];
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
return {
|
|
71
|
-
name,
|
|
72
|
-
fileCount: files.length,
|
|
73
|
-
hasMetadata: mainFiles.length > 0,
|
|
74
|
-
mainFiles
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
40
|
/**
|
|
78
41
|
* Detects the root file for 3D Tiles tilesets
|
|
79
42
|
* Looks for tileset.json or similar entry point files
|
|
@@ -99,7 +62,7 @@ export function detectTilesetRootFile(files) {
|
|
|
99
62
|
* Normalizes file paths from ZIP archives
|
|
100
63
|
* Removes leading directory if all files share the same root folder
|
|
101
64
|
* @param files - Original file paths from the archive
|
|
102
|
-
* @returns Normalized file paths
|
|
65
|
+
* @returns Normalized file paths (original -> normalized)
|
|
103
66
|
*/
|
|
104
67
|
export function normalizeArchivePaths(files) {
|
|
105
68
|
const pathMap = new Map();
|
|
@@ -124,21 +87,25 @@ export function normalizeArchivePaths(files) {
|
|
|
124
87
|
* 1. Extracts all files from the ZIP
|
|
125
88
|
* 2. Normalizes paths (removes common root directory if present)
|
|
126
89
|
* 3. Stores each file using the storage service with a unique base path
|
|
127
|
-
* 4. Returns
|
|
90
|
+
* 4. Returns the root file path and file count
|
|
91
|
+
*
|
|
92
|
+
* Files are uploaded in parallel batches for performance.
|
|
128
93
|
*
|
|
129
94
|
* @param zipBuffer - The ZIP file content as a Buffer
|
|
130
95
|
* @param storage - The storage service to use for saving files
|
|
131
|
-
* @param basePath - Base path/folder for storing extracted files (e.g., '
|
|
132
|
-
* @returns ExtractedArchiveResult with file
|
|
96
|
+
* @param basePath - Base path/folder for storing extracted files (e.g., 'tilesets/1234567890')
|
|
97
|
+
* @returns ExtractedArchiveResult with root file and file count
|
|
133
98
|
*
|
|
134
99
|
* @example
|
|
135
100
|
* ```typescript
|
|
136
|
-
* const result = await extractAndStoreArchive(zipBuffer, storage, '
|
|
137
|
-
* // result.files = [
|
|
138
|
-
* // { path: 'tilesets_manager/123/tileset.json', name: 'tileset.json', size: 1024 },
|
|
139
|
-
* // { path: 'tilesets_manager/123/tiles/tile_0.b3dm', name: 'tiles/tile_0.b3dm', size: 50000 }
|
|
140
|
-
* // ]
|
|
101
|
+
* const result = await extractAndStoreArchive(zipBuffer, storage, 'tilesets/1234567890')
|
|
141
102
|
* // result.root_file = 'tileset.json'
|
|
103
|
+
* // result.file_count = 42
|
|
104
|
+
*
|
|
105
|
+
* // Files are stored at:
|
|
106
|
+
* // tilesets/1234567890/tileset.json
|
|
107
|
+
* // tilesets/1234567890/tiles/tile_0.b3dm
|
|
108
|
+
* // etc.
|
|
142
109
|
* ```
|
|
143
110
|
*/
|
|
144
111
|
export async function extractAndStoreArchive(zipBuffer, storage, basePath) {
|
|
@@ -150,39 +117,53 @@ export async function extractAndStoreArchive(zipBuffer, storage, basePath) {
|
|
|
150
117
|
.map(([name]) => name);
|
|
151
118
|
// Normalize paths (remove common root if present)
|
|
152
119
|
const normalizedPaths = normalizeArchivePaths(filePaths);
|
|
153
|
-
// Detect root file before normalization
|
|
120
|
+
// Detect root file before normalization, then get normalized path
|
|
154
121
|
const rootFileOriginal = detectTilesetRootFile(filePaths);
|
|
155
122
|
const rootFileNormalized = rootFileOriginal ? normalizedPaths.get(rootFileOriginal) : undefined;
|
|
156
|
-
|
|
157
|
-
// Extract and store files in parallel (batched to avoid overwhelming OVH)
|
|
123
|
+
// Extract and store files in parallel (batched to avoid overwhelming storage)
|
|
158
124
|
const PARALLEL_UPLOADS = 10;
|
|
159
125
|
const entries = Array.from(normalizedPaths.entries());
|
|
160
126
|
const totalFiles = entries.length;
|
|
127
|
+
let uploadedCount = 0;
|
|
128
|
+
const uploadedPaths = [];
|
|
161
129
|
console.log(`[ZipUtils] Extracting ${totalFiles} files to ${basePath}`);
|
|
162
|
-
|
|
163
|
-
const batch = entries.slice(i, i + PARALLEL_UPLOADS);
|
|
164
|
-
const batchNum = Math.floor(i / PARALLEL_UPLOADS) + 1;
|
|
130
|
+
try {
|
|
165
131
|
const totalBatches = Math.ceil(totalFiles / PARALLEL_UPLOADS);
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
path:
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
132
|
+
// Log progress every 10% or at least every 10 batches
|
|
133
|
+
const logInterval = Math.max(1, Math.floor(totalBatches / 10));
|
|
134
|
+
for (let i = 0; i < entries.length; i += PARALLEL_UPLOADS) {
|
|
135
|
+
const batch = entries.slice(i, i + PARALLEL_UPLOADS);
|
|
136
|
+
const batchNum = Math.floor(i / PARALLEL_UPLOADS) + 1;
|
|
137
|
+
await Promise.all(batch.map(async ([originalPath, normalizedPath]) => {
|
|
138
|
+
const zipObject = zipContent.files[originalPath];
|
|
139
|
+
const content = await zipObject.async('nodebuffer');
|
|
140
|
+
// Build storage path: basePath/normalizedPath
|
|
141
|
+
const storagePath = `${basePath}/${normalizedPath}`;
|
|
142
|
+
// Save the file using saveWithPath which preserves the exact path
|
|
143
|
+
await storage.saveWithPath(content, storagePath);
|
|
144
|
+
uploadedPaths.push(storagePath);
|
|
145
|
+
}));
|
|
146
|
+
uploadedCount += batch.length;
|
|
147
|
+
// Log progress periodically (every ~10%) or on last batch
|
|
148
|
+
if (batchNum % logInterval === 0 || batchNum === totalBatches) {
|
|
149
|
+
const percent = Math.round((uploadedCount / totalFiles) * 100);
|
|
150
|
+
console.log(`[ZipUtils] Progress: ${percent}% (${uploadedCount}/${totalFiles} files)`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
// Clean up any files that were already uploaded before the error
|
|
156
|
+
if (uploadedPaths.length > 0) {
|
|
157
|
+
console.log(`[ZipUtils] Error during extraction, cleaning up ${uploadedPaths.length} uploaded files...`);
|
|
158
|
+
await storage.deleteBatch(uploadedPaths).catch(cleanupErr => {
|
|
159
|
+
console.error(`[ZipUtils] Failed to clean up files after error:`, cleanupErr);
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
throw error;
|
|
181
163
|
}
|
|
182
164
|
return {
|
|
183
|
-
files: storedFiles,
|
|
184
165
|
root_file: rootFileNormalized,
|
|
185
|
-
file_count:
|
|
166
|
+
file_count: uploadedCount
|
|
186
167
|
};
|
|
187
168
|
}
|
|
188
169
|
//# sourceMappingURL=zip_utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"zip_utils.js","sourceRoot":"","sources":["../../src/utils/zip_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAA;
|
|
1
|
+
{"version":3,"file":"zip_utils.js","sourceRoot":"","sources":["../../src/utils/zip_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAA;AAazB;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAAC,SAAiB;IAC5D,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAA;IACvB,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;IAEjD,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/D,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;YACjB,mBAAmB;YACnB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAEnD,8DAA8D;YAC9D,IAAI,CAAC;gBACD,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;gBAC7C,wDAAwD;gBACxD,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;gBACjC,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACL,4CAA4C;gBAC5C,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YACzB,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAiB;IAC7C,MAAM,MAAM,GAAoC,EAAE,CAAA;IAElD,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAA;IAC1B,CAAC;IAED,OAAO,MAAM,CAAA;AACjB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAe;IACjD,yCAAyC;IACzC,MAAM,gBAAgB,GAAG;QACrB,kBAAkB;QAClB,mBAAmB;QACnB,yBAAyB,EAAE,iBAAiB;QAC5C,iBAAiB,CAAC,+BAA+B;KACpD,CAAA;IAED,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;QAC9C,IAAI,KAAK;YAAE,OAAO,KAAK,CAAA;IAC3B,CAAC;IAED,OAAO,SAAS,CAAA;AACpB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAe;IACjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEzC,mDAAmD;IACnD,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAClD,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAE5G,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,EAAE,CAAC;YAClD,gCAAgC;YAChC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAC5D,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QAC3B,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAA;AAClB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CACxC,SAAiB,EACjB,OAAuB,EACvB,QAAgB;IAEhB,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAA;IACvB,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;IAEjD,6CAA6C;IAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;SAC7C,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC;SAC1C,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;IAE1B,kDAAkD;IAClD,MAAM,eAAe,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAA;IAExD,kEAAkE;IAClE,MAAM,gBAAgB,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAA;IACzD,MAAM,kBAAkB,GAAG,gBAAgB,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAE/F,8EAA8E;IAC9E,MAAM,gBAAgB,GAAG,EAAE,CAAA;IAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAA;IACrD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAA;IACjC,IAAI,aAAa,GAAG,CAAC,CAAA;IACrB,MAAM,aAAa,GAAa,EAAE,CAAA;IAElC,OAAO,CAAC,GAAG,CAAC,yBAAyB,UAAU,aAAa,QAAQ,EAAE,CAAC,CAAA;IAEvE,IAAI,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,gBAAgB,CAAC,CAAA;QAC7D,sDAAsD;QACtD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAA;QAE9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAA;YACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAA;YAErD,MAAM,OAAO,CAAC,GAAG,CACb,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,EAAE,EAAE;gBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;gBAChD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;gBAEnD,8CAA8C;gBAC9C,MAAM,WAAW,GAAG,GAAG,QAAQ,IAAI,cAAc,EAAE,CAAA;gBAEnD,kEAAkE;gBAClE,MAAM,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;gBAChD,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YACnC,CAAC,CAAC,CACL,CAAA;YAED,aAAa,IAAI,KAAK,CAAC,MAAM,CAAA;YAE7B,0DAA0D;YAC1D,IAAI,QAAQ,GAAG,WAAW,KAAK,CAAC,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;gBAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAA;gBAC9D,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,MAAM,aAAa,IAAI,UAAU,SAAS,CAAC,CAAA;YAC1F,CAAC;QACL,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,iEAAiE;QACjE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,mDAAmD,aAAa,CAAC,MAAM,oBAAoB,CAAC,CAAA;YACxG,MAAM,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE;gBACxD,OAAO,CAAC,KAAK,CAAC,kDAAkD,EAAE,UAAU,CAAC,CAAA;YACjF,CAAC,CAAC,CAAA;QACN,CAAC;QACD,MAAM,KAAK,CAAA;IACf,CAAC;IAED,OAAO;QACH,SAAS,EAAE,kBAAkB;QAC7B,UAAU,EAAE,aAAa;KAC5B,CAAA;AACL,CAAC"}
|