@loaders.gl/tile-converter 3.3.0-alpha.7 → 3.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "3.3.0-alpha.7",
+  "version": "3.3.0",
   "description": "Converter",
   "license": "MIT",
   "publishConfig": {
@@ -33,29 +33,30 @@
     "fs": false,
     "path": false,
     "process": false,
-    "archiver": false
+    "archiver": false,
+    "join-images": false
   },
   "scripts": {
     "pre-build": "npm run build-bundle && npm run build-converter-bundle && npm run build-i3s-attributes-worker && npm run build-3d-tiles-attributes-worker",
-    "build-bundle": "esbuild ./src/index.ts --bundle --outfile=dist/dist.min.js --platform=node",
+    "build-bundle": "esbuild ./src/index.ts --bundle --outfile=dist/dist.min.js --platform=node  --external:join-images",
     "build-converter-bundle": "webpack --display errors-only --config ./converter-webpack/bundle.js",
-    "build-i3s-attributes-worker": "esbuild src/workers/i3s-attributes-worker.ts --outfile=dist/i3s-attributes-worker.js --platform=node --target=esnext,node12 --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\"",
-    "build-3d-tiles-attributes-worker": "esbuild src/workers/3d-tiles-attributes-worker.ts --outfile=dist/3d-tiles-attributes-worker.js --platform=node --target=esnext,node12 --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
+    "build-i3s-attributes-worker": "esbuild src/workers/i3s-attributes-worker.ts --outfile=dist/i3s-attributes-worker.js --platform=node --target=esnext,node12 --external:join-images --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-3d-tiles-attributes-worker": "esbuild src/workers/3d-tiles-attributes-worker.ts --outfile=dist/3d-tiles-attributes-worker.js --platform=node --target=esnext,node12 --external:join-images --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "3.3.0-alpha.7",
-    "@loaders.gl/crypto": "3.3.0-alpha.7",
-    "@loaders.gl/draco": "3.3.0-alpha.7",
-    "@loaders.gl/gltf": "3.3.0-alpha.7",
-    "@loaders.gl/i3s": "3.3.0-alpha.7",
-    "@loaders.gl/images": "3.3.0-alpha.7",
-    "@loaders.gl/loader-utils": "3.3.0-alpha.7",
-    "@loaders.gl/polyfills": "3.3.0-alpha.7",
-    "@loaders.gl/schema": "3.3.0-alpha.7",
-    "@loaders.gl/textures": "3.3.0-alpha.7",
-    "@loaders.gl/tiles": "3.3.0-alpha.7",
-    "@loaders.gl/worker-utils": "3.3.0-alpha.7",
-    "@loaders.gl/zip": "3.3.0-alpha.7",
+    "@loaders.gl/3d-tiles": "3.3.0",
+    "@loaders.gl/crypto": "3.3.0",
+    "@loaders.gl/draco": "3.3.0",
+    "@loaders.gl/gltf": "3.3.0",
+    "@loaders.gl/i3s": "3.3.0",
+    "@loaders.gl/images": "3.3.0",
+    "@loaders.gl/loader-utils": "3.3.0",
+    "@loaders.gl/polyfills": "3.3.0",
+    "@loaders.gl/schema": "3.3.0",
+    "@loaders.gl/textures": "3.3.0",
+    "@loaders.gl/tiles": "3.3.0",
+    "@loaders.gl/worker-utils": "3.3.0",
+    "@loaders.gl/zip": "3.3.0",
     "@luma.gl/engine": "^8.5.4",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
@@ -65,7 +66,7 @@
     "cors": "^2.8.5",
     "crypt": "^0.0.2",
     "debug": "~2.6.9",
-    "express": "~4.16.1",
+    "express": "~4.17.3",
     "json-map-transform": "^1.2.6",
     "jszip": "^3.5.0",
     "md5": "^2.3.0",
@@ -75,5 +76,9 @@
   "peerDependencies": {
     "@loaders.gl/core": "^3.2.0"
   },
-  "gitHead": "29b08f3519c50984e84bf4234e607cab7c7d1c3e"
+  "quarantinedDependencies": {
+    "join-images": "^1.1.3",
+    "sharp": "^0.31.3"
+  },
+  "gitHead": "566d194f175a03e042d3e588cdd514048f428cc1"
 }

package/src/converter-cli.ts

@@ -14,6 +14,10 @@ type TileConversionOptions = {
   /** Output folder. This folder will be created by converter if doesn't exist. It is relative to the converter path.
    * Default: "data" folder */
   output: string;
+  /** Keep created 3DNodeIndexDocument files on disk instead of memory. This option reduce memory usage but decelerates conversion speed */
+  instantNodeWriting: boolean;
+  /** Try to merge similar materials to be able to merge meshes into one node (I3S to 3DTiles conversion only) */
+  mergeMaterials: boolean;
   /** 3DTiles->I3S only. location of 7z.exe archiver to create slpk on Windows OS, default: "C:\Program Files\7-Zip\7z.exe" */
   sevenZipExe: string;
   /** location of the Earth Gravity Model (*.pgm) file to convert heights from ellipsoidal to gravity-related format,
@@ -99,6 +103,12 @@ function printHelp(): void {
   );
   console.log('--name [Tileset name]');
   console.log('--output [Output folder, default: "data" folder]');
+  console.log(
+    '--instant-node-writing [Keep created 3DNodeIndexDocument files on disk instead of memory. This option reduce memory usage but decelerates conversion speed]'
+  );
+  console.log(
+    '--split-nodes [Prevent to merge similar materials that could lead to incorrect visualization (I3S to 3DTiles conversion only)]'
+  );
   console.log('--slpk [Generate slpk (Scene Layer Packages) I3S output file]');
   console.log(
     '--tileset [tileset.json file (3DTiles) / http://..../SceneServer/layers/0 resource (I3S)]'
@@ -154,9 +164,11 @@ async function convert(options: ValidatedTileConversionOptions) {
         egmFilePath: options.egm,
         token: options.token,
         draco: options.draco,
+        mergeMaterials: options.mergeMaterials,
         generateTextures: options.generateTextures,
         generateBoundingVolumes: options.generateBoundingVolumes,
-        validate: options.validate
+        validate: options.validate,
+        instantNodeWriting: options.instantNodeWriting
       });
       break;
     default:
@@ -222,6 +234,8 @@ function validateOptionsWithEqual(args: string[]): string[] {
 function parseOptions(args: string[]): TileConversionOptions {
   const opts: TileConversionOptions = {
     output: 'data',
+    instantNodeWriting: false,
+    mergeMaterials: true,
     sevenZipExe: 'C:\\Program Files\\7-Zip\\7z.exe',
     egm: join(process.cwd(), 'deps', 'egm2008-5.pgm'),
     draco: true,
@@ -248,6 +262,12 @@ function parseOptions(args: string[]): TileConversionOptions {
         case '--output':
           opts.output = getStringValue(index, args);
           break;
+        case '--instant-node-writing':
+          opts.instantNodeWriting = getBooleanValue(index, args);
+          break;
+        case '--split-nodes':
+          opts.mergeMaterials = getBooleanValue(index, args);
+          break;
         case '--max-depth':
           opts.maxDepth = getIntegerValue(index, args);
           break;
@@ -340,7 +360,7 @@ function getIntegerValue(index: number, args: string[]): number {
 
 function getBooleanValue(index: number, args: string[]): boolean {
   const stringValue: string = getStringValue(index, args).toLowerCase().trim();
-  if (args[index] === '--no-draco' && !stringValue) {
+  if (['--no-draco', '--split-nodes'].includes(args[index]) && !stringValue) {
     return false;
   }
   if (!stringValue || stringValue === 'true') {

package/src/deps-installer/deps-installer.ts

@@ -2,6 +2,7 @@ import {load, fetchFile} from '@loaders.gl/core';
 import {ZipLoader} from '@loaders.gl/zip';
 import {writeFile} from '../lib/utils/file-utils';
 import {join} from 'path';
+import {ChildProcessProxy} from '@loaders.gl/worker-utils';
 
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'beta';
@@ -42,6 +43,14 @@ export class DepsInstaller {
     console.log('Installing "Basis Loader worker"'); // eslint-disable-line no-console
     await this.installWorker('textures', 'basis-nodejs-worker.js', workersPath);
 
+    console.log('Installing "join-images" npm package');
+    const childProcess = new ChildProcessProxy();
+    await childProcess.start({
+      command: 'npm',
+      arguments: ['install', 'sharp', 'join-images'],
+      wait: 0
+    });
+
     console.log('All dependencies were installed succesfully.'); // eslint-disable-line no-console
   }
 

package/src/i3s-converter/helpers/batch-ids-extensions.ts

@@ -13,9 +13,9 @@ const EXT_FEATURE_METADATA = 'EXT_feature_metadata';
 
 /**
  * Getting batchIds from 3DTilesNext extensions.
- * @param attributes
- * @param primitive
- * @param textures
+ * @param attributes - gltf accessors
+ * @param primitive - gltf primitive data
+ * @param images - gltf texture images
  */
 export function handleBatchIdsExtensions(
   attributes: {

package/src/i3s-converter/helpers/geometry-attributes.ts

@@ -10,7 +10,7 @@ const POSITIONS_AND_NORMALS_PER_TRIANGLE = 9;
  * @returns attirbutes with featureCount, featureIds and changed faceRange.
  */
 export function generateAttributes(attributes: ConvertedAttributes): GeometryAttributes {
-  const {positions, normals, texCoords, colors, featureIndices} = attributes;
+  const {positions, normals, texCoords, colors, uvRegions, featureIndices} = attributes;
   const triangleCount = positions.length / POSITIONS_AND_NORMALS_PER_TRIANGLE;
 
   if (!featureIndices.length) {
@@ -21,7 +21,8 @@ export function generateAttributes(attributes: ConvertedAttributes): GeometryAtt
       positions,
       normals,
       texCoords,
-      colors
+      colors,
+      uvRegions
     };
   }
 
@@ -113,6 +114,7 @@ function makeAttributeObjects(attributes: GeometryAttributes): GroupedByFeatureI
     positions,
     normals,
     colors,
+    uvRegions,
     texCoords,
     faceRange = new Uint32Array(0)
   } = attributes;
@@ -122,16 +124,16 @@ function makeAttributeObjects(attributes: GeometryAttributes): GroupedByFeatureI
   let normalsList = new Float32Array(normals);
   let colorsList = new Uint8Array(colors);
   let texCoordsList = new Float32Array(texCoords);
-
-  let faceRangeIndex = 0;
+  let uvRegionsList = new Uint16Array(uvRegions);
 
   for (let index = 0; index < featureIds.length; index++) {
-    const startIndex = faceRange[index + faceRangeIndex];
-    const endIndex = faceRange[index + faceRangeIndex + 1];
+    const startIndex = faceRange[index * 2];
+    const endIndex = faceRange[index * 2 + 1];
 
     const positionsCount = getSliceAttributeCount('positions', startIndex, endIndex);
     const normalsCount = getSliceAttributeCount('normals', startIndex, endIndex);
     const colorsCount = getSliceAttributeCount('colors', startIndex, endIndex);
+    const uvRegionsCount = getSliceAttributeCount('uvRegions', startIndex, endIndex);
     const texCoordsCount = getSliceAttributeCount('texCoords', startIndex, endIndex);
 
     groupedData.push({
@@ -139,15 +141,15 @@ function makeAttributeObjects(attributes: GeometryAttributes): GroupedByFeatureI
       positions: positionsList.slice(0, positionsCount),
       normals: normalsList.slice(0, normalsCount),
       colors: colorsList.slice(0, colorsCount),
+      uvRegions: uvRegionsList.slice(0, uvRegionsCount),
       texCoords: texCoordsList.slice(0, texCoordsCount)
     });
 
     positionsList = positionsList.slice(positionsCount);
     normalsList = normalsList.slice(normalsCount);
     colorsList = colorsList.slice(colorsCount);
+    uvRegionsList = uvRegionsList.slice(uvRegionsCount);
     texCoordsList = texCoordsList.slice(texCoordsCount);
-
-    faceRangeIndex += 1;
   }
 
   return groupedData.sort((first, second) => first.featureId - second.featureId);
@@ -165,7 +167,7 @@ function getSliceAttributeCount(
   startIndex: number,
   endIndex: number
 ): number {
-  const colorsPerVertex = 4;
+  const itemsPerVertex4 = 4;
   const texCoordsPerVertex = 2;
 
   const trianglesCount = endIndex - startIndex + 1;
@@ -176,7 +178,8 @@ function getSliceAttributeCount(
     case 'normals':
       return trianglesCount * POSITIONS_AND_NORMALS_PER_TRIANGLE;
     case 'colors':
-      return vertexCount * colorsPerVertex;
+    case 'uvRegions':
+      return vertexCount * itemsPerVertex4;
     case 'texCoords':
       return vertexCount * texCoordsPerVertex;
     default:
@@ -232,6 +235,7 @@ function groupAttributesAndRangesByFeatureId(
   let positions = new Float32Array(firstAttributeObject.positions);
   let normals = new Float32Array(firstAttributeObject.normals);
   let colors = new Uint8Array(firstAttributeObject.colors);
+  let uvRegions = new Uint16Array(firstAttributeObject.uvRegions);
   let texCoords = new Float32Array(firstAttributeObject.texCoords);
   const range = [0];
 
@@ -245,6 +249,7 @@ function groupAttributesAndRangesByFeatureId(
     positions = concatenateTypedArrays(positions, currentAttributesObject.positions);
     normals = concatenateTypedArrays(normals, currentAttributesObject.normals);
     colors = concatenateTypedArrays(colors, currentAttributesObject.colors);
+    uvRegions = concatenateTypedArrays(uvRegions, currentAttributesObject.uvRegions);
     texCoords = concatenateTypedArrays(texCoords, currentAttributesObject.texCoords);
 
     const groupedObject = unifiedObjects[objIndex];
@@ -258,5 +263,5 @@ function groupAttributesAndRangesByFeatureId(
   range.push(positions.length / POSITIONS_AND_NORMALS_PER_TRIANGLE - 1);
 
   const faceRange = new Uint32Array(range);
-  return {faceRange, featureIds, positions, normals, colors, texCoords, featureCount};
+  return {faceRange, featureIds, positions, normals, colors, uvRegions, texCoords, featureCount};
 }