@loaders.gl/tile-converter 3.2.12 → 3.3.0-alpha.10

package/dist/workers/i3s-attributes-worker.js

@@ -2,4 +2,4 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const worker_utils_1 = require("@loaders.gl/worker-utils");
 const geometry_converter_1 = require("../i3s-converter/helpers/geometry-converter");
-(0, worker_utils_1.createWorker)(async (data, options = {}) => await (0, geometry_converter_1.convertAttributes)(data, options.useCartesianPositions));
+(0, worker_utils_1.createWorker)(async (data, options = {}) => await (0, geometry_converter_1.convertAttributes)(data, options.materialAndTextureList, options.useCartesianPositions));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "3.2.12",
+  "version": "3.3.0-alpha.10",
   "description": "Converter",
   "license": "MIT",
   "publishConfig": {
@@ -43,19 +43,19 @@
     "build-3d-tiles-attributes-worker": "esbuild src/workers/3d-tiles-attributes-worker.ts --outfile=dist/3d-tiles-attributes-worker.js --platform=node --target=esnext,node12 --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "3.2.12",
-    "@loaders.gl/crypto": "3.2.12",
-    "@loaders.gl/draco": "3.2.12",
-    "@loaders.gl/gltf": "3.2.12",
-    "@loaders.gl/i3s": "3.2.12",
-    "@loaders.gl/images": "3.2.12",
-    "@loaders.gl/loader-utils": "3.2.12",
-    "@loaders.gl/polyfills": "3.2.12",
-    "@loaders.gl/schema": "3.2.12",
-    "@loaders.gl/textures": "3.2.12",
-    "@loaders.gl/tiles": "3.2.12",
-    "@loaders.gl/worker-utils": "3.2.12",
-    "@loaders.gl/zip": "3.2.12",
+    "@loaders.gl/3d-tiles": "3.3.0-alpha.10",
+    "@loaders.gl/crypto": "3.3.0-alpha.10",
+    "@loaders.gl/draco": "3.3.0-alpha.10",
+    "@loaders.gl/gltf": "3.3.0-alpha.10",
+    "@loaders.gl/i3s": "3.3.0-alpha.10",
+    "@loaders.gl/images": "3.3.0-alpha.10",
+    "@loaders.gl/loader-utils": "3.3.0-alpha.10",
+    "@loaders.gl/polyfills": "3.3.0-alpha.10",
+    "@loaders.gl/schema": "3.3.0-alpha.10",
+    "@loaders.gl/textures": "3.3.0-alpha.10",
+    "@loaders.gl/tiles": "3.3.0-alpha.10",
+    "@loaders.gl/worker-utils": "3.3.0-alpha.10",
+    "@loaders.gl/zip": "3.3.0-alpha.10",
     "@luma.gl/engine": "^8.5.4",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
@@ -65,15 +65,17 @@
     "cors": "^2.8.5",
     "crypt": "^0.0.2",
     "debug": "~2.6.9",
-    "express": "~4.16.1",
+    "express": "~4.17.3",
+    "join-images": "^1.1.3",
     "json-map-transform": "^1.2.6",
     "jszip": "^3.5.0",
     "md5": "^2.3.0",
     "morgan": "~1.9.1",
+    "sharp": "^0.31.3",
     "uuid": "^8.1.0"
   },
   "peerDependencies": {
     "@loaders.gl/core": "^3.2.0"
   },
-  "gitHead": "7d46e4ff10047e479374e9b3411f5142a79df15f"
+  "gitHead": "53bc8d3f18fbc3445a00f8509038479119895c78"
 }

package/src/3d-tiles-attributes-worker.ts

@@ -26,7 +26,7 @@ export const Tile3dAttributesWorker = {
   module: 'tile-converter',
   version: VERSION,
   options: {
-    attributes: null
+    featureAttributes: null
   }
 };
 

package/src/3d-tiles-converter/3d-tiles-converter.ts

@@ -15,12 +15,11 @@ import {convertScreenThresholdToGeometricError} from '../lib/utils/lod-conversio
 import {writeFile, removeDir} from '../lib/utils/file-utils';
 import {calculateFilesSize, timeConverter} from '../lib/utils/statistic-utills';
 import {TILESET as tilesetTemplate} from './json-templates/tileset';
-import B3dmConverter from './helpers/b3dm-converter';
 import {createObbFromMbs} from '../i3s-converter/helpers/coordinate-converter';
 import {
   I3SAttributesData,
-  Tile3dAttributesWorker
-  /*transform3DTilesAttributesOnWorker*/
+  Tile3dAttributesWorker,
+  transform3DTilesAttributesOnWorker
 } from '../3d-tiles-attributes-worker';
 import {getWorkerURL, WorkerFarm} from '@loaders.gl/worker-utils';
 import {BROWSER_ERROR_MESSAGE} from '../constants';
@@ -85,6 +84,11 @@ export default class Tiles3DConverter {
 
     this.sourceTileset = new Tileset3D(sourceTilesetJson, {
       loadOptions: {
+        _nodeWorkers: true,
+        reuseWorkers: true,
+        'i3s-content-nodejs': {
+          workerUrl: './modules/i3s/dist/i3s-content-nodejs-worker.js'
+        },
         i3s: {coordinateSystem: COORDINATE_SYSTEM.LNGLAT_OFFSETS, decodeTextures: false}
       }
     });
@@ -124,6 +128,67 @@ export default class Tiles3DConverter {
     workerFarm.destroy();
   }
 
+  /**
+   * Convert particular I3S Node
+   * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
+   * @param parentNode object in resulting tileset
+   * @param level a current level of a tree depth
+   * @param childNodeInfo child node to convert
+   */
+  private async convertChildNode(
+    parentSourceNode: Tile3D,
+    parentNode: Node3D,
+    level: number,
+    childNodeInfo: NodeReference
+  ): Promise<void> {
+    const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
+    parentSourceNode.children.push(sourceChild);
+    if (sourceChild.contentUrl) {
+      await this.sourceTileset!._loadTile(sourceChild);
+      this.vertexCounter += sourceChild.content.vertexCount;
+
+      let featureAttributes: FeatureAttribute | null = null;
+      if (this.attributeStorageInfo) {
+        featureAttributes = await this._loadChildAttributes(sourceChild, this.attributeStorageInfo);
+      }
+
+      if (!sourceChild.header.obb) {
+        sourceChild.header.obb = createObbFromMbs(sourceChild.header.mbs);
+      }
+
+      const boundingVolume = {
+        box: i3sObbTo3dTilesObb(sourceChild.header.obb, this.geoidHeightModel)
+      };
+      const child: Node3D = {
+        boundingVolume,
+        geometricError: convertScreenThresholdToGeometricError(sourceChild),
+        children: []
+      };
+
+      const i3sAttributesData: I3SAttributesData = {
+        tileContent: sourceChild.content,
+        textureFormat: sourceChild?.header?.textureFormat
+      };
+
+      const b3dm = await transform3DTilesAttributesOnWorker(i3sAttributesData, {
+        source: this.workerSource.tile3dWorkerSource,
+        featureAttributes
+      });
+
+      child.content = {
+        uri: `${sourceChild.id}.b3dm`,
+        boundingVolume
+      };
+      await writeFile(this.tilesetPath, new Uint8Array(b3dm), `${sourceChild.id}.b3dm`);
+      parentNode.children.push(child);
+
+      sourceChild.unloadContent();
+      await this._addChildren(sourceChild, child, level + 1);
+    } else {
+      await this._addChildren(sourceChild, parentNode, level + 1);
+    }
+  }
+
   /**
    * The recursive function of traversal of a nodes tree
    * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
@@ -138,60 +203,11 @@ export default class Tiles3DConverter {
     if (this.options.maxDepth && level > this.options.maxDepth) {
       return;
     }
+    const promises: Promise<void>[] = [];
     for (const childNodeInfo of parentSourceNode.header.children || []) {
-      const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
-      parentSourceNode.children.push(sourceChild);
-      if (sourceChild.contentUrl) {
-        await this.sourceTileset!._loadTile(sourceChild);
-        this.vertexCounter += sourceChild.content.vertexCount;
-
-        let featureAttributes: FeatureAttribute | null = null;
-        if (this.attributeStorageInfo) {
-          featureAttributes = await this._loadChildAttributes(
-            sourceChild,
-            this.attributeStorageInfo
-          );
-        }
-
-        if (!sourceChild.header.obb) {
-          sourceChild.header.obb = createObbFromMbs(sourceChild.header.mbs);
-        }
-
-        const boundingVolume = {
-          box: i3sObbTo3dTilesObb(sourceChild.header.obb, this.geoidHeightModel)
-        };
-        const child: Node3D = {
-          boundingVolume,
-          geometricError: convertScreenThresholdToGeometricError(sourceChild),
-          children: []
-        };
-
-        const i3sAttributesData: I3SAttributesData = {
-          tileContent: sourceChild.content,
-          textureFormat: sourceChild?.header?.textureFormat
-        };
-
-        // TODO Uncomment when 3d-tiles-attributes-worker will be published on CDN.
-        // const b3dm = await transform3DTilesAttributesOnWorker(i3sAttributesData, {
-        //   source: this.workerSource.tile3dWorkerSource,
-        //   featureAttributes
-        // });
-
-        const b3dm = await new B3dmConverter().convert(i3sAttributesData, featureAttributes);
-
-        child.content = {
-          uri: `${sourceChild.id}.b3dm`,
-          boundingVolume
-        };
-        await writeFile(this.tilesetPath, new Uint8Array(b3dm), `${sourceChild.id}.b3dm`);
-        parentNode.children.push(child);
-
-        sourceChild.unloadContent();
-        await this._addChildren(sourceChild, child, level + 1);
-      } else {
-        await this._addChildren(sourceChild, parentNode, level + 1);
-      }
+      promises.push(this.convertChildNode(parentSourceNode, parentNode, level, childNodeInfo));
     }
+    await Promise.all(promises);
   }
 
   /**

package/src/3d-tiles-converter/helpers/b3dm-converter.ts

@@ -27,7 +27,7 @@ export default class B3dmConverter {
     i3sAttributesData: I3SAttributesData,
     featureAttributes: any = null
   ): Promise<ArrayBuffer> {
-    const gltf = await this.buildGltf(i3sAttributesData);
+    const gltf = await this.buildGltf(i3sAttributesData, featureAttributes);
     const b3dm = encodeSync(
       {
         gltfEncoded: new Uint8Array(gltf),
@@ -45,7 +45,10 @@ export default class B3dmConverter {
    * @param i3sTile - Tile3D instance for I3S node
    * @returns - encoded glb content
    */
-  async buildGltf(i3sAttributesData: I3SAttributesData): Promise<ArrayBuffer> {
+  async buildGltf(
+    i3sAttributesData: I3SAttributesData,
+    featureAttributes: any
+  ): Promise<ArrayBuffer> {
     const {tileContent, textureFormat} = i3sAttributesData;
     const {
       material,
@@ -77,6 +80,7 @@ export default class B3dmConverter {
       cartographicOrigin,
       modelMatrix
     );
+    this._createBatchIds(tileContent, featureAttributes);
     if (attributes.normals && !this._checkNormals(attributes.normals.value)) {
       delete attributes.normals;
     }
@@ -163,25 +167,28 @@ export default class B3dmConverter {
   }
 
   /**
-   * Generate batchId attribute from faceRanges.
-   * @param {Uint32Array} faceRanges - the source array
-   * @returns {Float32Array} batchId list.
+   * Create _BATCHID attribute
+   * @param {Object} i3sContent - the source object
+   * @returns {void}
    */
-  _generateBatchId(faceRanges) {
-    const batchIdArraySize = (faceRanges[faceRanges.length - 1] + 1) * 3;
-    const batchId = new Float32Array(batchIdArraySize);
-    let rangeIndex = 0;
-    let currentBatchId = 0;
-
-    for (let index = 0; index < faceRanges.length / 2; index++) {
-      const fromIndex = faceRanges[rangeIndex] * 3;
-      const untilPosition = (faceRanges[rangeIndex + 1] + 1) * 3;
+  _createBatchIds(i3sContent, featureAttributes) {
+    const {featureIds} = i3sContent;
+    const {OBJECTID: objectIds} = featureAttributes || {};
+    if (!featureIds || !objectIds) {
+      return;
+    }
 
-      batchId.fill(currentBatchId, fromIndex, untilPosition);
-      rangeIndex += 2;
-      currentBatchId += 1;
+    for (let i = 0; i < featureIds.length; i++) {
+      const featureId = featureIds[i];
+      const batchId = objectIds.indexOf(featureId);
+      featureIds[i] = batchId;
     }
-    return batchId;
+
+    i3sContent.attributes._BATCHID = {
+      size: 1,
+      byteOffset: 0,
+      value: featureIds
+    };
   }
 
   /**

package/src/converter-cli.ts

@@ -14,6 +14,10 @@ type TileConversionOptions = {
   /** Output folder. This folder will be created by converter if doesn't exist. It is relative to the converter path.
    * Default: "data" folder */
   output: string;
+  /** Keep created 3DNodeIndexDocument files on disk instead of memory. This option reduce memory usage but decelerates conversion speed */
+  instantNodeWriting: boolean;
+  /** Try to merge similar materials to be able to merge meshes into one node (I3S to 3DTiles conversion only) */
+  mergeMaterials: boolean;
   /** 3DTiles->I3S only. location of 7z.exe archiver to create slpk on Windows OS, default: "C:\Program Files\7-Zip\7z.exe" */
   sevenZipExe: string;
   /** location of the Earth Gravity Model (*.pgm) file to convert heights from ellipsoidal to gravity-related format,
@@ -66,7 +70,9 @@ async function main() {
     printHelp();
   }
 
-  const options: TileConversionOptions = parseOptions(args);
+  const validatedOptionsArr = validateOptionsWithEqual(args);
+
+  const options: TileConversionOptions = parseOptions(validatedOptionsArr);
 
   if (options.installDependencies) {
     const depthInstaller = new DepsInstaller();
@@ -97,6 +103,12 @@ function printHelp(): void {
   );
   console.log('--name [Tileset name]');
   console.log('--output [Output folder, default: "data" folder]');
+  console.log(
+    '--instant-node-writing [Keep created 3DNodeIndexDocument files on disk instead of memory. This option reduce memory usage but decelerates conversion speed]'
+  );
+  console.log(
+    '--split-nodes [Prevent to merge similar materials that could lead to incorrect visualization (I3S to 3DTiles conversion only)]'
+  );
   console.log('--slpk [Generate slpk (Scene Layer Packages) I3S output file]');
   console.log(
     '--tileset [tileset.json file (3DTiles) / http://..../SceneServer/layers/0 resource (I3S)]'
@@ -152,9 +164,11 @@ async function convert(options: ValidatedTileConversionOptions) {
         egmFilePath: options.egm,
         token: options.token,
         draco: options.draco,
+        mergeMaterials: options.mergeMaterials,
         generateTextures: options.generateTextures,
         generateBoundingVolumes: options.generateBoundingVolumes,
-        validate: options.validate
+        validate: options.validate,
+        instantNodeWriting: options.instantNodeWriting
       });
       break;
     default:
@@ -199,6 +213,19 @@ function validateOptions(options: TileConversionOptions): ValidatedTileConversio
   return <ValidatedTileConversionOptions>options;
 }
 
+function validateOptionsWithEqual(args: string[]): string[] {
+  return args.reduce((acc: string[], curr) => {
+    const equalSignIndex = curr.indexOf('=');
+    const beforeEqual = curr.slice(0, equalSignIndex);
+    const afterEqual = curr.slice(equalSignIndex + 1, curr.length);
+    const condition = curr.includes('=') && curr.startsWith('--') && afterEqual;
+    if (condition) {
+      return acc.concat(beforeEqual, afterEqual);
+    }
+    return acc.concat(curr);
+  }, []);
+}
+
 /**
  * Parse option from the cli arguments array
  * @param args
@@ -207,6 +234,8 @@ function validateOptions(options: TileConversionOptions): ValidatedTileConversio
 function parseOptions(args: string[]): TileConversionOptions {
   const opts: TileConversionOptions = {
     output: 'data',
+    instantNodeWriting: false,
+    mergeMaterials: true,
     sevenZipExe: 'C:\\Program Files\\7-Zip\\7z.exe',
     egm: join(process.cwd(), 'deps', 'egm2008-5.pgm'),
     draco: true,
@@ -233,11 +262,17 @@ function parseOptions(args: string[]): TileConversionOptions {
         case '--output':
           opts.output = getStringValue(index, args);
           break;
+        case '--instant-node-writing':
+          opts.instantNodeWriting = getBooleanValue(index, args);
+          break;
+        case '--split-nodes':
+          opts.mergeMaterials = getBooleanValue(index, args);
+          break;
         case '--max-depth':
           opts.maxDepth = getIntegerValue(index, args);
           break;
         case '--slpk':
-          opts.slpk = true;
+          opts.slpk = getBooleanValue(index, args);
           break;
         case '--7zExe':
           opts.sevenZipExe = getStringValue(index, args);
@@ -249,19 +284,19 @@ function parseOptions(args: string[]): TileConversionOptions {
           opts.token = getStringValue(index, args);
           break;
         case '--no-draco':
-          opts.draco = false;
+          opts.draco = getBooleanValue(index, args);
           break;
         case '--validate':
-          opts.validate = true;
+          opts.validate = getBooleanValue(index, args);
           break;
         case '--install-dependencies':
-          opts.installDependencies = true;
+          opts.installDependencies = getBooleanValue(index, args);
           break;
         case '--generate-textures':
-          opts.generateTextures = true;
+          opts.generateTextures = getBooleanValue(index, args);
           break;
         case '--generate-bounding-volumes':
-          opts.generateBoundingVolumes = true;
+          opts.generateBoundingVolumes = getBooleanValue(index, args);
           break;
         case '--help':
           printHelp();
@@ -322,3 +357,14 @@ function getIntegerValue(index: number, args: string[]): number {
   }
   return NaN;
 }
+
+function getBooleanValue(index: number, args: string[]): boolean {
+  const stringValue: string = getStringValue(index, args).toLowerCase().trim();
+  if (['--no-draco', '--split-nodes'].includes(args[index]) && !stringValue) {
+    return false;
+  }
+  if (!stringValue || stringValue === 'true') {
+    return true;
+  }
+  return false;
+}

package/src/deps-installer/deps-installer.ts

@@ -1,7 +1,11 @@
-import {load} from '@loaders.gl/core';
+import {load, fetchFile} from '@loaders.gl/core';
 import {ZipLoader} from '@loaders.gl/zip';
 import {writeFile} from '../lib/utils/file-utils';
 import {join} from 'path';
+import {ChildProcessProxy} from '@loaders.gl/worker-utils';
+
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'beta';
 
 const PGM_LINK = 'https://raw.githubusercontent.com/visgl/deck.gl-data/master/egm/egm2008-5.zip';
 
@@ -15,8 +19,11 @@ export class DepsInstaller {
   /**
    * Run instalation
    * @param path destination folder
+   * @param workersPath destination folder for workers.
+   *    This path is '' by default and is not used by tile-converter.
+   *    It is used in tests to prevent rewriting actual workers during tests running
    */
-  async install(path: string = ''): Promise<void> {
+  async install(path: string = '', workersPath: string = ''): Promise<void> {
     console.log('Installing "EGM2008-5" model...'); // eslint-disable-line no-console
     const fileMap = await load(PGM_LINK, ZipLoader, {});
 
@@ -27,6 +34,35 @@ export class DepsInstaller {
 
     await writeFile(depsPath, new Uint8Array(fileMap['geoids/egm2008-5.pgm']), 'egm2008-5.pgm');
 
+    console.log('Installing "I3S Content Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('i3s', 'i3s-content-nodejs-worker.js', workersPath);
+
+    console.log('Installing "Draco Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('draco', 'draco-nodejs-worker.js', workersPath);
+
+    console.log('Installing "Basis Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('textures', 'basis-nodejs-worker.js', workersPath);
+
+    console.log('Installing "join-images" npm package');
+    const childProcess = new ChildProcessProxy();
+    await childProcess.start({
+      command: 'npm',
+      arguments: ['install', 'sharp', 'join-images'],
+      wait: 0
+    });
+
     console.log('All dependencies were installed succesfully.'); // eslint-disable-line no-console
   }
+
+  private async installWorker(module: string, name: string, extraPath: string) {
+    const fileResponse = await fetchFile(
+      `https://unpkg.com/@loaders.gl/${module}@${VERSION}/dist/${name}`
+    );
+    const fileData = await fileResponse.arrayBuffer();
+    if (!fileData) {
+      return;
+    }
+    const path = join(process.cwd(), extraPath, 'modules', module, 'dist');
+    await writeFile(path, fileData, name);
+  }
 }

package/src/i3s-attributes-worker.ts

@@ -1,6 +1,7 @@
 import type {WorkerObject} from '@loaders.gl/worker-utils';
 import type {ConvertedAttributes} from './i3s-converter/types';
 import type {Matrix4, Vector3} from '@math.gl/core';
+import type {GLTFImagePostprocessed, GLTFNodePostprocessed} from '@loaders.gl/gltf';
 
 import {processOnWorker} from '@loaders.gl/worker-utils';
 
@@ -9,13 +10,16 @@ import {processOnWorker} from '@loaders.gl/worker-utils';
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 export type I3SAttributesWorkerOptions = {
+  _nodeWorkers: boolean;
+  reuseWorkers: boolean;
   useCartesianPositions: boolean;
   source: string;
 };
 
 export type B3DMAttributesData = {
   gltfMaterials?: {id: string}[];
-  nodes: any;
+  nodes: GLTFNodePostprocessed[];
+  images: GLTFImagePostprocessed[];
   cartographicOrigin: Vector3;
   cartesianModelMatrix: Matrix4;
 };