@loaders.gl/tile-converter 4.0.0-alpha.5 → 4.0.0-alpha.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "4.0.0-alpha.5",
+  "version": "4.0.0-alpha.7",
   "description": "Converter",
   "license": "MIT",
   "publishConfig": {
@@ -16,8 +16,8 @@
     "i3s"
   ],
   "types": "dist/index.d.ts",
-  "main": "dist/index.js",
-  "module": "dist/index.js",
+  "main": "dist/es5/index.js",
+  "module": "dist/esm/index.js",
   "sideEffects": false,
   "bin": {
     "tile-converter": "./bin/converter.js",
@@ -30,28 +30,32 @@
     "README.md"
   ],
   "browser": {
-    "fs": false
+    "fs": false,
+    "path": false,
+    "process": false,
+    "archiver": false,
+    "join-images": false
   },
   "scripts": {
-    "pre-build": "npm run build-bundle  && npm run build-converter-bundle",
-    "build-bundle": "esbuild ./src/index.ts --bundle --outfile=dist/dist.min.js --platform=node",
-    "build-converter-bundle": "webpack --display errors-only --config ./converter-webpack/bundle.js",
-    "build-bin": "BABEL_ENV=es5 babel scripts --config-file ../../babel.config.js --out-dir dist/scripts"
+    "pre-build": "npm run build-bundle && npm run build-converter-bundle && npm run build-i3s-attributes-worker && npm run build-3d-tiles-attributes-worker",
+    "build-bundle": "esbuild ./src/index.ts --bundle --outfile=dist/dist.min.js --platform=node  --external:join-images",
+    "build-converter-bundle": "esbuild src/converter-cli.ts --outfile=dist/converter.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-i3s-attributes-worker": "esbuild src/workers/i3s-attributes-worker.ts --outfile=dist/i3s-attributes-worker.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-3d-tiles-attributes-worker": "esbuild src/workers/3d-tiles-attributes-worker.ts --outfile=dist/3d-tiles-attributes-worker.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "4.0.0-alpha.5",
-    "@loaders.gl/crypto": "4.0.0-alpha.5",
-    "@loaders.gl/draco": "4.0.0-alpha.5",
-    "@loaders.gl/gltf": "4.0.0-alpha.5",
-    "@loaders.gl/i3s": "4.0.0-alpha.5",
-    "@loaders.gl/images": "4.0.0-alpha.5",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.5",
-    "@loaders.gl/polyfills": "4.0.0-alpha.5",
-    "@loaders.gl/textures": "4.0.0-alpha.5",
-    "@loaders.gl/tiles": "4.0.0-alpha.5",
-    "@loaders.gl/worker-utils": "4.0.0-alpha.5",
-    "@loaders.gl/zip": "4.0.0-alpha.5",
-    "@luma.gl/engine": "^8.5.4",
+    "@loaders.gl/3d-tiles": "4.0.0-alpha.7",
+    "@loaders.gl/crypto": "4.0.0-alpha.7",
+    "@loaders.gl/draco": "4.0.0-alpha.7",
+    "@loaders.gl/gltf": "4.0.0-alpha.7",
+    "@loaders.gl/i3s": "4.0.0-alpha.7",
+    "@loaders.gl/images": "4.0.0-alpha.7",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.7",
+    "@loaders.gl/polyfills": "4.0.0-alpha.7",
+    "@loaders.gl/textures": "4.0.0-alpha.7",
+    "@loaders.gl/tiles": "4.0.0-alpha.7",
+    "@loaders.gl/worker-utils": "4.0.0-alpha.7",
+    "@loaders.gl/zip": "4.0.0-alpha.7",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
     "@math.gl/geoid": "^3.5.1",
@@ -60,7 +64,7 @@
     "cors": "^2.8.5",
     "crypt": "^0.0.2",
     "debug": "~2.6.9",
-    "express": "~4.16.1",
+    "express": "~4.17.3",
     "json-map-transform": "^1.2.6",
     "jszip": "^3.5.0",
     "md5": "^2.3.0",
@@ -68,7 +72,11 @@
     "uuid": "^8.1.0"
   },
   "peerDependencies": {
-    "@loaders.gl/core": "4.0.0-alpha.4"
+    "@loaders.gl/core": "4.0.0-alpha.6"
   },
-  "gitHead": "7a71a54bdf1ddf985cc3af3db90b82e7fa97d025"
+  "quarantinedDependencies": {
+    "join-images": "^1.1.3",
+    "sharp": "^0.31.3"
+  },
+  "gitHead": "afb59c4d8e5d8ebb9c28f111cb0c96c5527d0ffd"
 }

package/src/3d-tiles-attributes-worker.ts

@@ -0,0 +1,43 @@
+import type {WorkerObject} from '@loaders.gl/worker-utils';
+import type {FeatureAttribute} from '@loaders.gl/i3s';
+
+import {processOnWorker} from '@loaders.gl/worker-utils';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type Tile3DAttributesWorkerOptions = {
+  featureAttributes: FeatureAttribute | null;
+  source: string;
+};
+
+export type I3SAttributesData = {
+  tileContent: any;
+  textureFormat: string;
+};
+
+/**
+ * I3S Attributes Worker to handle B3DM object
+ */
+export const Tile3dAttributesWorker = {
+  id: '3d-tiles-attributes',
+  name: '3DTiles Attributes Worker',
+  module: 'tile-converter',
+  version: VERSION,
+  options: {
+    featureAttributes: null
+  }
+};
+
+/**
+ * Performs I3S attributes transformation
+ */
+export function transform3DTilesAttributesOnWorker(
+  i3sAttributesData: I3SAttributesData,
+  options: Tile3DAttributesWorkerOptions
+): Promise<ArrayBuffer> {
+  return processOnWorker(Tile3dAttributesWorker, i3sAttributesData, options);
+}
+
+export const _typecheckI3SAttributesWorker: WorkerObject = Tile3dAttributesWorker;

package/src/3d-tiles-converter/3d-tiles-converter.ts

@@ -4,7 +4,7 @@ import type {Node3D} from '@loaders.gl/3d-tiles';
 import {join} from 'path';
 import process from 'process';
 import transform from 'json-map-transform';
-import {load} from '@loaders.gl/core';
+import {fetchFile, getLoaderOptions, load, isBrowser} from '@loaders.gl/core';
 import {I3SLoader, I3SAttributeLoader, COORDINATE_SYSTEM} from '@loaders.gl/i3s';
 import {Tileset3D, Tile3D} from '@loaders.gl/tiles';
 import {Geoid} from '@math.gl/geoid';
@@ -15,8 +15,14 @@ import {convertScreenThresholdToGeometricError} from '../lib/utils/lod-conversio
 import {writeFile, removeDir} from '../lib/utils/file-utils';
 import {calculateFilesSize, timeConverter} from '../lib/utils/statistic-utills';
 import {TILESET as tilesetTemplate} from './json-templates/tileset';
-import B3dmConverter from './helpers/b3dm-converter';
 import {createObbFromMbs} from '../i3s-converter/helpers/coordinate-converter';
+import {
+  I3SAttributesData,
+  Tile3dAttributesWorker,
+  transform3DTilesAttributesOnWorker
+} from '../3d-tiles-attributes-worker';
+import {getWorkerURL, WorkerFarm} from '@loaders.gl/worker-utils';
+import {BROWSER_ERROR_MESSAGE} from '../constants';
 
 const I3S = 'I3S';
 
@@ -31,6 +37,7 @@ export default class Tiles3DConverter {
   geoidHeightModel: Geoid | null;
   sourceTileset: Tileset3D | null;
   attributeStorageInfo: AttributeStorageInfo | null;
+  workerSource: {[key: string]: string} = {};
 
   constructor() {
     this.options = {};
@@ -40,6 +47,7 @@ export default class Tiles3DConverter {
     this.geoidHeightModel = null;
     this.sourceTileset = null;
     this.attributeStorageInfo = null;
+    this.workerSource = {};
   }
 
   /**
@@ -55,9 +63,13 @@ export default class Tiles3DConverter {
     inputUrl: string;
     outputPath: string;
     tilesetName: string;
-    maxDepth: number;
+    maxDepth?: number;
     egmFilePath: string;
   }): Promise<any> {
+    if (isBrowser) {
+      console.log(BROWSER_ERROR_MESSAGE);
+      return BROWSER_ERROR_MESSAGE;
+    }
     const {inputUrl, outputPath, tilesetName, maxDepth, egmFilePath} = options;
     this.conversionStartTime = process.hrtime();
     this.options = {maxDepth};
@@ -66,14 +78,23 @@ export default class Tiles3DConverter {
     this.geoidHeightModel = await load(egmFilePath, PGMLoader);
     console.log('Loading egm file completed!'); // eslint-disable-line
 
+    await this.loadWorkers();
+
     const sourceTilesetJson = await load(inputUrl, I3SLoader, {});
 
     this.sourceTileset = new Tileset3D(sourceTilesetJson, {
       loadOptions: {
+        _nodeWorkers: true,
+        reuseWorkers: true,
         i3s: {coordinateSystem: COORDINATE_SYSTEM.LNGLAT_OFFSETS, decodeTextures: false}
+        // TODO should no longer be needed with new workers
+        // 'i3s-content-nodejs': {
+        //   workerUrl: './modules/i3s/dist/i3s-content-nodejs-worker.js'
+        // }
       }
     });
 
+    await this.sourceTileset.tilesetInitializationPromise;
     const rootNode = this.sourceTileset.root!;
     if (!rootNode.header.obb) {
       rootNode.header.obb = createObbFromMbs(rootNode.header.mbs);
@@ -102,6 +123,71 @@ export default class Tiles3DConverter {
     await writeFile(this.tilesetPath, JSON.stringify(tileset), 'tileset.json');
 
     this._finishConversion({slpk: false, outputPath, tilesetName});
+
+    // Clean up worker pools
+    const workerFarm = WorkerFarm.getWorkerFarm({});
+    workerFarm.destroy();
+  }
+
+  /**
+   * Convert particular I3S Node
+   * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
+   * @param parentNode object in resulting tileset
+   * @param level a current level of a tree depth
+   * @param childNodeInfo child node to convert
+   */
+  private async convertChildNode(
+    parentSourceNode: Tile3D,
+    parentNode: Node3D,
+    level: number,
+    childNodeInfo: NodeReference
+  ): Promise<void> {
+    const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
+    parentSourceNode.children.push(sourceChild);
+    if (sourceChild.contentUrl) {
+      await this.sourceTileset!._loadTile(sourceChild);
+      this.vertexCounter += sourceChild.content.vertexCount;
+
+      let featureAttributes: FeatureAttribute | null = null;
+      if (this.attributeStorageInfo) {
+        featureAttributes = await this._loadChildAttributes(sourceChild, this.attributeStorageInfo);
+      }
+
+      if (!sourceChild.header.obb) {
+        sourceChild.header.obb = createObbFromMbs(sourceChild.header.mbs);
+      }
+
+      const boundingVolume = {
+        box: i3sObbTo3dTilesObb(sourceChild.header.obb, this.geoidHeightModel)
+      };
+      const child: Node3D = {
+        boundingVolume,
+        geometricError: convertScreenThresholdToGeometricError(sourceChild),
+        children: []
+      };
+
+      const i3sAttributesData: I3SAttributesData = {
+        tileContent: sourceChild.content,
+        textureFormat: sourceChild?.header?.textureFormat
+      };
+
+      const b3dm = await transform3DTilesAttributesOnWorker(i3sAttributesData, {
+        source: this.workerSource.tile3dWorkerSource,
+        featureAttributes
+      });
+
+      child.content = {
+        uri: `${sourceChild.id}.b3dm`,
+        boundingVolume
+      };
+      await writeFile(this.tilesetPath, new Uint8Array(b3dm), `${sourceChild.id}.b3dm`);
+      parentNode.children.push(child);
+
+      sourceChild.unloadContent();
+      await this._addChildren(sourceChild, child, level + 1);
+    } else {
+      await this._addChildren(sourceChild, parentNode, level + 1);
+    }
   }
 
   /**
@@ -118,45 +204,11 @@ export default class Tiles3DConverter {
     if (this.options.maxDepth && level > this.options.maxDepth) {
       return;
     }
+    const promises: Promise<void>[] = [];
     for (const childNodeInfo of parentSourceNode.header.children || []) {
-      const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
-      parentSourceNode.children.push(sourceChild);
-      if (sourceChild.contentUrl) {
-        await this.sourceTileset!._loadTile(sourceChild);
-        this.vertexCounter += sourceChild.content.vertexCount;
-
-        let attributes: FeatureAttribute | null = null;
-        if (this.attributeStorageInfo) {
-          attributes = await this._loadChildAttributes(sourceChild, this.attributeStorageInfo);
-        }
-
-        if (!sourceChild.header.obb) {
-          sourceChild.header.obb = createObbFromMbs(sourceChild.header.mbs);
-        }
-
-        const boundingVolume = {
-          box: i3sObbTo3dTilesObb(sourceChild.header.obb, this.geoidHeightModel)
-        };
-        const child: Node3D = {
-          boundingVolume,
-          geometricError: convertScreenThresholdToGeometricError(sourceChild),
-          children: []
-        };
-
-        const b3dm = await new B3dmConverter().convert(sourceChild, attributes);
-        child.content = {
-          uri: `${sourceChild.id}.b3dm`,
-          boundingVolume
-        };
-        await writeFile(this.tilesetPath, new Uint8Array(b3dm), `${sourceChild.id}.b3dm`);
-        parentNode.children.push(child);
-
-        sourceChild.unloadContent();
-        await this._addChildren(sourceChild, child, level + 1);
-      } else {
-        await this._addChildren(sourceChild, parentNode, level + 1);
-      }
+      promises.push(this.convertChildNode(parentSourceNode, parentNode, level, childNodeInfo));
     }
+    await Promise.all(promises);
   }
 
   /**
@@ -290,4 +342,14 @@ export default class Tiles3DConverter {
     console.log(`File(s) size: `, filesSize, ' bytes'); // eslint-disable-line
     console.log(`------------------------------------------------`); // eslint-disable-line
   }
+
+  private async loadWorkers(): Promise<void> {
+    console.log(`Loading workers source...`); // eslint-disable-line no-undef, no-console
+    const tile3dAttributesWorkerUrl = getWorkerURL(Tile3dAttributesWorker, {...getLoaderOptions()});
+    const sourceResponse = await fetchFile(tile3dAttributesWorkerUrl);
+    const source = await sourceResponse.text();
+
+    this.workerSource.tile3dWorkerSource = source;
+    console.log(`Loading workers source completed!`); // eslint-disable-line no-undef, no-console
+  }
 }

package/src/3d-tiles-converter/helpers/b3dm-converter.ts

@@ -1,9 +1,12 @@
+import type {I3SAttributesData} from '../../3d-tiles-attributes-worker';
+
 import {encodeSync} from '@loaders.gl/core';
 import {GLTFScenegraph, GLTFWriter} from '@loaders.gl/gltf';
 import {Tile3DWriter} from '@loaders.gl/3d-tiles';
 import {Matrix4, Vector3} from '@math.gl/core';
 import {Ellipsoid} from '@math.gl/geospatial';
 import {convertTextureAtlas} from './texture-atlas';
+import {generateSyntheticIndices} from '../../lib/utils/geometry-utils';
 
 const Z_UP_TO_Y_UP_MATRIX = new Matrix4([1, 0, 0, 0, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1]);
 const scratchVector = new Vector3();
@@ -21,15 +24,17 @@ export default class B3dmConverter {
    * @param i3sTile - Tile3D instance for I3S node
    * @returns - encoded content
    */
-  async convert(i3sTile: Object, attributes: any = null): Promise<ArrayBuffer> {
-    this.i3sTile = i3sTile;
-    const gltf = await this.buildGltf(i3sTile);
+  async convert(
+    i3sAttributesData: I3SAttributesData,
+    featureAttributes: any = null
+  ): Promise<ArrayBuffer> {
+    const gltf = await this.buildGltf(i3sAttributesData, featureAttributes);
     const b3dm = encodeSync(
       {
         gltfEncoded: new Uint8Array(gltf),
         type: 'b3dm',
-        featuresLength: this._getFeaturesLength(attributes),
-        batchTable: attributes
+        featuresLength: this._getFeaturesLength(featureAttributes),
+        batchTable: featureAttributes
       },
       Tile3DWriter
     );
@@ -41,7 +46,11 @@ export default class B3dmConverter {
    * @param i3sTile - Tile3D instance for I3S node
    * @returns - encoded glb content
    */
-  async buildGltf(i3sTile): Promise<ArrayBuffer> {
+  async buildGltf(
+    i3sAttributesData: I3SAttributesData,
+    featureAttributes: any
+  ): Promise<ArrayBuffer> {
+    const {tileContent, textureFormat} = i3sAttributesData;
     const {
       material,
       attributes,
@@ -49,10 +58,10 @@ export default class B3dmConverter {
       cartesianOrigin,
       cartographicOrigin,
       modelMatrix
-    } = i3sTile.content;
+    } = tileContent;
     const gltfBuilder = new GLTFScenegraph();
 
-    const textureIndex = await this._addI3sTextureToGltf(i3sTile, gltfBuilder);
+    const textureIndex = await this._addI3sTextureToGltf(tileContent, textureFormat, gltfBuilder);
     const pbrMaterialInfo = this._convertI3sMaterialToGltfMaterial(material, textureIndex);
     const materialIndex = gltfBuilder.addMaterial(pbrMaterialInfo);
 
@@ -72,11 +81,12 @@ export default class B3dmConverter {
       cartographicOrigin,
       modelMatrix
     );
+    this._createBatchIds(tileContent, featureAttributes);
     if (attributes.normals && !this._checkNormals(attributes.normals.value)) {
       delete attributes.normals;
     }
     const indices =
-      originalIndices || this._generateSynteticIndices(positionsValue.length / positions.size);
+      originalIndices || generateSyntheticIndices(positionsValue.length / positions.size);
     const meshIndex = gltfBuilder.addMesh({
       attributes,
       indices,
@@ -101,11 +111,8 @@ export default class B3dmConverter {
    * @param {GLTFScenegraph} gltfBuilder - gltfScenegraph instance to construct GLTF
    * @returns {Promise<number | null>} - GLTF texture index
    */
-  async _addI3sTextureToGltf(i3sTile, gltfBuilder) {
-    const {
-      content: {texture, material, attributes},
-      header: {textureFormat}
-    } = i3sTile;
+  async _addI3sTextureToGltf(tileContent, textureFormat, gltfBuilder) {
+    const {texture, material, attributes} = tileContent;
     let textureIndex = null;
     let selectedTexture = texture;
     if (!texture && material) {
@@ -161,40 +168,28 @@ export default class B3dmConverter {
   }
 
   /**
-   * Generate batchId attribute from faceRanges.
-   * @param {Uint32Array} faceRanges - the source array
-   * @returns {Float32Array} batchId list.
+   * Create _BATCHID attribute
+   * @param {Object} i3sContent - the source object
+   * @returns {void}
    */
-  _generateBatchId(faceRanges) {
-    const batchIdArraySize = (faceRanges[faceRanges.length - 1] + 1) * 3;
-    const batchId = new Float32Array(batchIdArraySize);
-    let rangeIndex = 0;
-    let currentBatchId = 0;
-
-    for (let index = 0; index < faceRanges.length / 2; index++) {
-      const fromIndex = faceRanges[rangeIndex] * 3;
-      const untilPosition = (faceRanges[rangeIndex + 1] + 1) * 3;
-
-      batchId.fill(currentBatchId, fromIndex, untilPosition);
-      rangeIndex += 2;
-      currentBatchId += 1;
+  _createBatchIds(i3sContent, featureAttributes) {
+    const {featureIds} = i3sContent;
+    const {OBJECTID: objectIds} = featureAttributes || {};
+    if (!featureIds || !objectIds) {
+      return;
     }
-    return batchId;
-  }
 
-  /**
-   * luma.gl can not work without indices now:
-   * https://github.com/visgl/luma.gl/blob/d8cad75b9f8ca3e578cf078ed9d19e619c2ddbc9/modules/experimental/src/gltf/gltf-instantiator.js#L115
-   * This method generates syntetic indices array: [0, 1, 2, 3, .... , vertexCount-1]
-   * @param {number} vertexCount - vertex count in the geometry
-   * @returns {Uint32Array} indices array.
-   */
-  _generateSynteticIndices(vertexCount) {
-    const result = new Uint32Array(vertexCount);
-    for (let index = 0; index < vertexCount; index++) {
-      result.set([index], index);
+    for (let i = 0; i < featureIds.length; i++) {
+      const featureId = featureIds[i];
+      const batchId = objectIds.indexOf(featureId);
+      featureIds[i] = batchId;
     }
-    return result;
+
+    i3sContent.attributes._BATCHID = {
+      size: 1,
+      byteOffset: 0,
+      value: featureIds
+    };
   }
 
   /**

package/src/constants.ts

@@ -0,0 +1,2 @@
+export const BROWSER_ERROR_MESSAGE =
+  'Tile converter does not work in browser, only in node js environment';