@loaders.gl/tile-converter 4.0.0-alpha.1 → 4.0.0-alpha.11

package/dist/workers/3d-tiles-attributes-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=3d-tiles-attributes-worker.d.ts.map

package/dist/workers/3d-tiles-attributes-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"3d-tiles-attributes-worker.d.ts","sourceRoot":"","sources":["../../src/workers/3d-tiles-attributes-worker.ts"],"names":[],"mappings":""}

package/dist/workers/3d-tiles-attributes-worker.js

@@ -0,0 +1,9 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+const b3dm_converter_1 = __importDefault(require("../3d-tiles-converter/helpers/b3dm-converter"));
+const b3dmConverter = new b3dm_converter_1.default();
+(0, worker_utils_1.createWorker)(async (data, options = {}) => b3dmConverter.convert(data, options.featureAttributes));

package/dist/workers/i3s-attributes-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=i3s-attributes-worker.d.ts.map

package/dist/workers/i3s-attributes-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"i3s-attributes-worker.d.ts","sourceRoot":"","sources":["../../src/workers/i3s-attributes-worker.ts"],"names":[],"mappings":""}

package/dist/workers/i3s-attributes-worker.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+const geometry_converter_1 = require("../i3s-converter/helpers/geometry-converter");
+(0, worker_utils_1.createWorker)(async (data, options = {}) => await (0, geometry_converter_1.convertAttributes)(data, options.materialAndTextureList, options.useCartesianPositions));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "4.0.0-alpha.1",
+  "version": "4.0.0-alpha.11",
   "description": "Converter",
   "license": "MIT",
   "publishConfig": {
@@ -15,12 +15,13 @@
     "3dTiles",
     "i3s"
   ],
-  "types": "src/index.ts",
-  "main": "dist/index.js",
-  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "main": "dist/es5/index.js",
+  "module": "dist/esm/index.js",
   "sideEffects": false,
   "bin": {
     "tile-converter": "./bin/converter.js",
+    "slpk-extractor": "./bin/slpk-extractor.js",
     "i3s-server": "./src/i3s-server/bin/www"
   },
   "files": [
@@ -30,35 +31,42 @@
     "README.md"
   ],
   "browser": {
-    "fs": false
+    "fs": false,
+    "path": false,
+    "process": false,
+    "archiver": false,
+    "join-images": false
   },
   "scripts": {
-    "pre-build": "npm run build-bundle",
-    "post-build": "npm run build-converter-bundle",
-    "build-bundle": "esbuild ./src/index.ts --bundle --outfile=dist/bundle.js --platform=node",
-    "build-converter-bundle": "webpack --display errors-only --config ./converter-webpack/bundle.js"
+    "pre-build": "npm run build-bundle && npm run build-converter-bundle && npm run build-i3s-attributes-worker && npm run build-slpk-extractor-bundle && npm run build-3d-tiles-attributes-worker",
+    "build-bundle": "esbuild ./src/index.ts --bundle --outfile=dist/dist.min.js --platform=node  --external:join-images",
+    "build-converter-bundle": "esbuild src/converter-cli.ts --outfile=dist/converter.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-slpk-extractor-bundle": "esbuild src/slpk-extractor-cli.ts --outfile=dist/slpk-extractor.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-i3s-attributes-worker": "esbuild src/workers/i3s-attributes-worker.ts --outfile=dist/i3s-attributes-worker.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-3d-tiles-attributes-worker": "esbuild src/workers/3d-tiles-attributes-worker.ts --outfile=dist/3d-tiles-attributes-worker.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "4.0.0-alpha.1",
-    "@loaders.gl/crypto": "4.0.0-alpha.1",
-    "@loaders.gl/draco": "4.0.0-alpha.1",
-    "@loaders.gl/gltf": "4.0.0-alpha.1",
-    "@loaders.gl/i3s": "4.0.0-alpha.1",
-    "@loaders.gl/images": "4.0.0-alpha.1",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.1",
-    "@loaders.gl/polyfills": "4.0.0-alpha.1",
-    "@loaders.gl/tiles": "4.0.0-alpha.1",
-    "@loaders.gl/worker-utils": "4.0.0-alpha.1",
-    "@loaders.gl/zip": "4.0.0-alpha.1",
-    "@luma.gl/engine": "^8.5.4",
+    "@loaders.gl/3d-tiles": "4.0.0-alpha.11",
+    "@loaders.gl/crypto": "4.0.0-alpha.11",
+    "@loaders.gl/draco": "4.0.0-alpha.11",
+    "@loaders.gl/gltf": "4.0.0-alpha.11",
+    "@loaders.gl/i3s": "4.0.0-alpha.11",
+    "@loaders.gl/images": "4.0.0-alpha.11",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.11",
+    "@loaders.gl/polyfills": "4.0.0-alpha.11",
+    "@loaders.gl/textures": "4.0.0-alpha.11",
+    "@loaders.gl/tiles": "4.0.0-alpha.11",
+    "@loaders.gl/worker-utils": "4.0.0-alpha.11",
+    "@loaders.gl/zip": "4.0.0-alpha.11",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
+    "@math.gl/geoid": "^3.5.1",
     "@math.gl/geospatial": "^3.5.1",
     "archiver": "^5.0.0",
     "cors": "^2.8.5",
     "crypt": "^0.0.2",
     "debug": "~2.6.9",
-    "express": "~4.16.1",
+    "express": "~4.17.3",
     "json-map-transform": "^1.2.6",
     "jszip": "^3.5.0",
     "md5": "^2.3.0",
@@ -66,7 +74,11 @@
     "uuid": "^8.1.0"
   },
   "peerDependencies": {
-    "@loaders.gl/core": "^3.0.0"
+    "@loaders.gl/core": "^4.0.0-alpha.8"
   },
-  "gitHead": "e48f5534fe7188c810f23f965a440c144b3cf6d0"
+  "quarantinedDependencies": {
+    "join-images": "^1.1.3",
+    "sharp": "^0.31.3"
+  },
+  "gitHead": "bc680098cfea790c67b7fb95bab96e8d9288d34f"
 }

package/src/3d-tiles-attributes-worker.ts

@@ -0,0 +1,43 @@
+import type {WorkerObject} from '@loaders.gl/worker-utils';
+import type {FeatureAttribute} from '@loaders.gl/i3s';
+
+import {processOnWorker} from '@loaders.gl/worker-utils';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type Tile3DAttributesWorkerOptions = {
+  featureAttributes: FeatureAttribute | null;
+  source: string;
+};
+
+export type I3SAttributesData = {
+  tileContent: any;
+  textureFormat: string;
+};
+
+/**
+ * I3S Attributes Worker to handle B3DM object
+ */
+export const Tile3dAttributesWorker = {
+  id: '3d-tiles-attributes',
+  name: '3DTiles Attributes Worker',
+  module: 'tile-converter',
+  version: VERSION,
+  options: {
+    featureAttributes: null
+  }
+};
+
+/**
+ * Performs I3S attributes transformation
+ */
+export function transform3DTilesAttributesOnWorker(
+  i3sAttributesData: I3SAttributesData,
+  options: Tile3DAttributesWorkerOptions
+): Promise<ArrayBuffer> {
+  return processOnWorker(Tile3dAttributesWorker, i3sAttributesData, options);
+}
+
+export const _typecheckI3SAttributesWorker: WorkerObject = Tile3dAttributesWorker;

package/src/3d-tiles-converter/3d-tiles-converter.ts

@@ -1,12 +1,13 @@
 import type {AttributeStorageInfo, FeatureAttribute, NodeReference} from '@loaders.gl/i3s';
-import type {Node3D} from '@loaders.gl/3d-tiles';
+import type {Tiles3DTileJSON} from '@loaders.gl/3d-tiles';
 
 import {join} from 'path';
 import process from 'process';
 import transform from 'json-map-transform';
-import {load} from '@loaders.gl/core';
-import {I3SLoader, I3SAttributeLoader} from '@loaders.gl/i3s';
+import {fetchFile, getLoaderOptions, load, isBrowser} from '@loaders.gl/core';
+import {I3SLoader, I3SAttributeLoader, COORDINATE_SYSTEM} from '@loaders.gl/i3s';
 import {Tileset3D, Tile3D} from '@loaders.gl/tiles';
+import {Geoid} from '@math.gl/geoid';
 
 import {PGMLoader} from '../pgm-loader';
 import {i3sObbTo3dTilesObb} from './helpers/i3s-obb-to-3d-tiles-obb';
@@ -14,9 +15,14 @@ import {convertScreenThresholdToGeometricError} from '../lib/utils/lod-conversio
 import {writeFile, removeDir} from '../lib/utils/file-utils';
 import {calculateFilesSize, timeConverter} from '../lib/utils/statistic-utills';
 import {TILESET as tilesetTemplate} from './json-templates/tileset';
-import B3dmConverter from './helpers/b3dm-converter';
 import {createObbFromMbs} from '../i3s-converter/helpers/coordinate-converter';
-import {GeoidHeightModel} from '../lib/geoid-height-model';
+import {
+  I3SAttributesData,
+  Tile3dAttributesWorker,
+  transform3DTilesAttributesOnWorker
+} from '../3d-tiles-attributes-worker';
+import {getWorkerURL, WorkerFarm} from '@loaders.gl/worker-utils';
+import {BROWSER_ERROR_MESSAGE} from '../constants';
 
 const I3S = 'I3S';
 
@@ -28,9 +34,10 @@ export default class Tiles3DConverter {
   tilesetPath: string;
   vertexCounter: number;
   conversionStartTime: [number, number];
-  geoidHeightModel: GeoidHeightModel;
-  sourceTileset: Tileset3D;
-  attributeStorageInfo: AttributeStorageInfo;
+  geoidHeightModel: Geoid | null;
+  sourceTileset: Tileset3D | null;
+  attributeStorageInfo: AttributeStorageInfo | null;
+  workerSource: {[key: string]: string} = {};
 
   constructor() {
     this.options = {};
@@ -40,6 +47,7 @@ export default class Tiles3DConverter {
     this.geoidHeightModel = null;
     this.sourceTileset = null;
     this.attributeStorageInfo = null;
+    this.workerSource = {};
   }
 
   /**
@@ -51,13 +59,17 @@ export default class Tiles3DConverter {
    * @param options.egmFilePath location of *.pgm file to convert heights from ellipsoidal to gravity-related format
    * @param options.maxDepth The max tree depth of conversion
    */
-  private async convert(options: {
+  public async convert(options: {
     inputUrl: string;
     outputPath: string;
     tilesetName: string;
-    maxDepth: number;
+    maxDepth?: number;
     egmFilePath: string;
   }): Promise<any> {
+    if (isBrowser) {
+      console.log(BROWSER_ERROR_MESSAGE);
+      return BROWSER_ERROR_MESSAGE;
+    }
     const {inputUrl, outputPath, tilesetName, maxDepth, egmFilePath} = options;
     this.conversionStartTime = process.hrtime();
     this.options = {maxDepth};
@@ -66,11 +78,26 @@ export default class Tiles3DConverter {
     this.geoidHeightModel = await load(egmFilePath, PGMLoader);
     console.log('Loading egm file completed!'); // eslint-disable-line
 
+    await this.loadWorkers();
+
     const sourceTilesetJson = await load(inputUrl, I3SLoader, {});
-    this.sourceTileset = new Tileset3D(sourceTilesetJson, {});
 
-    if (!this.sourceTileset.root.header.obb) {
-      this.sourceTileset.root.header.obb = createObbFromMbs(this.sourceTileset.root.header.mbs);
+    this.sourceTileset = new Tileset3D(sourceTilesetJson, {
+      loadOptions: {
+        _nodeWorkers: true,
+        reuseWorkers: true,
+        i3s: {coordinateSystem: COORDINATE_SYSTEM.LNGLAT_OFFSETS, decodeTextures: false}
+        // TODO should no longer be needed with new workers
+        // 'i3s-content-nodejs': {
+        //   workerUrl: './modules/i3s/dist/i3s-content-nodejs-worker.js'
+        // }
+      }
+    });
+
+    await this.sourceTileset.tilesetInitializationPromise;
+    const rootNode = this.sourceTileset.root!;
+    if (!rootNode.header.obb) {
+      rootNode.header.obb = createObbFromMbs(rootNode.header.mbs);
     }
 
     this.tilesetPath = join(`${outputPath}`, `${tilesetName}`);
@@ -82,20 +109,85 @@ export default class Tiles3DConverter {
       // do nothing
     }
 
-    const rootTile: Node3D = {
+    const rootTile: Tiles3DTileJSON = {
       boundingVolume: {
-        box: i3sObbTo3dTilesObb(this.sourceTileset.root.header.obb, this.geoidHeightModel)
+        box: i3sObbTo3dTilesObb(rootNode.header.obb, this.geoidHeightModel)
       },
-      geometricError: convertScreenThresholdToGeometricError(this.sourceTileset.root),
+      geometricError: convertScreenThresholdToGeometricError(rootNode),
       children: []
     };
 
-    await this._addChildren(this.sourceTileset.root, rootTile, 1);
+    await this._addChildren(rootNode, rootTile, 1);
 
-    const tileset = transform({root: rootTile}, tilesetTemplate);
+    const tileset = transform({root: rootTile}, tilesetTemplate());
     await writeFile(this.tilesetPath, JSON.stringify(tileset), 'tileset.json');
 
     this._finishConversion({slpk: false, outputPath, tilesetName});
+
+    // Clean up worker pools
+    const workerFarm = WorkerFarm.getWorkerFarm({});
+    workerFarm.destroy();
+  }
+
+  /**
+   * Convert particular I3S Node
+   * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
+   * @param parentNode object in resulting tileset
+   * @param level a current level of a tree depth
+   * @param childNodeInfo child node to convert
+   */
+  private async convertChildNode(
+    parentSourceNode: Tile3D,
+    parentNode: Tiles3DTileJSON,
+    level: number,
+    childNodeInfo: NodeReference
+  ): Promise<void> {
+    const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
+    parentSourceNode.children.push(sourceChild);
+    if (sourceChild.contentUrl) {
+      await this.sourceTileset!._loadTile(sourceChild);
+      this.vertexCounter += sourceChild.content.vertexCount;
+
+      let featureAttributes: FeatureAttribute | null = null;
+      if (this.attributeStorageInfo) {
+        featureAttributes = await this._loadChildAttributes(sourceChild, this.attributeStorageInfo);
+      }
+
+      if (!sourceChild.header.obb) {
+        sourceChild.header.obb = createObbFromMbs(sourceChild.header.mbs);
+      }
+
+      const boundingVolume = {
+        box: i3sObbTo3dTilesObb(sourceChild.header.obb, this.geoidHeightModel)
+      };
+      const child: Tiles3DTileJSON = {
+        boundingVolume,
+        geometricError: convertScreenThresholdToGeometricError(sourceChild),
+        children: []
+      };
+
+      const i3sAttributesData: I3SAttributesData = {
+        tileContent: sourceChild.content,
+        textureFormat: sourceChild?.header?.textureFormat
+      };
+
+      const b3dm = await transform3DTilesAttributesOnWorker(i3sAttributesData, {
+        source: this.workerSource.tile3dWorkerSource,
+        featureAttributes
+      });
+
+      child.content = {
+        uri: `${sourceChild.id}.b3dm`,
+        boundingVolume
+      };
+      await writeFile(this.tilesetPath, new Uint8Array(b3dm), `${sourceChild.id}.b3dm`);
+      parentNode.children.push(child);
+
+      sourceChild.unloadContent();
+      await this._addChildren(sourceChild, child, level + 1);
+    } else {
+      await this._addChildren(sourceChild, parentNode, level + 1);
+    }
   }
 
   /**
@@ -106,51 +198,17 @@ export default class Tiles3DConverter {
    */
   private async _addChildren(
     parentSourceNode: Tile3D,
-    parentNode: Node3D,
+    parentNode: Tiles3DTileJSON,
     level: number
   ): Promise<void> {
     if (this.options.maxDepth && level > this.options.maxDepth) {
       return;
     }
+    const promises: Promise<void>[] = [];
     for (const childNodeInfo of parentSourceNode.header.children || []) {
-      const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
-      parentSourceNode.children.push(sourceChild);
-      if (sourceChild.contentUrl) {
-        await this.sourceTileset._loadTile(sourceChild);
-        this.vertexCounter += sourceChild.content.vertexCount;
-
-        let attributes = null;
-        if (this.attributeStorageInfo) {
-          attributes = await this._loadChildAttributes(sourceChild, this.attributeStorageInfo);
-        }
-
-        if (!sourceChild.header.obb) {
-          sourceChild.header.obb = createObbFromMbs(sourceChild.header.mbs);
-        }
-
-        const boundingVolume = {
-          box: i3sObbTo3dTilesObb(sourceChild.header.obb, this.geoidHeightModel)
-        };
-        const child: Node3D = {
-          boundingVolume,
-          geometricError: convertScreenThresholdToGeometricError(sourceChild),
-          children: []
-        };
-
-        const b3dm = await new B3dmConverter().convert(sourceChild, attributes);
-        child.content = {
-          uri: `${sourceChild.id}.b3dm`,
-          boundingVolume
-        };
-        await writeFile(this.tilesetPath, new Uint8Array(b3dm), `${sourceChild.id}.b3dm`);
-        parentNode.children.push(child);
-
-        sourceChild.unloadContent();
-        await this._addChildren(sourceChild, child, level + 1);
-      } else {
-        await this._addChildren(sourceChild, parentNode, level + 1);
-      }
+      promises.push(this.convertChildNode(parentSourceNode, parentNode, level, childNodeInfo));
     }
+    await Promise.all(promises);
   }
 
   /**
@@ -161,18 +219,18 @@ export default class Tiles3DConverter {
    */
   private async _loadChildNode(parentNode: Tile3D, childNodeInfo: NodeReference): Promise<Tile3D> {
     let header;
-    if (this.sourceTileset.tileset.nodePages) {
+    if (this.sourceTileset!.tileset.nodePages) {
       console.log(`Node conversion: ${childNodeInfo.id}`); // eslint-disable-line no-console,no-undef
-      header = await this.sourceTileset.tileset.nodePagesTile.formTileFromNodePages(
+      header = await this.sourceTileset!.tileset.nodePagesTile.formTileFromNodePages(
         childNodeInfo.id
       );
     } else {
-      const {loader} = this.sourceTileset;
-      const nodeUrl = this._relativeUrlToFullUrl(parentNode.url, childNodeInfo.href);
+      const {loader} = this.sourceTileset!;
+      const nodeUrl = this._relativeUrlToFullUrl(parentNode.url, childNodeInfo.href!);
       // load metadata
       const options = {
         i3s: {
-          ...this.sourceTileset.loadOptions,
+          ...this.sourceTileset!.loadOptions,
           isTileHeader: true,
           loadContent: false
         }
@@ -181,7 +239,7 @@ export default class Tiles3DConverter {
       console.log(`Node conversion: ${nodeUrl}`); // eslint-disable-line no-console,no-undef
       header = await load(nodeUrl, loader, options);
     }
-    return new Tile3D(this.sourceTileset, header, parentNode);
+    return new Tile3D(this.sourceTileset!, header, parentNode);
   }
 
   /**
@@ -216,7 +274,7 @@ export default class Tiles3DConverter {
     sourceChild: Tile3D,
     attributeStorageInfo: AttributeStorageInfo
   ): Promise<FeatureAttribute> {
-    const promises = [];
+    const promises: any[] = [];
     const {attributeUrls} = sourceChild.header;
 
     for (let index = 0; index < attributeUrls.length; index++) {
@@ -284,4 +342,14 @@ export default class Tiles3DConverter {
     console.log(`File(s) size: `, filesSize, ' bytes'); // eslint-disable-line
     console.log(`------------------------------------------------`); // eslint-disable-line
   }
+
+  private async loadWorkers(): Promise<void> {
+    console.log(`Loading workers source...`); // eslint-disable-line no-undef, no-console
+    const tile3dAttributesWorkerUrl = getWorkerURL(Tile3dAttributesWorker, {...getLoaderOptions()});
+    const sourceResponse = await fetchFile(tile3dAttributesWorkerUrl);
+    const source = await sourceResponse.text();
+
+    this.workerSource.tile3dWorkerSource = source;
+    console.log(`Loading workers source completed!`); // eslint-disable-line no-undef, no-console
+  }
 }