@loaders.gl/tile-converter 3.2.0-alpha.1 → 3.2.0-alpha.4

package/dist/lib/utils/write-queue.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const queue_1 = require("./queue");
+class WriteQueue extends queue_1.Queue {
+    constructor(listeningInterval = 2000, writeConcurrency = 400) {
+        super();
+        this.writePromise = null;
+        this.fileMap = {};
+        this.listeningInterval = listeningInterval;
+        this.writeConcurrency = writeConcurrency;
+    }
+    startListening() {
+        this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);
+    }
+    stopListening() {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+        }
+    }
+    async startWrite() {
+        if (this.writePromise) {
+            await this.writePromise;
+            this.writePromise = null;
+            return;
+        }
+        this.writePromise = this.doWrite();
+        await this.writePromise;
+        this.writePromise = null;
+    }
+    async finalize() {
+        this.stopListening();
+        await this.startWrite();
+    }
+    async doWrite() {
+        while (this.length) {
+            const promises = [];
+            const archiveKeys = [];
+            for (let i = 0; i < this.writeConcurrency; i++) {
+                const item = this.dequeue();
+                if (!item) {
+                    break;
+                }
+                const { archiveKey, writePromise } = item;
+                archiveKeys.push(archiveKey);
+                promises.push(writePromise);
+            }
+            const writeResults = await Promise.all(promises);
+            this.updateFileMap(archiveKeys, writeResults);
+        }
+        this.writePromise = null;
+    }
+    updateFileMap(archiveKeys, writeResults) {
+        for (let i = 0; i < archiveKeys.length; i++) {
+            const archiveKey = archiveKeys[i];
+            if (!archiveKey) {
+                continue;
+            }
+            this.fileMap[archiveKey] = writeResults[i];
+        }
+    }
+}
+exports.default = WriteQueue;

package/dist/workers/3d-tiles-attributes-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=3d-tiles-attributes-worker.d.ts.map

package/dist/workers/3d-tiles-attributes-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"3d-tiles-attributes-worker.d.ts","sourceRoot":"","sources":["../../src/workers/3d-tiles-attributes-worker.ts"],"names":[],"mappings":""}

package/dist/workers/3d-tiles-attributes-worker.js

@@ -0,0 +1,9 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+const b3dm_converter_1 = __importDefault(require("../3d-tiles-converter/helpers/b3dm-converter"));
+const b3dmConverter = new b3dm_converter_1.default();
+(0, worker_utils_1.createWorker)(async (data, options = {}) => b3dmConverter.convert(data, options.attributes));

package/dist/workers/i3s-attributes-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=i3s-attributes-worker.d.ts.map

package/dist/workers/i3s-attributes-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"i3s-attributes-worker.d.ts","sourceRoot":"","sources":["../../src/workers/i3s-attributes-worker.ts"],"names":[],"mappings":""}

package/dist/workers/i3s-attributes-worker.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+const geometry_converter_1 = require("../i3s-converter/helpers/geometry-converter");
+(0, worker_utils_1.createWorker)(async (data, options = {}) => await (0, geometry_converter_1.convertAttributes)(data, options.useCartesianPositions));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "3.2.0-alpha.1",
+  "version": "3.2.0-alpha.4",
   "description": "Converter",
   "license": "MIT",
   "publishConfig": {
@@ -30,27 +30,32 @@
     "README.md"
   ],
   "browser": {
-    "fs": false
+    "fs": false,
+    "path": false,
+    "process": false,
+    "archiver": false
   },
   "scripts": {
-    "pre-build": "npm run build-bundle  && npm run build-converter-bundle",
+    "pre-build": "npm run build-bundle && npm run build-converter-bundle && npm run build-i3s-attributes-worker && npm run build-3d-tiles-attributes-worker",
     "build-bundle": "esbuild ./src/index.ts --bundle --outfile=dist/dist.min.js --platform=node",
     "build-converter-bundle": "webpack --display errors-only --config ./converter-webpack/bundle.js",
-    "build-bin": "BABEL_ENV=es5 babel scripts --config-file ../../babel.config.js --out-dir dist/scripts"
+    "build-i3s-attributes-worker": "esbuild src/workers/i3s-attributes-worker.ts --outfile=dist/i3s-attributes-worker.js --platform=node --target=esnext,node12 --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-3d-tiles-attributes-worker": "esbuild src/workers/3d-tiles-attributes-worker.ts --outfile=dist/3d-tiles-attributes-worker.js --platform=node --target=esnext,node12 --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "3.2.0-alpha.1",
-    "@loaders.gl/crypto": "3.2.0-alpha.1",
-    "@loaders.gl/draco": "3.2.0-alpha.1",
-    "@loaders.gl/gltf": "3.2.0-alpha.1",
-    "@loaders.gl/i3s": "3.2.0-alpha.1",
-    "@loaders.gl/images": "3.2.0-alpha.1",
-    "@loaders.gl/loader-utils": "3.2.0-alpha.1",
-    "@loaders.gl/polyfills": "3.2.0-alpha.1",
-    "@loaders.gl/textures": "3.2.0-alpha.1",
-    "@loaders.gl/tiles": "3.2.0-alpha.1",
-    "@loaders.gl/worker-utils": "3.2.0-alpha.1",
-    "@loaders.gl/zip": "3.2.0-alpha.1",
+    "@loaders.gl/3d-tiles": "3.2.0-alpha.4",
+    "@loaders.gl/crypto": "3.2.0-alpha.4",
+    "@loaders.gl/draco": "3.2.0-alpha.4",
+    "@loaders.gl/gltf": "3.2.0-alpha.4",
+    "@loaders.gl/i3s": "3.2.0-alpha.4",
+    "@loaders.gl/images": "3.2.0-alpha.4",
+    "@loaders.gl/loader-utils": "3.2.0-alpha.4",
+    "@loaders.gl/polyfills": "3.2.0-alpha.4",
+    "@loaders.gl/schema": "3.2.0-alpha.4",
+    "@loaders.gl/textures": "3.2.0-alpha.4",
+    "@loaders.gl/tiles": "3.2.0-alpha.4",
+    "@loaders.gl/worker-utils": "3.2.0-alpha.4",
+    "@loaders.gl/zip": "3.2.0-alpha.4",
     "@luma.gl/engine": "^8.5.4",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
@@ -68,7 +73,7 @@
     "uuid": "^8.1.0"
   },
   "peerDependencies": {
-    "@loaders.gl/core": "4.0.0-alpha.4"
+    "@loaders.gl/core": "3.2.0-alpha.3"
   },
-  "gitHead": "423a2815092b08dcf93ad5b7dc2940b167305afe"
+  "gitHead": "f030d39d5cdedb137e43e757a9da10dd637857fd"
 }

package/src/3d-tiles-attributes-worker.ts

@@ -0,0 +1,43 @@
+import type {WorkerObject} from '@loaders.gl/worker-utils';
+import type {FeatureAttribute} from '@loaders.gl/i3s';
+
+import {processOnWorker} from '@loaders.gl/worker-utils';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type Tile3DAttributesWorkerOptions = {
+  featureAttributes: FeatureAttribute | null;
+  source: string;
+};
+
+export type I3SAttributesData = {
+  tileContent: any;
+  textureFormat: string;
+};
+
+/**
+ * I3S Attributes Worker to handle B3DM object
+ */
+export const Tile3dAttributesWorker = {
+  id: '3d-tiles-attributes',
+  name: '3DTiles Attributes Worker',
+  module: 'tile-converter',
+  version: VERSION,
+  options: {
+    attributes: null
+  }
+};
+
+/**
+ * Performs I3S attributes transformation
+ */
+export function transform3DTilesAttributesOnWorker(
+  i3sAttributesData: I3SAttributesData,
+  options: Tile3DAttributesWorkerOptions
+): Promise<ArrayBuffer> {
+  return processOnWorker(Tile3dAttributesWorker, i3sAttributesData, options);
+}
+
+export const _typecheckI3SAttributesWorker: WorkerObject = Tile3dAttributesWorker;

package/src/3d-tiles-converter/3d-tiles-converter.ts

@@ -4,7 +4,7 @@ import type {Node3D} from '@loaders.gl/3d-tiles';
 import {join} from 'path';
 import process from 'process';
 import transform from 'json-map-transform';
-import {load} from '@loaders.gl/core';
+import {fetchFile, getLoaderOptions, load, isBrowser} from '@loaders.gl/core';
 import {I3SLoader, I3SAttributeLoader, COORDINATE_SYSTEM} from '@loaders.gl/i3s';
 import {Tileset3D, Tile3D} from '@loaders.gl/tiles';
 import {Geoid} from '@math.gl/geoid';
@@ -17,6 +17,13 @@ import {calculateFilesSize, timeConverter} from '../lib/utils/statistic-utills';
 import {TILESET as tilesetTemplate} from './json-templates/tileset';
 import B3dmConverter from './helpers/b3dm-converter';
 import {createObbFromMbs} from '../i3s-converter/helpers/coordinate-converter';
+import {
+  I3SAttributesData,
+  Tile3dAttributesWorker
+  /*transform3DTilesAttributesOnWorker*/
+} from '../3d-tiles-attributes-worker';
+import {getWorkerURL, WorkerFarm} from '@loaders.gl/worker-utils';
+import {BROWSER_ERROR_MESSAGE} from '../constants';
 
 const I3S = 'I3S';
 
@@ -31,6 +38,7 @@ export default class Tiles3DConverter {
   geoidHeightModel: Geoid | null;
   sourceTileset: Tileset3D | null;
   attributeStorageInfo: AttributeStorageInfo | null;
+  workerSource: {[key: string]: string} = {};
 
   constructor() {
     this.options = {};
@@ -40,6 +48,7 @@ export default class Tiles3DConverter {
     this.geoidHeightModel = null;
     this.sourceTileset = null;
     this.attributeStorageInfo = null;
+    this.workerSource = {};
   }
 
   /**
@@ -55,9 +64,13 @@ export default class Tiles3DConverter {
     inputUrl: string;
     outputPath: string;
     tilesetName: string;
-    maxDepth: number;
+    maxDepth?: number;
     egmFilePath: string;
   }): Promise<any> {
+    if (isBrowser) {
+      console.log(BROWSER_ERROR_MESSAGE);
+      return BROWSER_ERROR_MESSAGE;
+    }
     const {inputUrl, outputPath, tilesetName, maxDepth, egmFilePath} = options;
     this.conversionStartTime = process.hrtime();
     this.options = {maxDepth};
@@ -66,6 +79,8 @@ export default class Tiles3DConverter {
     this.geoidHeightModel = await load(egmFilePath, PGMLoader);
     console.log('Loading egm file completed!'); // eslint-disable-line
 
+    await this.loadWorkers();
+
     const sourceTilesetJson = await load(inputUrl, I3SLoader, {});
 
     this.sourceTileset = new Tileset3D(sourceTilesetJson, {
@@ -74,6 +89,7 @@ export default class Tiles3DConverter {
       }
     });
 
+    await this.sourceTileset.tilesetInitializationPromise;
     const rootNode = this.sourceTileset.root!;
     if (!rootNode.header.obb) {
       rootNode.header.obb = createObbFromMbs(rootNode.header.mbs);
@@ -102,6 +118,10 @@ export default class Tiles3DConverter {
     await writeFile(this.tilesetPath, JSON.stringify(tileset), 'tileset.json');
 
     this._finishConversion({slpk: false, outputPath, tilesetName});
+
+    // Clean up worker pools
+    const workerFarm = WorkerFarm.getWorkerFarm({});
+    workerFarm.destroy();
   }
 
   /**
@@ -125,9 +145,12 @@ export default class Tiles3DConverter {
         await this.sourceTileset!._loadTile(sourceChild);
         this.vertexCounter += sourceChild.content.vertexCount;
 
-        let attributes: FeatureAttribute | null = null;
+        let featureAttributes: FeatureAttribute | null = null;
         if (this.attributeStorageInfo) {
-          attributes = await this._loadChildAttributes(sourceChild, this.attributeStorageInfo);
+          featureAttributes = await this._loadChildAttributes(
+            sourceChild,
+            this.attributeStorageInfo
+          );
         }
 
         if (!sourceChild.header.obb) {
@@ -143,7 +166,19 @@ export default class Tiles3DConverter {
           children: []
         };
 
-        const b3dm = await new B3dmConverter().convert(sourceChild, attributes);
+        const i3sAttributesData: I3SAttributesData = {
+          tileContent: sourceChild.content,
+          textureFormat: sourceChild?.header?.textureFormat
+        };
+
+        // TODO Uncomment when 3d-tiles-attributes-worker will be published on CDN.
+        // const b3dm = await transform3DTilesAttributesOnWorker(i3sAttributesData, {
+        //   source: this.workerSource.tile3dWorkerSource,
+        //   featureAttributes
+        // });
+
+        const b3dm = await new B3dmConverter().convert(i3sAttributesData, featureAttributes);
+
         child.content = {
           uri: `${sourceChild.id}.b3dm`,
           boundingVolume
@@ -290,4 +325,14 @@ export default class Tiles3DConverter {
     console.log(`File(s) size: `, filesSize, ' bytes'); // eslint-disable-line
     console.log(`------------------------------------------------`); // eslint-disable-line
   }
+
+  private async loadWorkers(): Promise<void> {
+    console.log(`Loading workers source...`); // eslint-disable-line no-undef, no-console
+    const tile3dAttributesWorkerUrl = getWorkerURL(Tile3dAttributesWorker, {...getLoaderOptions()});
+    const sourceResponse = await fetchFile(tile3dAttributesWorkerUrl);
+    const source = await sourceResponse.text();
+
+    this.workerSource.tile3dWorkerSource = source;
+    console.log(`Loading workers source completed!`); // eslint-disable-line no-undef, no-console
+  }
 }

package/src/3d-tiles-converter/helpers/b3dm-converter.ts

@@ -1,3 +1,5 @@
+import type {I3SAttributesData} from '../../3d-tiles-attributes-worker';
+
 import {encodeSync} from '@loaders.gl/core';
 import {GLTFScenegraph, GLTFWriter} from '@loaders.gl/gltf';
 import {Tile3DWriter} from '@loaders.gl/3d-tiles';
@@ -21,15 +23,17 @@ export default class B3dmConverter {
    * @param i3sTile - Tile3D instance for I3S node
    * @returns - encoded content
    */
-  async convert(i3sTile: Object, attributes: any = null): Promise<ArrayBuffer> {
-    this.i3sTile = i3sTile;
-    const gltf = await this.buildGltf(i3sTile);
+  async convert(
+    i3sAttributesData: I3SAttributesData,
+    featureAttributes: any = null
+  ): Promise<ArrayBuffer> {
+    const gltf = await this.buildGltf(i3sAttributesData);
     const b3dm = encodeSync(
       {
         gltfEncoded: new Uint8Array(gltf),
         type: 'b3dm',
-        featuresLength: this._getFeaturesLength(attributes),
-        batchTable: attributes
+        featuresLength: this._getFeaturesLength(featureAttributes),
+        batchTable: featureAttributes
       },
       Tile3DWriter
     );
@@ -41,7 +45,8 @@ export default class B3dmConverter {
    * @param i3sTile - Tile3D instance for I3S node
    * @returns - encoded glb content
    */
-  async buildGltf(i3sTile): Promise<ArrayBuffer> {
+  async buildGltf(i3sAttributesData: I3SAttributesData): Promise<ArrayBuffer> {
+    const {tileContent, textureFormat} = i3sAttributesData;
     const {
       material,
       attributes,
@@ -49,10 +54,10 @@ export default class B3dmConverter {
       cartesianOrigin,
       cartographicOrigin,
       modelMatrix
-    } = i3sTile.content;
+    } = tileContent;
     const gltfBuilder = new GLTFScenegraph();
 
-    const textureIndex = await this._addI3sTextureToGltf(i3sTile, gltfBuilder);
+    const textureIndex = await this._addI3sTextureToGltf(tileContent, textureFormat, gltfBuilder);
     const pbrMaterialInfo = this._convertI3sMaterialToGltfMaterial(material, textureIndex);
     const materialIndex = gltfBuilder.addMaterial(pbrMaterialInfo);
 
@@ -101,11 +106,8 @@ export default class B3dmConverter {
    * @param {GLTFScenegraph} gltfBuilder - gltfScenegraph instance to construct GLTF
    * @returns {Promise<number | null>} - GLTF texture index
    */
-  async _addI3sTextureToGltf(i3sTile, gltfBuilder) {
-    const {
-      content: {texture, material, attributes},
-      header: {textureFormat}
-    } = i3sTile;
+  async _addI3sTextureToGltf(tileContent, textureFormat, gltfBuilder) {
+    const {texture, material, attributes} = tileContent;
     let textureIndex = null;
     let selectedTexture = texture;
     if (!texture && material) {

package/src/constants.ts

@@ -0,0 +1,2 @@
+export const BROWSER_ERROR_MESSAGE =
+  'Tile converter does not work in browser, only in node js environment';