@loaders.gl/tile-converter 4.0.0-alpha.18 → 4.0.0-alpha.19

package/dist/i3s-converter/i3s-converter.js

@@ -51,7 +51,6 @@ const textures_1 = require("@loaders.gl/textures");
 const images_1 = require("@loaders.gl/images");
 const types_1 = require("./types");
 const worker_utils_1 = require("@loaders.gl/worker-utils");
-const draco_1 = require("@loaders.gl/draco");
 const write_queue_1 = __importDefault(require("../lib/utils/write-queue"));
 const constants_1 = require("../constants");
 const feature_attributes_1 = require("./helpers/feature-attributes");
@@ -81,6 +80,7 @@ class I3SConverter {
         this.loadOptions = {
             _nodeWorkers: true,
             reuseWorkers: true,
+            useLocalLibraries: true,
             basis: {
                 format: 'rgba32',
                 // We need to load local fs workers because nodejs can't load workers from the Internet
@@ -88,7 +88,8 @@ class I3SConverter {
             },
             // We need to load local fs workers because nodejs can't load workers from the Internet
             draco: { workerUrl: './modules/draco/dist/draco-worker-node.js' },
-            fetch: {}
+            fetch: {},
+            modules: {}
         };
         this.geoidHeightModel = null;
         this.Loader = _3d_tiles_1.Tiles3DLoader;
@@ -166,7 +167,6 @@ class I3SConverter {
         if (slpk) {
             this.nodePages.useWriteFunction(file_utils_1.writeFileForSlpk);
         }
-        await this.loadWorkers();
         try {
             const preloadOptions = await this._fetchPreloadOptions();
             let tilesetUrl = inputUrl;
@@ -513,7 +513,7 @@ class I3SConverter {
             halfSize: [],
             quaternion: []
         };
-        const resourcesData = await (0, geometry_converter_1.default)(tileContent, transformationMatrix, boundingVolume, async () => (await this.nodePages.push({ index: 0, obb: draftObb }, parentId)).index, propertyTable, this.featuresHashArray, this.layers0?.attributeStorageInfo, this.options.draco, this.generateBoundingVolumes, this.options.mergeMaterials, this.geoidHeightModel, this.workerSource);
+        const resourcesData = await (0, geometry_converter_1.default)(tileContent, transformationMatrix, boundingVolume, async () => (await this.nodePages.push({ index: 0, obb: draftObb }, parentId)).index, propertyTable, this.featuresHashArray, this.layers0?.attributeStorageInfo, this.options.draco, this.generateBoundingVolumes, this.options.mergeMaterials, this.geoidHeightModel, this.loadOptions.modules);
         return resourcesData;
     }
     /**
@@ -688,9 +688,13 @@ class I3SConverter {
                         const arrayToEncode = new Uint8Array(copyArrayBuffer);
                         const ktx2TextureData = (0, core_1.encode)({ ...texture.image, data: arrayToEncode }, textures_1.KTX2BasisWriterWorker, {
                             ...textures_1.KTX2BasisWriterWorker.options,
-                            source: this.workerSource.ktx2,
+                            ['ktx2-basis-writer']: {
+                                // We need to load local fs workers because nodejs can't load workers from the Internet
+                                workerUrl: './modules/textures/dist/ktx2-basis-writer-worker-node.js'
+                            },
                             reuseWorkers: true,
-                            _nodeWorkers: true
+                            _nodeWorkers: true,
+                            useLocalLibraries: true
                         });
                         await this.writeTextureFile(ktx2TextureData, '1', 'ktx2', childPath, slpkChildPath);
                     }
@@ -905,21 +909,5 @@ class I3SConverter {
     isContentSupported(sourceTile) {
         return ['b3dm', 'glTF', 'scenegraph'].includes(sourceTile.type || '');
     }
-    async loadWorkers() {
-        console.log(`Loading workers source...`); // eslint-disable-line no-undef, no-console
-        if (this.options.draco) {
-            const url = (0, worker_utils_1.getWorkerURL)(draco_1.DracoWriterWorker, { ...(0, core_1.getLoaderOptions)() });
-            const sourceResponse = await (0, core_1.fetchFile)(url);
-            const source = await sourceResponse.text();
-            this.workerSource.draco = source;
-        }
-        if (this.generateTextures) {
-            const url = (0, worker_utils_1.getWorkerURL)(textures_1.KTX2BasisWriterWorker, { ...(0, core_1.getLoaderOptions)() });
-            const sourceResponse = await (0, core_1.fetchFile)(url);
-            const source = await sourceResponse.text();
-            this.workerSource.ktx2 = source;
-        }
-        console.log(`Loading workers source completed!`); // eslint-disable-line no-undef, no-console
-    }
 }
 exports.default = I3SConverter;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "4.0.0-alpha.18",
+  "version": "4.0.0-alpha.19",
   "description": "Converter",
   "license": "MIT",
   "publishConfig": {
@@ -45,18 +45,18 @@
     "build-i3s-server-bundle": "esbuild src/i3s-server/bin/www.ts --outfile=dist/esm/i3s-server/bin/i3s-server.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "4.0.0-alpha.18",
-    "@loaders.gl/crypto": "4.0.0-alpha.18",
-    "@loaders.gl/draco": "4.0.0-alpha.18",
-    "@loaders.gl/gltf": "4.0.0-alpha.18",
-    "@loaders.gl/i3s": "4.0.0-alpha.18",
-    "@loaders.gl/images": "4.0.0-alpha.18",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.18",
-    "@loaders.gl/polyfills": "4.0.0-alpha.18",
-    "@loaders.gl/textures": "4.0.0-alpha.18",
-    "@loaders.gl/tiles": "4.0.0-alpha.18",
-    "@loaders.gl/worker-utils": "4.0.0-alpha.18",
-    "@loaders.gl/zip": "4.0.0-alpha.18",
+    "@loaders.gl/3d-tiles": "4.0.0-alpha.19",
+    "@loaders.gl/crypto": "4.0.0-alpha.19",
+    "@loaders.gl/draco": "4.0.0-alpha.19",
+    "@loaders.gl/gltf": "4.0.0-alpha.19",
+    "@loaders.gl/i3s": "4.0.0-alpha.19",
+    "@loaders.gl/images": "4.0.0-alpha.19",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.19",
+    "@loaders.gl/polyfills": "4.0.0-alpha.19",
+    "@loaders.gl/textures": "4.0.0-alpha.19",
+    "@loaders.gl/tiles": "4.0.0-alpha.19",
+    "@loaders.gl/worker-utils": "4.0.0-alpha.19",
+    "@loaders.gl/zip": "4.0.0-alpha.19",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
     "@math.gl/geoid": "^3.5.1",
@@ -79,7 +79,7 @@
     "join-images": "^1.1.3",
     "sharp": "^0.31.3"
   },
-  "gitHead": "7bc633f46560f661bdd46cf1015ea27b3694ebce",
+  "gitHead": "2ca50ec4e1d312c124eb7c93c60ab6fd17ee833e",
   "devDependencies": {
     "@types/express": "^4.17.17",
     "@types/node": "^20.4.2"

package/src/deps-installer/deps-installer.ts

@@ -3,6 +3,8 @@ import {ZipLoader} from '@loaders.gl/zip';
 import {writeFile} from '../lib/utils/file-utils';
 import {join} from 'path';
 import {ChildProcessProxy} from '@loaders.gl/worker-utils';
+import {DRACO_EXTERNAL_LIBRARIES, DRACO_EXTERNAL_LIBRARY_URLS} from '@loaders.gl/draco';
+import {BASIS_EXTERNAL_LIBRARIES} from '@loaders.gl/textures';
 
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'beta';
@@ -23,7 +25,7 @@ export class DepsInstaller {
    *    This path is '' by default and is not used by tile-converter.
    *    It is used in tests to prevent rewriting actual workers during tests running
    */
-  async install(path: string = '', workersPath: string = ''): Promise<void> {
+  async install(path: string = ''): Promise<void> {
     console.log('Installing "EGM2008-5" model...'); // eslint-disable-line no-console
     const fileMap = await load(PGM_LINK, ZipLoader, {});
 
@@ -34,14 +36,47 @@ export class DepsInstaller {
 
     await writeFile(depsPath, new Uint8Array(fileMap['geoids/egm2008-5.pgm']), 'egm2008-5.pgm');
 
-    console.log('Installing "I3S Content Loader worker"'); // eslint-disable-line no-console
-    await this.installWorker('i3s', 'i3s-content-worker-node.js', workersPath);
+    console.log('Installing "I3S Content Loader" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('i3s', 'i3s-content-worker-node.js');
 
-    console.log('Installing "Draco Loader worker"'); // eslint-disable-line no-console
-    await this.installWorker('draco', 'draco-worker-node.js', workersPath);
+    console.log('Installing "Draco Loader" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('draco', 'draco-worker-node.js');
 
-    console.log('Installing "Basis Loader worker"'); // eslint-disable-line no-console
-    await this.installWorker('textures', 'basis-worker-node.js', workersPath);
+    console.log('Installing "Draco Writer" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('draco', 'draco-writer-worker-node.js');
+
+    console.log('Installing "Basis Loader" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('textures', 'basis-worker-node.js');
+
+    console.log('Installing "KTX2 Basis Writer" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('textures', 'ktx2-basis-writer-worker-node.js');
+
+    console.log('Installing "Draco decoder" library'); // eslint-disable-line no-console
+    await this.installFromUrl(
+      DRACO_EXTERNAL_LIBRARY_URLS[DRACO_EXTERNAL_LIBRARIES.DECODER],
+      'draco',
+      DRACO_EXTERNAL_LIBRARIES.DECODER
+    );
+    await this.installFromUrl(
+      DRACO_EXTERNAL_LIBRARY_URLS[DRACO_EXTERNAL_LIBRARIES.DECODER_WASM],
+      'draco',
+      DRACO_EXTERNAL_LIBRARIES.DECODER_WASM
+    );
+
+    console.log('Installing "Draco encoder" library'); // eslint-disable-line no-console
+    await this.installFromUrl(
+      DRACO_EXTERNAL_LIBRARY_URLS[DRACO_EXTERNAL_LIBRARIES.ENCODER],
+      'draco',
+      DRACO_EXTERNAL_LIBRARIES.ENCODER
+    );
+
+    console.log('Installing "Basis transcoder" library'); // eslint-disable-line no-console
+    await this.installFromNpm('textures', BASIS_EXTERNAL_LIBRARIES.TRANSCODER, 'libs');
+    await this.installFromNpm('textures', BASIS_EXTERNAL_LIBRARIES.TRANSCODER_WASM, 'libs');
+
+    console.log('Installing "Basis encoder" library'); // eslint-disable-line no-console
+    await this.installFromNpm('textures', BASIS_EXTERNAL_LIBRARIES.ENCODER, 'libs');
+    await this.installFromNpm('textures', BASIS_EXTERNAL_LIBRARIES.ENCODER_WASM, 'libs');
 
     console.log('Installing "join-images" npm package');
     const childProcess = new ChildProcessProxy();
@@ -58,15 +93,25 @@ export class DepsInstaller {
     console.log('All dependencies were installed succesfully.'); // eslint-disable-line no-console
   }
 
-  private async installWorker(module: string, name: string, extraPath: string) {
+  private async installFromNpm(module: string, name: string, extraPath: string = '') {
     const fileResponse = await fetchFile(
-      `https://unpkg.com/@loaders.gl/${module}@${VERSION}/dist/${name}`
+      `https://unpkg.com/@loaders.gl/${module}@${VERSION}/dist/${extraPath}/${name}`
     );
     const fileData = await fileResponse.arrayBuffer();
     if (!fileData) {
       return;
     }
-    const path = join(process.cwd(), extraPath, 'modules', module, 'dist');
+    const path = join(process.cwd(), 'modules', module, 'dist', extraPath);
+    await writeFile(path, fileData, name);
+  }
+
+  private async installFromUrl(url: string, module: string, name: string) {
+    const fileResponse = await fetchFile(url);
+    const fileData = await fileResponse.arrayBuffer();
+    if (!fileData) {
+      return;
+    }
+    const path = join(process.cwd(), 'modules', module, 'dist', 'libs');
     await writeFile(path, fileData, name);
   }
 }

package/src/i3s-converter/helpers/geometry-converter.ts

@@ -87,7 +87,7 @@ let scratchVector = new Vector3();
  * @param generateBoundingVolumes - is converter should create accurate bounding voulmes from geometry attributes
  * @param shouldMergeMaterials - Try to merge similar materials to be able to merge meshes into one node
  * @param geoidHeightModel - model to convert elevation from elipsoidal to geoid
- * @param workerSource - source code of used workers
+ * @param libraries - dynamicaly loaded 3rd-party libraries
  * @returns Array of node resources to create one or more i3s nodes
  */
 export default async function convertB3dmToI3sGeometry(
@@ -102,7 +102,7 @@ export default async function convertB3dmToI3sGeometry(
   generateBoundingVolumes: boolean,
   shouldMergeMaterials: boolean,
   geoidHeightModel: Geoid,
-  workerSource: {[key: string]: string}
+  libraries: Record<string, string>
 ): Promise<I3SConvertedResources[] | null> {
   const useCartesianPositions = generateBoundingVolumes;
   const materialAndTextureList: I3SMaterialWithTexture[] = await convertMaterials(
@@ -156,7 +156,7 @@ export default async function convertB3dmToI3sGeometry(
         propertyTable,
         attributeStorageInfo,
         draco,
-        workerSource
+        libraries
       })
     );
   }
@@ -208,7 +208,7 @@ function _generateBoundingVolumesFromGeometry(
  * @param params.propertyTable - batch table (corresponding to feature attributes data)
  * @param params.attributeStorageInfo - attributes metadata from 3DSceneLayer json
  * @param params.draco - is converter should create draco compressed geometry
- * @param params.workerSource - source code of used workers
+ * @param libraries - dynamicaly loaded 3rd-party libraries
  * @returns Array of I3S node resources
  */
 async function _makeNodeResources({
@@ -221,7 +221,7 @@ async function _makeNodeResources({
   propertyTable,
   attributeStorageInfo,
   draco,
-  workerSource
+  libraries
 }: {
   convertedAttributes: ConvertedAttributes;
   material: I3SMaterialDefinition;
@@ -232,7 +232,7 @@ async function _makeNodeResources({
   propertyTable: FeatureTableJson | null;
   attributeStorageInfo?: AttributeStorageInfo[];
   draco: boolean;
-  workerSource: {[key: string]: string};
+  libraries: Record<string, string>;
 }): Promise<I3SConvertedResources> {
   const boundingVolumes = convertedAttributes.boundingVolumes;
   const vertexCount = convertedAttributes.positions.length / VALUES_PER_VERTEX;
@@ -277,7 +277,7 @@ async function _makeNodeResources({
           featureIds,
           faceRange
         },
-        workerSource.draco
+        libraries
       )
     : null;
 
@@ -1451,17 +1451,17 @@ function generateBigUint64Array(featureIds: any[]): BigUint64Array {
 
 /**
  * Generates draco compressed geometry
- * @param {Number} vertexCount
- * @param {Object} convertedAttributes - get rid of this argument here
- * @param {Object} attributes - geometry attributes to compress
- * @param {string} dracoWorkerSoure - draco worker source code
- * @returns {Promise<object>} - COmpressed geometry.
+ * @param vertexCount
+ * @param convertedAttributes - get rid of this argument here
+ * @param attributes - geometry attributes to compress
+ * @param libraries - dynamicaly loaded 3rd-party libraries
+ * @returns - Compressed geometry.
  */
 async function generateCompressedGeometry(
   vertexCount: number,
   convertedAttributes: Record<string, any>,
   attributes: Record<string, any>,
-  dracoWorkerSoure: string
+  libraries: Record<string, string>
 ): Promise<ArrayBuffer> {
   const {positions, normals, texCoords, colors, uvRegions, featureIds, faceRange} = attributes;
   const indices = new Uint32Array(vertexCount);
@@ -1510,12 +1510,17 @@ async function generateCompressedGeometry(
 
   return encode({attributes: compressedAttributes, indices}, DracoWriterWorker, {
     ...DracoWriterWorker.options,
-    source: dracoWorkerSoure,
     reuseWorkers: true,
     _nodeWorkers: true,
+    modules: libraries,
+    useLocalLibraries: true,
     draco: {
       method: 'MESH_SEQUENTIAL_ENCODING',
       attributesMetadata
+    },
+    ['draco-writer']: {
+      // We need to load local fs workers because nodejs can't load workers from the Internet
+      workerUrl: './modules/draco/dist/draco-writer-worker-node.js'
     }
   });
 }

package/src/i3s-converter/i3s-converter.ts

@@ -14,7 +14,7 @@ import type {
   MaxScreenThresholdSQ,
   NodeInPage
 } from '@loaders.gl/i3s';
-import {load, encode, fetchFile, getLoaderOptions, isBrowser} from '@loaders.gl/core';
+import {load, encode, isBrowser} from '@loaders.gl/core';
 import {CesiumIonLoader, Tiles3DLoader} from '@loaders.gl/3d-tiles';
 import {Geoid} from '@math.gl/geoid';
 import {join} from 'path';
@@ -56,8 +56,7 @@ import {
   PreprocessData,
   SharedResourcesArrays
 } from './types';
-import {getWorkerURL, WorkerFarm} from '@loaders.gl/worker-utils';
-import {DracoWriterWorker} from '@loaders.gl/draco';
+import {WorkerFarm} from '@loaders.gl/worker-utils';
 import WriteQueue from '../lib/utils/write-queue';
 import {BROWSER_ERROR_MESSAGE} from '../constants';
 import {
@@ -111,6 +110,7 @@ export default class I3SConverter {
   loadOptions: Tiles3DLoaderOptions = {
     _nodeWorkers: true,
     reuseWorkers: true,
+    useLocalLibraries: true,
     basis: {
       format: 'rgba32',
       // We need to load local fs workers because nodejs can't load workers from the Internet
@@ -118,7 +118,8 @@ export default class I3SConverter {
     },
     // We need to load local fs workers because nodejs can't load workers from the Internet
     draco: {workerUrl: './modules/draco/dist/draco-worker-node.js'},
-    fetch: {}
+    fetch: {},
+    modules: {}
   };
   geoidHeightModel: Geoid | null = null;
   Loader: LoaderWithParser = Tiles3DLoader;
@@ -236,8 +237,6 @@ export default class I3SConverter {
       this.nodePages.useWriteFunction(writeFileForSlpk);
     }
 
-    await this.loadWorkers();
-
     try {
       const preloadOptions = await this._fetchPreloadOptions();
       let tilesetUrl = inputUrl;
@@ -719,7 +718,7 @@ export default class I3SConverter {
       this.generateBoundingVolumes,
       this.options.mergeMaterials,
       this.geoidHeightModel!,
-      this.workerSource
+      this.loadOptions.modules as Record<string, string>
     );
     return resourcesData;
   }
@@ -935,9 +934,13 @@ export default class I3SConverter {
               KTX2BasisWriterWorker,
               {
                 ...KTX2BasisWriterWorker.options,
-                source: this.workerSource.ktx2,
+                ['ktx2-basis-writer']: {
+                  // We need to load local fs workers because nodejs can't load workers from the Internet
+                  workerUrl: './modules/textures/dist/ktx2-basis-writer-worker-node.js'
+                },
                 reuseWorkers: true,
-                _nodeWorkers: true
+                _nodeWorkers: true,
+                useLocalLibraries: true
               }
             );
 
@@ -1197,23 +1200,4 @@ export default class I3SConverter {
   private isContentSupported(sourceTile: Tiles3DTileJSONPostprocessed): boolean {
     return ['b3dm', 'glTF', 'scenegraph'].includes(sourceTile.type || '');
   }
-
-  private async loadWorkers(): Promise<void> {
-    console.log(`Loading workers source...`); // eslint-disable-line no-undef, no-console
-    if (this.options.draco) {
-      const url = getWorkerURL(DracoWriterWorker, {...getLoaderOptions()});
-      const sourceResponse = await fetchFile(url);
-      const source = await sourceResponse.text();
-      this.workerSource.draco = source;
-    }
-
-    if (this.generateTextures) {
-      const url = getWorkerURL(KTX2BasisWriterWorker, {...getLoaderOptions()});
-      const sourceResponse = await fetchFile(url);
-      const source = await sourceResponse.text();
-      this.workerSource.ktx2 = source;
-    }
-
-    console.log(`Loading workers source completed!`); // eslint-disable-line no-undef, no-console
-  }
 }