@loaders.gl/tile-converter 4.0.0-alpha.10 → 4.0.0-alpha.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "4.0.0-alpha.10",
+  "version": "4.0.0-alpha.12",
   "description": "Converter",
   "license": "MIT",
   "publishConfig": {
@@ -21,6 +21,7 @@
   "sideEffects": false,
   "bin": {
     "tile-converter": "./bin/converter.js",
+    "slpk-extractor": "./bin/slpk-extractor.js",
     "i3s-server": "./src/i3s-server/bin/www"
   },
   "files": [
@@ -37,25 +38,26 @@
     "join-images": false
   },
   "scripts": {
-    "pre-build": "npm run build-bundle && npm run build-converter-bundle && npm run build-i3s-attributes-worker && npm run build-3d-tiles-attributes-worker",
+    "pre-build": "npm run build-bundle && npm run build-converter-bundle && npm run build-i3s-attributes-worker && npm run build-slpk-extractor-bundle && npm run build-3d-tiles-attributes-worker",
     "build-bundle": "esbuild ./src/index.ts --bundle --outfile=dist/dist.min.js --platform=node  --external:join-images",
     "build-converter-bundle": "esbuild src/converter-cli.ts --outfile=dist/converter.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-slpk-extractor-bundle": "esbuild src/slpk-extractor-cli.ts --outfile=dist/slpk-extractor.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\"",
     "build-i3s-attributes-worker": "esbuild src/workers/i3s-attributes-worker.ts --outfile=dist/i3s-attributes-worker.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\"",
     "build-3d-tiles-attributes-worker": "esbuild src/workers/3d-tiles-attributes-worker.ts --outfile=dist/3d-tiles-attributes-worker.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "4.0.0-alpha.10",
-    "@loaders.gl/crypto": "4.0.0-alpha.10",
-    "@loaders.gl/draco": "4.0.0-alpha.10",
-    "@loaders.gl/gltf": "4.0.0-alpha.10",
-    "@loaders.gl/i3s": "4.0.0-alpha.10",
-    "@loaders.gl/images": "4.0.0-alpha.10",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.10",
-    "@loaders.gl/polyfills": "4.0.0-alpha.10",
-    "@loaders.gl/textures": "4.0.0-alpha.10",
-    "@loaders.gl/tiles": "4.0.0-alpha.10",
-    "@loaders.gl/worker-utils": "4.0.0-alpha.10",
-    "@loaders.gl/zip": "4.0.0-alpha.10",
+    "@loaders.gl/3d-tiles": "4.0.0-alpha.12",
+    "@loaders.gl/crypto": "4.0.0-alpha.12",
+    "@loaders.gl/draco": "4.0.0-alpha.12",
+    "@loaders.gl/gltf": "4.0.0-alpha.12",
+    "@loaders.gl/i3s": "4.0.0-alpha.12",
+    "@loaders.gl/images": "4.0.0-alpha.12",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.12",
+    "@loaders.gl/polyfills": "4.0.0-alpha.12",
+    "@loaders.gl/textures": "4.0.0-alpha.12",
+    "@loaders.gl/tiles": "4.0.0-alpha.12",
+    "@loaders.gl/worker-utils": "4.0.0-alpha.12",
+    "@loaders.gl/zip": "4.0.0-alpha.12",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
     "@math.gl/geoid": "^3.5.1",
@@ -78,5 +80,5 @@
     "join-images": "^1.1.3",
     "sharp": "^0.31.3"
   },
-  "gitHead": "7efdbe09e02098aad6d985e4d6465d08806e19a9"
+  "gitHead": "42dfc47a41e3e6089eec22a1e1d4f3387e0cb6e9"
 }

package/src/converter-cli.ts

@@ -3,6 +3,13 @@ import '@loaders.gl/polyfills';
 import {join} from 'path';
 import {I3SConverter, Tiles3DConverter} from '@loaders.gl/tile-converter';
 import {DepsInstaller} from './deps-installer/deps-installer';
+import {
+  getBooleanValue,
+  getIntegerValue,
+  getStringValue,
+  getURLValue,
+  validateOptionsWithEqual
+} from './lib/utils/cli-utils';
 
 type TileConversionOptions = {
   /** "I3S" - for I3S to 3DTiles conversion, "3DTILES" for 3DTiles to I3S conversion */
@@ -213,19 +220,6 @@ function validateOptions(options: TileConversionOptions): ValidatedTileConversio
   return <ValidatedTileConversionOptions>options;
 }
 
-function validateOptionsWithEqual(args: string[]): string[] {
-  return args.reduce((acc: string[], curr) => {
-    const equalSignIndex = curr.indexOf('=');
-    const beforeEqual = curr.slice(0, equalSignIndex);
-    const afterEqual = curr.slice(equalSignIndex + 1, curr.length);
-    const condition = curr.includes('=') && curr.startsWith('--') && afterEqual;
-    if (condition) {
-      return acc.concat(beforeEqual, afterEqual);
-    }
-    return acc.concat(curr);
-  }, []);
-}
-
 /**
  * Parse option from the cli arguments array
  * @param args
@@ -309,62 +303,3 @@ function parseOptions(args: string[]): TileConversionOptions {
   });
   return opts;
 }
-
-/**
- * Get string option value from cli arguments
- * @param index - option's name index in the argument's array.
- *                The value of the option should be next to name of the option.
- * @param args - cli arguments array
- * @returns - string value of the option
- */
-function getStringValue(index: number, args: string[]): string {
-  if (index + 1 >= args.length) {
-    return '';
-  }
-  const value = args[index + 1];
-  if (value.indexOf('--') === 0) {
-    return '';
-  }
-  return value;
-}
-
-/**
- * Modyfy URL path to be compatible with fetch
- * @param index - option's name index in the argument's array.
- *                The value of the option should be next to name of the option.
- * @param args - cli arguments array
- * @returns - string value of the option
- */
-function getURLValue(index: number, args: string[]): string {
-  const value = getStringValue(index, args);
-  console.log(`Input tileset value: ${value}`);
-  console.log(`Modified tileset value: ${value.replace(/\\/g, '/')}`);
-  return value.replace(/\\/g, '/');
-}
-
-/**
- * Get integer option value from cli arguments
- * @param index - option's name index in the argument's array
- *                The value of the option should be next to name of the option.
- * @param args - cli arguments array
- * @returns - number value of the option
- */
-function getIntegerValue(index: number, args: string[]): number {
-  const stringValue: string = getStringValue(index, args);
-  const result: number = Number.parseInt(stringValue);
-  if (isFinite(result)) {
-    return result;
-  }
-  return NaN;
-}
-
-function getBooleanValue(index: number, args: string[]): boolean {
-  const stringValue: string = getStringValue(index, args).toLowerCase().trim();
-  if (['--no-draco', '--split-nodes'].includes(args[index]) && !stringValue) {
-    return false;
-  }
-  if (!stringValue || stringValue === 'true') {
-    return true;
-  }
-  return false;
-}

package/src/i3s-converter/helpers/coordinate-converter.ts

@@ -103,7 +103,6 @@ export function convertPositionsToVectors(positions: Float32Array): Vector3[] {
 
 /**
  * Convert common coordinate to fullExtent https://github.com/Esri/i3s-spec/blob/master/docs/1.8/fullExtent.cmn.md
- * @param
  * @param boundingVolume
  * @returns - fullExtent object
  */
@@ -142,7 +141,7 @@ export function convertBoundingVolumeToI3SFullExtent(
 /**
  * Creates oriented boundinb box from mbs.
  * @param mbs - Minimum Bounding Sphere
- * @returns - Oriented BOunding Box
+ * @returns - Oriented Bounding Box
  */
 export function createObbFromMbs(mbs: Mbs): Obb {
   const radius = mbs[3];

package/src/i3s-converter/helpers/gltf-attributes.ts

@@ -1,21 +1,17 @@
 import type {Tiles3DTileContent} from '@loaders.gl/3d-tiles';
 import type {GLTFAccessorPostprocessed, GLTFNodePostprocessed} from '@loaders.gl/gltf';
 import type {B3DMAttributesData} from '../../i3s-attributes-worker';
-import {Matrix4, Vector3} from '@math.gl/core';
+import {Matrix4, TypedArray, Vector3} from '@math.gl/core';
 import {BoundingSphere, OrientedBoundingBox} from '@math.gl/culling';
 import {Ellipsoid} from '@math.gl/geospatial';
 
-type AttributesObject = {
-  [k: string]: GLTFAccessorPostprocessed;
-};
-
 /**
  * Prepare attributes for conversion to avoid binary data breaking in worker thread.
  * @param tileContent - 3DTiles tile content
  * @param tileTransform - transformation matrix of the tile, calculated recursively multiplying
  *                        transform of all parent tiles and transform of the current tile
  * @param boundingVolume - initialized bounding volume of the source tile
- * @returns
+ * @returns 3DTiles content data, prepared for conversion
  */
 export function prepareDataForAttributesConversion(
   tileContent: Tiles3DTileContent,
@@ -67,9 +63,12 @@ export function prepareDataForAttributesConversion(
 /**
  * Keep only values for glTF attributes to pass data to worker thread.
  * @param attributes - geometry attributes
+ * @returns attributes with only `value` item
  */
-function getB3DMAttributesWithoutBufferView(attributes: AttributesObject): AttributesObject {
-  const attributesWithoutBufferView = {};
+function getB3DMAttributesWithoutBufferView(
+  attributes: Record<string, GLTFAccessorPostprocessed>
+): Record<string, {value: TypedArray}> {
+  const attributesWithoutBufferView: Record<string, {value: TypedArray}> = {};
 
   for (const attributeName in attributes) {
     attributesWithoutBufferView[attributeName] = {

package/src/i3s-converter/helpers/preprocess-3d-tiles.ts

@@ -1,4 +1,4 @@
-import {Tiles3DTileContent, Tiles3DTileJSONPostprocessed} from '@loaders.gl/3d-tiles';
+import {Tiles3DTileContent} from '@loaders.gl/3d-tiles';
 import {GltfPrimitiveModeString, PreprocessData} from '../types';
 import {GLTF, GLTFLoader} from '@loaders.gl/gltf';
 import {parse} from '@loaders.gl/core';
@@ -20,12 +20,10 @@ export const GLTF_PRIMITIVE_MODES = [
 /**
  * Analyze tile content. This function is used during preprocess stage of
  * conversion
- * @param tile - 3DTiles tile JSON metadata
  * @param tileContent - 3DTiles tile content ArrayBuffer
  * @returns
  */
 export const analyzeTileContent = async (
-  tile: Tiles3DTileJSONPostprocessed,
   tileContent: Tiles3DTileContent | null
 ): Promise<PreprocessData> => {
   const result: PreprocessData = {

package/src/i3s-converter/i3s-converter.ts

@@ -313,11 +313,19 @@ export default class I3SConverter {
     if (sourceTile.id) {
       console.log(`[analyze]: ${sourceTile.id}`); // eslint-disable-line
     }
-    const tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, {
-      ...this.loadOptions,
-      '3d-tiles': {...this.loadOptions['3d-tiles'], loadGLTF: false}
-    });
-    const tilePreprocessData = await analyzeTileContent(sourceTile, tileContent);
+
+    let tileContent: Tiles3DTileContent | null = null;
+    try {
+      tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, {
+        ...this.loadOptions,
+        '3d-tiles': {...this.loadOptions['3d-tiles'], loadGLTF: false}
+      });
+    } catch (error) {
+      console.log(
+        `[warning]: Failed to load ${sourceTile.contentUrl}. An I3S tile with empty content will be added to the output tileset`
+      );
+    }
+    const tilePreprocessData = await analyzeTileContent(tileContent);
     mergePreprocessData(this.preprocessData, tilePreprocessData);
 
     return null;
@@ -576,7 +584,12 @@ export default class I3SConverter {
 
     await this._updateTilesetOptions();
 
-    const tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, this.loadOptions);
+    let tileContent: Tiles3DTileContent | null = null;
+    try {
+      tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, this.loadOptions);
+    } catch (error) {
+      console.log(`[warning]: Failed to load ${sourceTile.contentUrl}`);
+    }
     const sourceBoundingVolume = createBoundingVolume(
       sourceTile.boundingVolume,
       transformationMatrix,
@@ -753,13 +766,17 @@ export default class I3SConverter {
     }
 
     let nodeId = resources.nodeId;
-    let node;
+    let node: NodeInPage;
     if (!nodeId) {
       node = await this.nodePages.push(nodeInPage, parentId);
     } else {
       node = await this.nodePages.getNodeById(nodeId);
     }
 
+    if (!nodeInPage.mesh) {
+      console.log(`[warning]: node ${node.index} is created with empty content`);
+    }
+
     NodePages.updateAll(node, nodeInPage);
     if (meshMaterial) {
       NodePages.updateMaterialByNodeId(node, this._findOrCreateMaterial(meshMaterial));

package/src/lib/utils/cli-utils.ts

@@ -0,0 +1,78 @@
+/**
+ * Get string option value from cli arguments
+ * @param index - option's name index in the argument's array.
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - string value of the option
+ */
+export function getStringValue(index: number, args: string[]): string {
+  if (index + 1 >= args.length) {
+    return '';
+  }
+  const value = args[index + 1];
+  if (value.indexOf('--') === 0) {
+    return '';
+  }
+  return value;
+}
+
+/**
+ * Modyfy URL path to be compatible with fetch
+ * @param index - option's name index in the argument's array.
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - string value of the option
+ */
+export function getURLValue(index: number, args: string[]): string {
+  const value = getStringValue(index, args);
+  console.log(`Input tileset value: ${value}`);
+  console.log(`Modified tileset value: ${value.replace(/\\/g, '/')}`);
+  return value.replace(/\\/g, '/');
+}
+
+export function validateOptionsWithEqual(args: string[]): string[] {
+  return args.reduce((acc: string[], curr) => {
+    const equalSignIndex = curr.indexOf('=');
+    const beforeEqual = curr.slice(0, equalSignIndex);
+    const afterEqual = curr.slice(equalSignIndex + 1, curr.length);
+    const condition = curr.includes('=') && curr.startsWith('--') && afterEqual;
+    if (condition) {
+      return acc.concat(beforeEqual, afterEqual);
+    }
+    return acc.concat(curr);
+  }, []);
+}
+
+/**
+ * Get integer option value from cli arguments
+ * @param index - option's name index in the argument's array
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - number value of the option
+ */
+export function getIntegerValue(index: number, args: string[]): number {
+  const stringValue: string = getStringValue(index, args);
+  const result: number = Number.parseInt(stringValue);
+  if (isFinite(result)) {
+    return result;
+  }
+  return NaN;
+}
+
+/**
+ * Get boolean option value from cli arguments
+ * @param index - option's name index in the argument's array
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - boolean value of the option
+ */
+export function getBooleanValue(index: number, args: string[]): boolean {
+  const stringValue: string = getStringValue(index, args).toLowerCase().trim();
+  if (['--no-draco', '--split-nodes'].includes(args[index]) && !stringValue) {
+    return false;
+  }
+  if (!stringValue || stringValue === 'true') {
+    return true;
+  }
+  return false;
+}

package/src/slpk-extractor/helpers/file-handle-provider.ts

@@ -0,0 +1,91 @@
+import {FileProvider} from '@loaders.gl/i3s';
+import {promises as fsPromises, PathLike} from 'fs';
+
+/**
+ * Provides file data using node fs library
+ */
+export class FileHandleProvider implements FileProvider {
+  /**
+   * Returns a new copy of FileHandleProvider
+   * @param path The path to the file in file system
+   */
+  static async from(path: PathLike): Promise<FileHandleProvider> {
+    const fileDescriptor = await fsPromises.open(path);
+    return new FileHandleProvider(fileDescriptor, (await fileDescriptor.stat()).size);
+  }
+
+  /**
+   * The FileHandle from which data is provided
+   */
+  private fileDescriptor: fsPromises.FileHandle;
+
+  /**
+   * The file length in bytes
+   */
+  private size: number;
+
+  private constructor(fileDescriptor: fsPromises.FileHandle, size: number) {
+    this.fileDescriptor = fileDescriptor;
+    this.size = size;
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer (unsigned byte) at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: number): Promise<number> {
+    const val = new Uint8Array(
+      (await this.fileDescriptor.read(Buffer.alloc(1), 0, 1, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer (unsigned byte) at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: number): Promise<number> {
+    const val = new Uint16Array(
+      (await this.fileDescriptor.read(Buffer.alloc(2), 0, 2, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: number): Promise<number> {
+    const val = new Uint32Array(
+      (await this.fileDescriptor.read(Buffer.alloc(4), 0, 4, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: number, endOffset: number): Promise<ArrayBuffer> {
+    const length = endOffset - startOffset;
+    return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffset)).buffer
+      .buffer;
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length(): number {
+    return this.size;
+  }
+}

package/src/slpk-extractor/slpk-extractor.ts

@@ -0,0 +1,102 @@
+import {isBrowser} from '@loaders.gl/core';
+
+import {BROWSER_ERROR_MESSAGE} from '../constants';
+import {FileHandleProvider} from './helpers/file-handle-provider';
+import {parseZipLocalFileHeader} from '@loaders.gl/i3s';
+import {path} from '@loaders.gl/loader-utils';
+import {GZipCompression} from '@loaders.gl/compression';
+import {writeFile} from '../lib/utils/file-utils';
+
+/**
+ * names of files that should be changed to index
+ */
+const indexNames = [
+  '3dSceneLayer.json.gz',
+  '3dNodeIndexDocument.json.gz',
+  'sharedResource.json.gz'
+];
+
+/**
+ * Description of the file in the SLPK
+ */
+type File = {
+  name: string | null;
+  data: ArrayBuffer;
+};
+
+/**
+ * Converter from slpk to i3s
+ */
+export default class SLPKExtractor {
+  /**
+   * extract slpk to i3s
+   * @param options
+   * @param options.inputUrl the url to read SLPK file
+   * @param options.outputPath the output filename
+   */
+  public async extract(options: {inputUrl: string; outputPath: string}): Promise<string> {
+    if (isBrowser) {
+      console.log(BROWSER_ERROR_MESSAGE);
+      return BROWSER_ERROR_MESSAGE;
+    }
+    const {inputUrl} = options;
+
+    const provider = await FileHandleProvider.from(inputUrl);
+
+    let localHeader = await parseZipLocalFileHeader(0, provider);
+    while (localHeader) {
+      await this.writeFile(
+        await this.unGzip({
+          name: this.correctIndexNames(localHeader.fileName),
+          data: await provider.slice(
+            localHeader.fileDataOffset,
+            localHeader.fileDataOffset + localHeader.compressedSize
+          )
+        }),
+        options.outputPath
+      );
+      localHeader = await parseZipLocalFileHeader(
+        localHeader?.fileDataOffset + localHeader?.compressedSize,
+        provider
+      );
+    }
+
+    return 'success';
+  }
+
+  /**
+   * Defines file name and path for i3s format
+   * @param fileName initial file name and path
+   */
+  private correctIndexNames(fileName: string): string | null {
+    if (indexNames.includes(path.filename(path.join('/', fileName)))) {
+      return path.join(path.dirname(fileName), 'index.json.gz');
+    }
+    // finds path with name part and extention part
+    let parts = /^(.*\/[^\/\.]*)(\..+)$/.exec(fileName);
+    if (!parts) {
+      return null;
+    }
+    return `${parts?.at(1)}/index${parts?.at(2)}`;
+  }
+
+  private async unGzip(file: File): Promise<File> {
+    if (/\.gz$/.test(file.name ?? '')) {
+      const compression = new GZipCompression();
+
+      const decompressedData = await compression.decompress(file.data);
+      return {data: decompressedData, name: (file.name ?? '').slice(0, -3)};
+    }
+    return Promise.resolve(file);
+  }
+
+  private async writeFile(options: File, outputPath: string): Promise<void> {
+    if (!options.name) {
+      return;
+    }
+    const finalPath = path.join(outputPath, options.name);
+    const dirName = path.dirname(finalPath);
+    const fileName = path.filename(finalPath);
+    await writeFile(dirName, options.data, fileName);
+  }
+}