@loaders.gl/tile-converter 3.3.0-alpha.1 → 3.3.0-alpha.10

package/src/i3s-converter/json-templates/geometry-definitions.ts

@@ -0,0 +1,83 @@
+import transform from 'json-map-transform';
+
+const PLAIN_GEOMETRY_DEFINITION = () => ({
+  offset: {
+    default: 8
+  },
+  position: {
+    default: {
+      type: 'Float32',
+      component: 3
+    }
+  },
+  normal: {
+    default: {
+      type: 'Float32',
+      component: 3
+    }
+  },
+  uv0: {
+    path: 'hasTexture',
+    transform: (val) => (val && {type: 'Float32', component: 2}) || false,
+    omitValues: [false]
+  },
+  color: {
+    default: {
+      type: 'UInt8',
+      component: 4
+    }
+  },
+  uvRegion: {
+    path: 'hasUvRegions',
+    transform: (val) => (val && {type: 'UInt16', component: 4}) || false,
+    omitValues: [false]
+  },
+  featureId: {
+    default: {
+      binding: 'per-feature',
+      type: 'UInt64',
+      component: 1
+    }
+  },
+  faceRange: {
+    default: {
+      binding: 'per-feature',
+      type: 'UInt32',
+      component: 2
+    }
+  }
+});
+
+const COMPRESSED_GEOMETRY_DEFINITION = () => ({
+  'compressedAttributes.encoding': {
+    default: 'draco'
+  },
+  'compressedAttributes.attributes': {
+    path: 'geometryConfig',
+    transform: (val) => {
+      const result = ['position', 'normal'];
+      if (val.hasTexture) {
+        result.push('uv0');
+      }
+      result.push('color');
+      if (val.hasUvRegions) {
+        result.push('uv-region');
+      }
+      result.push('feature-index');
+      return result;
+    }
+  }
+});
+
+export const GEOMETRY_DEFINITION = () => ({
+  geometryBuffers: {
+    path: 'geometryConfig',
+    transform: (val) => {
+      const result = [transform(val, PLAIN_GEOMETRY_DEFINITION())];
+      if (val.draco) {
+        result.push(transform({geometryConfig: val}, COMPRESSED_GEOMETRY_DEFINITION()));
+      }
+      return result;
+    }
+  }
+});

package/src/i3s-converter/json-templates/layers.ts

@@ -1,76 +1,6 @@
 import transform from 'json-map-transform';
 import {STORE} from './store';
 
-const PLAIN_GEOMETRY_DEFINITION = () => ({
-  offset: 8,
-  position: {
-    type: 'Float32',
-    component: 3
-  },
-  normal: {
-    type: 'Float32',
-    component: 3
-  },
-  uv0: {
-    type: 'Float32',
-    component: 2
-  },
-  color: {
-    type: 'UInt8',
-    component: 4
-  },
-  featureId: {
-    binding: 'per-feature',
-    type: 'UInt64',
-    component: 1
-  },
-  faceRange: {
-    binding: 'per-feature',
-    type: 'UInt32',
-    component: 2
-  }
-});
-
-const PLAIN_GEOMETRY_DEFINITION_WITHOUT_UV0 = () => ({
-  offset: 8,
-  position: {
-    type: 'Float32',
-    component: 3
-  },
-  normal: {
-    type: 'Float32',
-    component: 3
-  },
-  color: {
-    type: 'UInt8',
-    component: 4
-  },
-  featureId: {
-    binding: 'per-feature',
-    type: 'UInt64',
-    component: 1
-  },
-  faceRange: {
-    binding: 'per-feature',
-    type: 'UInt32',
-    component: 2
-  }
-});
-
-const COMPRESSED_GEOMETRY_DEFINITION = () => ({
-  compressedAttributes: {
-    encoding: 'draco',
-    attributes: ['position', 'normal', 'uv0', 'color', 'feature-index']
-  }
-});
-
-const COMPRESSED_GEOMETRY_DEFINITION_WITHOUT_UV0 = () => ({
-  compressedAttributes: {
-    encoding: 'draco',
-    attributes: ['position', 'normal', 'color', 'feature-index']
-  }
-});
-
 const SPATIAL_REFERENCE = () => ({
   wkid: {
     path: 'wkid',
@@ -189,27 +119,8 @@ export const LAYERS = () => ({
     default: []
   },
   geometryDefinitions: {
-    path: 'compressGeometry',
-    transform: (val) => {
-      const result = [{geometryBuffers: []}, {geometryBuffers: []}];
-
-      // @ts-expect-error
-      result[0].geometryBuffers.push(PLAIN_GEOMETRY_DEFINITION());
-      // @ts-expect-error
-      result[1].geometryBuffers.push(PLAIN_GEOMETRY_DEFINITION_WITHOUT_UV0());
-      if (val) {
-        // @ts-expect-error
-        result[0].geometryBuffers.push(COMPRESSED_GEOMETRY_DEFINITION());
-        // @ts-expect-error
-        result[1].geometryBuffers.push(COMPRESSED_GEOMETRY_DEFINITION_WITHOUT_UV0());
-      }
-      return result;
-    },
-    default: [
-      {
-        geometryBuffers: [PLAIN_GEOMETRY_DEFINITION(), PLAIN_GEOMETRY_DEFINITION_WITHOUT_UV0()]
-      }
-    ]
+    path: 'geometryDefinitions',
+    default: []
   },
   attributeStorageInfo: {
     path: 'attributeStorageInfo',

package/src/i3s-converter/json-templates/shared-resources.ts

@@ -15,15 +15,15 @@ const MATERIAL_DEFINITION_INFO_PARAMS = () => ({
   },
   ambient: {
     path: 'ambient',
-    default: [1, 1, 1, 1]
+    default: [1, 1, 1]
   },
   diffuse: {
     path: 'diffuse',
-    default: [1, 1, 1, 1]
+    default: [1, 1, 1]
   },
   specular: {
     path: 'specular',
-    default: [0, 0, 0, 0]
+    default: [0, 0, 0]
   },
   useVertexColorAlpha: {
     path: 'useVertexColorAlpha',

package/src/i3s-converter/types.ts

@@ -1,13 +1,17 @@
+import {GLTFImagePostprocessed} from '@loaders.gl/gltf';
 import {
   BoundingVolumes,
   I3SMaterialDefinition,
   MaterialDefinitionInfo,
   TextureDefinitionInfo
 } from '@loaders.gl/i3s';
-import {ImageDataType} from '@loaders.gl/images';
 
 /** Converted resources for specific node */
 export type I3SConvertedResources = {
+  /**
+   * Node id
+   */
+  nodeId?: number;
   /** Non-compressed geometry buffer that have structure met
    * https://github.com/Esri/i3s-spec/blob/master/docs/1.8/defaultGeometrySchema.cmn.md
    * (Geometry buffer)
@@ -21,6 +25,10 @@ export type I3SConvertedResources = {
    * Texture image content
    */
   texture: any | null;
+  /**
+   * If the resource has uvRegions geometry attribute
+   */
+  hasUvRegions: boolean;
   /**
    * Shared resources built from GLTF material
    */
@@ -59,6 +67,8 @@ export type ConvertedAttributes = {
   texCoords: Float32Array;
   /** COLOR_0 attribute value */
   colors: Uint8Array;
+  /** uvRegion attribute for a texture atlas */
+  uvRegions: Uint16Array;
   /** Feature indices grouped by ...
    * converted from "batch ids" of GLTF
    */
@@ -69,6 +79,8 @@ export type ConvertedAttributes = {
    * MBS and/or OBB bounding volumes of the node
    */
   boundingVolumes: null | BoundingVolumes;
+  /** merged materials data */
+  mergedMaterials: MergedMaterial[];
 };
 
 /** Postprocessed geometry and feature attributes
@@ -83,6 +95,8 @@ export type GeometryAttributes = {
   texCoords: Float32Array;
   /** COLOR_0 attribute value */
   colors: Uint8Array;
+  /** uvRegion attribute for a texture atlas */
+  uvRegions: Uint16Array;
   /** faceRanges attribute value  */
   faceRange: Uint32Array;
   /** feature Ids attribute value */
@@ -101,6 +115,8 @@ export type GroupedByFeatureIdAttributes = {
   normals: Float32Array;
   /** COLOR_0 attribute value */
   colors: Uint8Array;
+  /** uvRegion attribute for a texture atlas */
+  uvRegions: Uint16Array;
   /** TEXCOORD_0 attribute value */
   texCoords: Float32Array;
 };
@@ -120,56 +136,20 @@ export type I3SMaterialWithTexture = {
   /** Material definition https://github.com/Esri/i3s-spec/blob/master/docs/1.8/materialDefinitions.cmn.md */
   material: I3SMaterialDefinition;
   /** Texture content (image) */
-  texture?: ImageDataType;
-};
-
-/**
- * 3DTilesNext EXT_feature_metadata extension
- * Spec - https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_feature_metadata
- */
-export type ExtFeatureMetadata = {
-  /** Feature ids definition in attributes */
-  featureIdAttributes?: ExtFeatureMetadataAttribute[];
-  /** Feature ids definition in textures */
-  featureIdTextures?: ExtFeatureMetadataAttribute[];
+  texture?: GLTFImagePostprocessed;
+  /** Metadata of all merged materials */
+  mergedMaterials: MergedMaterial[];
 };
 
-/**
- * Attribute which described featureIds definition.
- */
-export type ExtFeatureMetadataAttribute = {
-  /** Name of feature table */
-  featureTable: string;
-  /** Described how feature ids are defined */
-  featureIds: ExtFeatureMetadataFeatureIds;
-};
-
-/**
- * Defining featureIds by attributes or implicitly.
- */
-type ExtFeatureMetadataFeatureIds = {
-  /** Name of attribute where featureIds are defined */
-  attribute?: string;
-  /** Sets a constant feature ID for each vertex. The default is 0. */
-  constant?: number;
-  /** Sets the rate at which feature IDs increment.
-   * If divisor is zero then constant is used.
-   * If divisor is greater than zero the feature ID increments once per divisor sets of vertices, starting at constant.
-   * The default is 0
-   */
-  divisor?: number;
-  /** gLTF textureInfo object - https://github.com/CesiumGS/glTF/blob/3d-tiles-next/specification/2.0/schema/textureInfo.schema.json */
-  texture?: ExtFeatureMetadataTexture;
-  /** Must be a single channel ("r", "g", "b", or "a") */
-  channels?: 'r' | 'g' | 'b' | 'a';
-};
-
-/**
- * Reference to a texture.
- */
-type ExtFeatureMetadataTexture = {
-  /** The set index of texture's TEXCOORD attribute used for texture coordinate mapping.*/
-  texCoord: number;
-  /** The index of the texture. */
-  index: number;
+/** Metadata of some original texture */
+export type MergedMaterial = {
+  /** Gltf material Id */
+  originalMaterialId: string;
+  /** Original texture size */
+  textureSize?: {
+    width: number;
+    height: number;
+  };
+  /** Uint16Array of 4 elements https://github.com/Esri/i3s-spec/blob/master/docs/1.7/geometryUVRegion.cmn.md */
+  uvRegion?: Uint16Array;
 };

package/src/lib/utils/file-utils.ts

@@ -1,3 +1,5 @@
+import {load} from '@loaders.gl/core';
+import {JSONLoader} from '@loaders.gl/loader-utils';
 import {promises as fs} from 'fs';
 import {isAbsolute, join} from 'path';
 import {compressFileWithGzip} from './compress-util';
@@ -40,23 +42,76 @@ export async function writeFile(
  * @param data - file content
  * @param fileName - name of output file (default: index.json)
  * @param compress - if need to compress file with gzip (default: true)
+ * @param compressList - if set - the file should be added to this list and compressed in the end of conversion
  */
 export async function writeFileForSlpk(
   path: string,
   data: string | Uint8Array | ArrayBuffer | Promise<ArrayBuffer>,
   fileName: string = 'index.json',
-  compress: boolean = true
-): Promise<string> {
+  compress: boolean = true,
+  compressList?: string[] | null
+): Promise<string | null> {
   const pathFile = await writeFile(path, data, fileName);
   if (compress) {
-    const pathGzFile = await compressFileWithGzip(pathFile);
-    // After compression, we don't need an uncompressed file
-    await removeFile(pathFile);
-    return pathGzFile;
+    if (compressList) {
+      if (!compressList.includes(pathFile)) {
+        compressList.push(pathFile);
+        return `${pathFile}.gz`;
+      } else {
+        return null;
+      }
+    } else {
+      const pathGzFile = await compressFileWithGzip(pathFile);
+      // After compression, we don't need an uncompressed file
+      await removeFile(pathFile);
+      return pathGzFile;
+    }
   }
   return pathFile;
 }
 
+/**
+ * Open json file
+ * @param path - path to the file
+ * @param fileName - file name
+ * @returns object
+ */
+export async function openJson(path: string, fileName: string): Promise<{[key: string]: any}> {
+  return new Promise((resolve, reject) => {
+    let count = 0;
+    console.log(`load ${path}/${fileName}.`); // eslint-disable-line
+    const intervalId = setInterval(() => {
+      const pathFile = join(path, fileName);
+      load(pathFile, JSONLoader)
+        .then((result) => {
+          clearInterval(intervalId);
+          resolve(result);
+        })
+        .catch(() => {
+          count++;
+          if (count > 100) {
+            clearInterval(intervalId);
+            reject(new Error(`Cannon load ${path}/${fileName}.`));
+          }
+        });
+    }, 200);
+  });
+}
+
+/**
+ * Check if the file exists
+ * @param fileName - full name of file
+ * @returns true if file exists, otherwise - false
+ */
+export async function isFileExists(fileName: string): Promise<boolean> {
+  try {
+    await fs.stat(fileName);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 /**
  * Remove dir with path
  *
@@ -80,5 +135,5 @@ export function removeFile(path: string) {
  * @param filePath
  */
 export function getAbsoluteFilePath(filePath: string) {
-  return isAbsolute(filePath) ? filePath : join(process.cwd(), filePath); // eslint-disable-line no-undef
+  return isAbsolute(filePath) ? filePath : join(process.cwd(), filePath);
 }

package/src/lib/utils/write-queue.ts

@@ -1,8 +1,28 @@
 import {Queue} from './queue';
+import process from 'process';
+
+/** Memory limit size is based on testing */
+const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB
 
 export type WriteQueueItem = {
   archiveKey?: string;
-  writePromise: Promise<string>;
+  /**
+   * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises);
+   * Arguments for this call are specified in writeQueue.enqueue call like this:
+   * await writeQueue.enqueue({
+   *     archiveKey: `nodePages/xxx.json.gz`,
+   *     writePromise: () => writeFileForSlpk(slpkPath, data, `xxx.json`)
+   * });
+   * Note, a function like writeFileForSlpk should NOT be called when initializing the object for enqueue().
+   * If he function is called, the promise will be created
+   * and the function will allocate resources (file descriptors) for file writing.
+   * It will be done for ALL items in the queue, which is not supposed to happen.
+   * That's why the function should be passed as
+   *   writePromise: () => writeFileForSlpk(slpkPath, content, `xxx.json`)
+   * instead of
+   *  writePromise: writeFileForSlpk(slpkPath, content, `xxx.json`) // INCORRECT !
+   */
+  writePromise: () => Promise<string | null>;
 };
 
 export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
@@ -18,6 +38,22 @@ export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
     this.writeConcurrency = writeConcurrency;
   }
 
+  async enqueue(val: T, writeImmediately: boolean = false) {
+    if (writeImmediately) {
+      const {archiveKey, writePromise} = val as WriteQueueItem;
+      const result = await writePromise();
+      if (archiveKey && result) {
+        this.fileMap[archiveKey] = result;
+      }
+    } else {
+      super.enqueue(val);
+      /** https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_memoryusage */
+      if (process.memoryUsage().rss > MEMORY_LIMIT) {
+        await this.startWrite();
+      }
+    }
+  }
+
   startListening() {
     this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);
   }
@@ -29,12 +65,9 @@ export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
   }
 
   async startWrite(): Promise<void> {
-    if (this.writePromise) {
-      await this.writePromise;
-      this.writePromise = null;
-      return;
+    if (!this.writePromise) {
+      this.writePromise = this.doWrite();
     }
-    this.writePromise = this.doWrite();
     await this.writePromise;
     this.writePromise = null;
   }
@@ -46,7 +79,7 @@ export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
 
   private async doWrite(): Promise<void> {
     while (this.length) {
-      const promises: Promise<string>[] = [];
+      const promises: Promise<string | null>[] = [];
       const archiveKeys: (string | undefined)[] = [];
       for (let i = 0; i < this.writeConcurrency; i++) {
         const item = this.dequeue();
@@ -55,17 +88,17 @@ export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
         }
         const {archiveKey, writePromise} = item as WriteQueueItem;
         archiveKeys.push(archiveKey);
-        promises.push(writePromise);
+        const promise = writePromise();
+        promises.push(promise);
       }
       const writeResults = await Promise.allSettled(promises);
       this.updateFileMap(archiveKeys, writeResults);
     }
-    this.writePromise = null;
   }
 
   private updateFileMap(
     archiveKeys: (string | undefined)[],
-    writeResults: PromiseSettledResult<string>[]
+    writeResults: PromiseSettledResult<string | null>[]
   ) {
     for (let i = 0; i < archiveKeys.length; i++) {
       const archiveKey = archiveKeys[i];

package/src/pgm-loader.ts

@@ -1,11 +1,10 @@
 import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import {parsePGM} from '@math.gl/geoid';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-import {parsePGM} from '@math.gl/geoid';
-
 /**
  * Loader for PGM - Netpbm grayscale image format
  */
@@ -15,6 +14,7 @@ export const PGMLoader: LoaderWithParser = {
   module: 'tile-converter',
   version: VERSION,
   mimeTypes: ['image/x-portable-graymap'],
+  // @ts-expect-error LoaderOptions does not have cubic parameter
   parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options),
   extensions: ['pgm'],
   options: {

package/src/workers/3d-tiles-attributes-worker.ts

@@ -3,4 +3,4 @@ import B3dmConverter from '../3d-tiles-converter/helpers/b3dm-converter';
 
 const b3dmConverter = new B3dmConverter();
 
-createWorker(async (data, options = {}) => b3dmConverter.convert(data, options.attributes));
+createWorker(async (data, options = {}) => b3dmConverter.convert(data, options.featureAttributes));

package/src/workers/i3s-attributes-worker.ts

@@ -2,5 +2,6 @@ import {createWorker} from '@loaders.gl/worker-utils';
 import {convertAttributes} from '../i3s-converter/helpers/geometry-converter';
 
 createWorker(
-  async (data, options = {}) => await convertAttributes(data, options.useCartesianPositions)
+  async (data, options = {}) =>
+    await convertAttributes(data, options.materialAndTextureList, options.useCartesianPositions)
 );