@loaders.gl/tile-converter 4.0.0-alpha.5 → 4.0.0-alpha.6

package/src/i3s-converter/json-templates/geometry-definitions.ts

@@ -0,0 +1,83 @@
+import transform from 'json-map-transform';
+
+const PLAIN_GEOMETRY_DEFINITION = () => ({
+  offset: {
+    default: 8
+  },
+  position: {
+    default: {
+      type: 'Float32',
+      component: 3
+    }
+  },
+  normal: {
+    default: {
+      type: 'Float32',
+      component: 3
+    }
+  },
+  uv0: {
+    path: 'hasTexture',
+    transform: (val) => (val && {type: 'Float32', component: 2}) || false,
+    omitValues: [false]
+  },
+  color: {
+    default: {
+      type: 'UInt8',
+      component: 4
+    }
+  },
+  uvRegion: {
+    path: 'hasUvRegions',
+    transform: (val) => (val && {type: 'UInt16', component: 4}) || false,
+    omitValues: [false]
+  },
+  featureId: {
+    default: {
+      binding: 'per-feature',
+      type: 'UInt64',
+      component: 1
+    }
+  },
+  faceRange: {
+    default: {
+      binding: 'per-feature',
+      type: 'UInt32',
+      component: 2
+    }
+  }
+});
+
+const COMPRESSED_GEOMETRY_DEFINITION = () => ({
+  'compressedAttributes.encoding': {
+    default: 'draco'
+  },
+  'compressedAttributes.attributes': {
+    path: 'geometryConfig',
+    transform: (val) => {
+      const result = ['position', 'normal'];
+      if (val.hasTexture) {
+        result.push('uv0');
+      }
+      result.push('color');
+      if (val.hasUvRegions) {
+        result.push('uv-region');
+      }
+      result.push('feature-index');
+      return result;
+    }
+  }
+});
+
+export const GEOMETRY_DEFINITION = () => ({
+  geometryBuffers: {
+    path: 'geometryConfig',
+    transform: (val) => {
+      const result = [transform(val, PLAIN_GEOMETRY_DEFINITION())];
+      if (val.draco) {
+        result.push(transform({geometryConfig: val}, COMPRESSED_GEOMETRY_DEFINITION()));
+      }
+      return result;
+    }
+  }
+});

package/src/i3s-converter/json-templates/layers.ts

@@ -1,76 +1,6 @@
 import transform from 'json-map-transform';
 import {STORE} from './store';
 
-const PLAIN_GEOMETRY_DEFINITION = () => ({
-  offset: 8,
-  position: {
-    type: 'Float32',
-    component: 3
-  },
-  normal: {
-    type: 'Float32',
-    component: 3
-  },
-  uv0: {
-    type: 'Float32',
-    component: 2
-  },
-  color: {
-    type: 'UInt8',
-    component: 4
-  },
-  featureId: {
-    binding: 'per-feature',
-    type: 'UInt64',
-    component: 1
-  },
-  faceRange: {
-    binding: 'per-feature',
-    type: 'UInt32',
-    component: 2
-  }
-});
-
-const PLAIN_GEOMETRY_DEFINITION_WITHOUT_UV0 = () => ({
-  offset: 8,
-  position: {
-    type: 'Float32',
-    component: 3
-  },
-  normal: {
-    type: 'Float32',
-    component: 3
-  },
-  color: {
-    type: 'UInt8',
-    component: 4
-  },
-  featureId: {
-    binding: 'per-feature',
-    type: 'UInt64',
-    component: 1
-  },
-  faceRange: {
-    binding: 'per-feature',
-    type: 'UInt32',
-    component: 2
-  }
-});
-
-const COMPRESSED_GEOMETRY_DEFINITION = () => ({
-  compressedAttributes: {
-    encoding: 'draco',
-    attributes: ['position', 'normal', 'uv0', 'color', 'feature-index']
-  }
-});
-
-const COMPRESSED_GEOMETRY_DEFINITION_WITHOUT_UV0 = () => ({
-  compressedAttributes: {
-    encoding: 'draco',
-    attributes: ['position', 'normal', 'color', 'feature-index']
-  }
-});
-
 const SPATIAL_REFERENCE = () => ({
   wkid: {
     path: 'wkid',
@@ -115,6 +45,27 @@ const NODE_PAGES = () => ({
   }
 });
 
+const FULL_EXTENT = () => ({
+  xmin: {
+    path: 'xmin'
+  },
+  ymin: {
+    path: 'ymin'
+  },
+  xmax: {
+    path: 'xmax'
+  },
+  ymax: {
+    path: 'ymax'
+  },
+  zmin: {
+    path: 'zmin'
+  },
+  zmax: {
+    path: 'zmax'
+  }
+});
+
 export const LAYERS = () => ({
   version: {
     path: 'version',
@@ -147,6 +98,10 @@ export const LAYERS = () => ({
     path: 'store',
     transform: (val) => transform(val, STORE)
   },
+  fullExtent: {
+    path: 'fullExtent',
+    transform: (val) => transform(val, FULL_EXTENT())
+  },
   heightModelInfo: {
     path: 'heightModelInfo',
     transform: (val) => transform(val, HEIGHT_MODEL_INFO())
@@ -164,27 +119,8 @@ export const LAYERS = () => ({
     default: []
   },
   geometryDefinitions: {
-    path: 'compressGeometry',
-    transform: (val) => {
-      const result = [{geometryBuffers: []}, {geometryBuffers: []}];
-
-      // @ts-expect-error
-      result[0].geometryBuffers.push(PLAIN_GEOMETRY_DEFINITION());
-      // @ts-expect-error
-      result[1].geometryBuffers.push(PLAIN_GEOMETRY_DEFINITION_WITHOUT_UV0());
-      if (val) {
-        // @ts-expect-error
-        result[0].geometryBuffers.push(COMPRESSED_GEOMETRY_DEFINITION());
-        // @ts-expect-error
-        result[1].geometryBuffers.push(COMPRESSED_GEOMETRY_DEFINITION_WITHOUT_UV0());
-      }
-      return result;
-    },
-    default: [
-      {
-        geometryBuffers: [PLAIN_GEOMETRY_DEFINITION(), PLAIN_GEOMETRY_DEFINITION_WITHOUT_UV0()]
-      }
-    ]
+    path: 'geometryDefinitions',
+    default: []
   },
   attributeStorageInfo: {
     path: 'attributeStorageInfo',

package/src/i3s-converter/json-templates/shared-resources.ts

@@ -15,15 +15,15 @@ const MATERIAL_DEFINITION_INFO_PARAMS = () => ({
   },
   ambient: {
     path: 'ambient',
-    default: [1, 1, 1, 1]
+    default: [1, 1, 1]
   },
   diffuse: {
     path: 'diffuse',
-    default: [1, 1, 1, 1]
+    default: [1, 1, 1]
   },
   specular: {
     path: 'specular',
-    default: [0, 0, 0, 0]
+    default: [0, 0, 0]
   },
   useVertexColorAlpha: {
     path: 'useVertexColorAlpha',

package/src/i3s-converter/types.ts

@@ -1,14 +1,165 @@
-import {BoundingVolumes, I3SMaterialDefinition, SharedResources} from '@loaders.gl/i3s';
+import {GLTFImagePostprocessed} from '@loaders.gl/gltf';
+import {
+  BoundingVolumes,
+  I3SMaterialDefinition,
+  MaterialDefinitionInfo,
+  TextureDefinitionInfo
+} from '@loaders.gl/i3s';
 
+/** Converted resources for specific node */
 export type I3SConvertedResources = {
+  /**
+   * Node id
+   */
+  nodeId?: number;
+  /** Non-compressed geometry buffer that have structure met
+   * https://github.com/Esri/i3s-spec/blob/master/docs/1.8/defaultGeometrySchema.cmn.md
+   * (Geometry buffer)
+   */
   geometry: ArrayBuffer | null;
-  compressedGeometry?: ArrayBuffer | null;
+  /**
+   * Draco compressed geometry
+   */
+  compressedGeometry?: Promise<ArrayBuffer> | null;
+  /**
+   * Texture image content
+   */
   texture: any | null;
-  sharedResources: SharedResources | null;
+  /**
+   * If the resource has uvRegions geometry attribute
+   */
+  hasUvRegions: boolean;
+  /**
+   * Shared resources built from GLTF material
+   */
+  sharedResources: SharedResourcesArrays | null;
+  /**
+   * Material definition of the node
+   */
   meshMaterial?: I3SMaterialDefinition | null;
+  /**
+   * Number of vertices in the node
+   */
   vertexCount: number | null;
-  attributes: any | null;
+  /**
+   * Feature attributes contents
+   */
+  attributes: ArrayBuffer[] | null;
+  /**
+   * Number of features in the node
+   */
   featureCount: number | null;
-  geometryBuffer?: ArrayBuffer;
+  /**
+   * MBS and/or OBB bounding volumes of the node
+   */
   boundingVolumes: BoundingVolumes | null;
 };
+
+/**
+ * Geometry and feature attributes converted from GLTF primitives
+ */
+export type ConvertedAttributes = {
+  /** POSITION attribute value */
+  positions: Float32Array;
+  /** NORMAL attribute value */
+  normals: Float32Array;
+  /** TEXCOORD_0 attribute value */
+  texCoords: Float32Array;
+  /** COLOR_0 attribute value */
+  colors: Uint8Array;
+  /** uvRegion attribute for a texture atlas */
+  uvRegions: Uint16Array;
+  /** Feature indices grouped by ...
+   * converted from "batch ids" of GLTF
+   */
+  featureIndicesGroups?: number[][];
+  /** Feature indices converted from "batch ids" */
+  featureIndices: number[];
+  /**
+   * MBS and/or OBB bounding volumes of the node
+   */
+  boundingVolumes: null | BoundingVolumes;
+  /** merged materials data */
+  mergedMaterials: MergedMaterial[];
+};
+
+/** Postprocessed geometry and feature attributes
+ * https://github.com/Esri/i3s-spec/blob/master/docs/1.8/defaultGeometrySchema.cmn.md
+ */
+export type GeometryAttributes = {
+  /** POSITION attribute value */
+  positions: Float32Array;
+  /** NORMAL attribute value */
+  normals: Float32Array;
+  /** TEXCOORD_0 attribute value */
+  texCoords: Float32Array;
+  /** COLOR_0 attribute value */
+  colors: Uint8Array;
+  /** uvRegion attribute for a texture atlas */
+  uvRegions: Uint16Array;
+  /** faceRanges attribute value  */
+  faceRange: Uint32Array;
+  /** feature Ids attribute value */
+  featureIds: number[];
+  /** number of features in the node */
+  featureCount: number;
+};
+
+/** Geometry attributes specific for the particular feature */
+export type GroupedByFeatureIdAttributes = {
+  /** Feature Id */
+  featureId: number;
+  /** POSITION attribute value */
+  positions: Float32Array;
+  /** NORMAL attribute value */
+  normals: Float32Array;
+  /** COLOR_0 attribute value */
+  colors: Uint8Array;
+  /** uvRegion attribute for a texture atlas */
+  uvRegions: Uint16Array;
+  /** TEXCOORD_0 attribute value */
+  texCoords: Float32Array;
+};
+
+/** Shared resources made from GLTF material */
+export type SharedResourcesArrays = {
+  /** material definitions list https://github.com/Esri/i3s-spec/blob/master/docs/1.8/materialDefinitionInfo.cmn.md */
+  materialDefinitionInfos?: MaterialDefinitionInfo[];
+  /** texture definitions list  https://github.com/Esri/i3s-spec/blob/master/docs/1.8/textureDefinitionInfo.cmn.md*/
+  textureDefinitionInfos?: TextureDefinitionInfo[];
+  /** node id to make unique SharedResource ids */
+  nodePath?: string;
+};
+
+/** I3S material definition and texture content taken from GLTF material */
+export type I3SMaterialWithTexture = {
+  /** Material definition https://github.com/Esri/i3s-spec/blob/master/docs/1.8/materialDefinitions.cmn.md */
+  material: I3SMaterialDefinition;
+  /** Texture content (image) */
+  texture?: GLTFImagePostprocessed;
+  /** Metadata of all merged materials */
+  mergedMaterials: MergedMaterial[];
+};
+
+/** Metadata of some original texture */
+export type MergedMaterial = {
+  /** Gltf material Id */
+  originalMaterialId: string;
+  /** Original texture size */
+  textureSize?: {
+    width: number;
+    height: number;
+  };
+  /** Uint16Array of 4 elements https://github.com/Esri/i3s-spec/blob/master/docs/1.7/geometryUVRegion.cmn.md */
+  uvRegion?: Uint16Array;
+};
+
+export type TypedArrayConstructor =
+  | Uint8ArrayConstructor
+  | Int8ArrayConstructor
+  | Uint16ArrayConstructor
+  | Int16ArrayConstructor
+  | Int32ArrayConstructor
+  | Uint32ArrayConstructor
+  | Float32ArrayConstructor
+  | Float64ArrayConstructor;

package/src/index.ts

@@ -1,6 +1,2 @@
 export {default as I3SConverter} from './i3s-converter/i3s-converter';
-export {default as NodePages} from './i3s-converter/helpers/node-pages';
-
 export {default as Tiles3DConverter} from './3d-tiles-converter/3d-tiles-converter';
-
-export {DepsInstaller} from './deps-installer/deps-installer';

package/src/lib/utils/{compress-util.js → compress-util.ts}

@@ -9,7 +9,13 @@ import {MD5Hash} from '@loaders.gl/crypto';
 import crypt from 'crypt';
 import {getAbsoluteFilePath} from './file-utils';
 
-export function compressFileWithGzip(pathFile) {
+/**
+ * Compress file to gzip file
+ *
+ * @param pathFile - the path to the file
+ * @return the path to the gzip file
+ */
+export function compressFileWithGzip(pathFile: string): Promise<string> {
   const compressedPathFile = `${pathFile}.gz`;
   const gzip = createGzip();
   const input = createReadStream(pathFile);
@@ -28,7 +34,18 @@ export function compressFileWithGzip(pathFile) {
   });
 }
 
-export async function compressFilesWithZip(fileMap, outputFile, level = 0) {
+/**
+ * Compress files from map into slpk file
+ *
+ * @param fileMap - map with file paths (key: output path, value: input path)
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ */
+export async function compressFilesWithZip(
+  fileMap: {[key: string]: string},
+  outputFile: string,
+  level: number = 0
+) {
   // Before creating a new file, we need to delete the old file
   try {
     await removeFile(outputFile);
@@ -82,16 +99,44 @@ export async function compressFilesWithZip(fileMap, outputFile, level = 0) {
   });
 }
 
-export async function compressWithChildProcess() {
+/**
+ * Compress files using external tool 'zip'/'7z'
+ *
+ * @param inputFolder - folder to archive - for cwd option
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ * @param inputFiles - input files path to pass to the executable as option
+ * @param sevenZipExe - path to 7z.exe executable
+ */
+export async function compressWithChildProcess(
+  inputFolder: string,
+  outputFile: string,
+  level: number,
+  inputFiles: string,
+  sevenZipExe: string
+) {
   // eslint-disable-next-line no-undef
   if (process.platform === 'win32') {
-    await compressWithChildProcessWindows(...arguments);
+    await compressWithChildProcessWindows(inputFolder, outputFile, level, inputFiles, sevenZipExe);
   } else {
-    await compressWithChildProcessUnix(...arguments);
+    await compressWithChildProcessUnix(inputFolder, outputFile, level, inputFiles);
   }
 }
 
-async function compressWithChildProcessUnix(inputFolder, outputFile, level = 0, inputFiles = '.') {
+/**
+ * Compress files using external linux tool 'zip'
+ *
+ * @param inputFolder - folder to archive - for cwd option
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ * @param inputFiles - input files path to pass to the executable as option
+ */
+async function compressWithChildProcessUnix(
+  inputFolder: string,
+  outputFile: string,
+  level: number = 0,
+  inputFiles: string = '.'
+) {
   const fullOutputFile = getAbsoluteFilePath(outputFile);
   const args = [`-${level}`, '-r', fullOutputFile, inputFiles];
   const childProcess = new ChildProcessProxy();
@@ -105,12 +150,21 @@ async function compressWithChildProcessUnix(inputFolder, outputFile, level = 0,
   });
 }
 
+/**
+ * Compress files using windows external tool '7z'
+ *
+ * @param inputFolder - folder to archive - for cwd option
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ * @param inputFiles - input files path to pass to the executable as option
+ * @param sevenZipExe - path to 7z.exe executable
+ */
 async function compressWithChildProcessWindows(
-  inputFolder,
-  outputFile,
-  level = 0,
-  inputFiles = join('.', '*'),
-  sevenZipExe
+  inputFolder: string,
+  outputFile: string,
+  level: number = 0,
+  inputFiles: string = join('.', '*'),
+  sevenZipExe: string
 ) {
   // Workaround for @listfile issue. In 7z.exe @-leading files are handled as listfiles
   // https://sevenzip.osdn.jp/chm/cmdline/syntax.htm
@@ -131,10 +185,17 @@ async function compressWithChildProcessWindows(
   });
 }
 
-export async function generateHash128FromZip(inputZipFile, outputFile) {
+/**
+ * Generate hash file from zip archive
+ * https://github.com/Esri/i3s-spec/blob/master/docs/1.7/slpk_hashtable.cmn.md
+ *
+ * @param inputZipFile
+ * @param outputFile
+ */
+export async function generateHash128FromZip(inputZipFile: string, outputFile: string) {
   const input = await fs.readFile(inputZipFile);
   const zip = await JSZip.loadAsync(input);
-  const hashTable = [];
+  const hashTable: {key: string; value: string}[] = [];
   const zipFiles = zip.files;
   for (const relativePath in zipFiles) {
     const zipEntry = zipFiles[relativePath];
@@ -169,7 +230,7 @@ export async function generateHash128FromZip(inputZipFile, outputFile) {
     });
     for (const key in hashTable) {
       const item = hashTable[key];
-      const value = _longToByteArray(item.value);
+      const value = longToByteArray(item.value);
       // TODO: perhaps you need to wait for the 'drain' event if the write returns 'false'
       // eslint-disable-next-line no-undef
       output.write(Buffer.from(crypt.hexToBytes(item.key).concat(value)));
@@ -178,19 +239,45 @@ export async function generateHash128FromZip(inputZipFile, outputFile) {
   });
 }
 
-function _longToByteArray(long) {
+/**
+ * Encode 64 bit value to byte array
+ *
+ * @param long - stringified number
+ * @returns
+ */
+function longToByteArray(long: string): number[] {
   const buffer = new ArrayBuffer(8); // JS numbers are 8 bytes long, or 64 bits
   const longNum = new Float64Array(buffer); // so equivalent to Float64
-  longNum[0] = long;
+  longNum[0] = parseInt(long);
   return Array.from(new Uint8Array(buffer)).reverse(); // reverse to get little endian
 }
 
-export async function addFileToZip(inputFolder, fileName, zipFile, sevenZipExe) {
+/**
+ * Add file to zip archive
+ *
+ * @param inputFile
+ * @param fileName
+ * @param zipFile
+ * @param sevenZipExe
+ */
+export async function addFileToZip(
+  inputFolder: string,
+  fileName: string,
+  zipFile: string,
+  sevenZipExe: string
+) {
   await compressWithChildProcess(inputFolder, zipFile, 0, fileName, sevenZipExe);
   console.log(`${fileName} added to ${zipFile}.`); // eslint-disable-line
 }
 
-function appendFileToArchive(archive, subFileName, subFileData) {
+/**
+ *
+ * @param archive zip archive instance
+ * @param subFileName file path inside archive
+ * @param subFileData source file path
+ * @returns
+ */
+function appendFileToArchive(archive: any, subFileName: string, subFileData: string) {
   return new Promise((resolve) => {
     const fileStream = createReadStream(subFileData);
     console.log(`Compression start: ${subFileName}`); // eslint-disable-line no-undef,no-console