@loaders.gl/tile-converter 4.0.0-alpha.1 → 4.0.0-alpha.11

package/src/deps-installer/deps-installer.ts

@@ -0,0 +1,72 @@
+import {load, fetchFile} from '@loaders.gl/core';
+import {ZipLoader} from '@loaders.gl/zip';
+import {writeFile} from '../lib/utils/file-utils';
+import {join} from 'path';
+import {ChildProcessProxy} from '@loaders.gl/worker-utils';
+
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'beta';
+
+const PGM_LINK = 'https://raw.githubusercontent.com/visgl/deck.gl-data/master/egm/egm2008-5.zip';
+
+/**
+ * Install external dependencies for converter:
+ * * PGM file (implemented);
+ * * Draco library (not implemented);
+ * * 7z archiver (not implemented);
+ */
+export class DepsInstaller {
+  /**
+   * Run instalation
+   * @param path destination folder
+   * @param workersPath destination folder for workers.
+   *    This path is '' by default and is not used by tile-converter.
+   *    It is used in tests to prevent rewriting actual workers during tests running
+   */
+  async install(path: string = '', workersPath: string = ''): Promise<void> {
+    console.log('Installing "EGM2008-5" model...'); // eslint-disable-line no-console
+    const fileMap = await load(PGM_LINK, ZipLoader, {});
+
+    let depsPath = process.cwd();
+    if (path) {
+      depsPath = join(depsPath, path);
+    }
+
+    await writeFile(depsPath, new Uint8Array(fileMap['geoids/egm2008-5.pgm']), 'egm2008-5.pgm');
+
+    console.log('Installing "I3S Content Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('i3s', 'i3s-content-worker-node.js', workersPath);
+
+    console.log('Installing "Draco Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('draco', 'draco-worker-node.js', workersPath);
+
+    console.log('Installing "Basis Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('textures', 'basis-worker-node.js', workersPath);
+
+    console.log('Installing "join-images" npm package');
+    const childProcess = new ChildProcessProxy();
+    await childProcess.start({
+      command: process.platform === 'win32' ? 'npm.cmd' : 'npm',
+      // `npm install sharp join-images` works unstable. It fails because installed `sharp` version
+      // may be different from the version required by `join-images`. Pointing to specific versions
+      // resolve this issue
+      arguments: ['install', 'sharp@0.30.4', 'join-images@1.1.3'],
+      wait: 0,
+      ignoreStderr: true
+    });
+
+    console.log('All dependencies were installed succesfully.'); // eslint-disable-line no-console
+  }
+
+  private async installWorker(module: string, name: string, extraPath: string) {
+    const fileResponse = await fetchFile(
+      `https://unpkg.com/@loaders.gl/${module}@${VERSION}/dist/${name}`
+    );
+    const fileData = await fileResponse.arrayBuffer();
+    if (!fileData) {
+      return;
+    }
+    const path = join(process.cwd(), extraPath, 'modules', module, 'dist');
+    await writeFile(path, fileData, name);
+  }
+}

package/src/i3s-attributes-worker.ts

@@ -0,0 +1,59 @@
+import type {WorkerObject} from '@loaders.gl/worker-utils';
+import type {ConvertedAttributes} from './i3s-converter/types';
+import type {Matrix4, Vector3} from '@math.gl/core';
+import type {GLTFNodePostprocessed} from '@loaders.gl/gltf';
+
+import {processOnWorker} from '@loaders.gl/worker-utils';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type I3SAttributesWorkerOptions = {
+  _nodeWorkers: boolean;
+  reuseWorkers: boolean;
+  useCartesianPositions: boolean;
+  source: string;
+};
+
+export type TextureImageProperties = {
+  data: Uint8Array;
+  compressed?: boolean;
+  height?: number;
+  width?: number;
+  components?: number;
+  mimeType?: string;
+};
+
+export type B3DMAttributesData = {
+  gltfMaterials?: {id: string}[];
+  nodes: GLTFNodePostprocessed[];
+  images: (null | TextureImageProperties)[];
+  cartographicOrigin: Vector3;
+  cartesianModelMatrix: Matrix4;
+};
+
+/**
+ * I3S Attributes Worker to handle B3DM object
+ */
+export const I3SAttributesWorker = {
+  id: 'i3s-attributes',
+  name: 'I3S Attributes Worker',
+  module: 'tile-converter',
+  version: VERSION,
+  options: {
+    useCartesianPositions: false
+  }
+};
+
+/**
+ * Performs I3S attributes transformation
+ */
+export function transformI3SAttributesOnWorker(
+  attributesData: B3DMAttributesData,
+  options: I3SAttributesWorkerOptions
+): Promise<Map<string, ConvertedAttributes>> {
+  return processOnWorker(I3SAttributesWorker, attributesData, options);
+}
+
+export const _typecheckI3SAttributesWorker: WorkerObject = I3SAttributesWorker;

package/src/i3s-converter/helpers/batch-ids-extensions.ts

@@ -0,0 +1,202 @@
+import {GLTFAccessorPostprocessed, GLTFMeshPrimitivePostprocessed} from '@loaders.gl/gltf';
+import type {NumericArray} from '@loaders.gl/loader-utils';
+import type {
+  GLTF_EXT_feature_metadata_attribute,
+  GLTF_EXT_feature_metadata_primitive
+} from '@loaders.gl/gltf';
+import {TypedArray} from '@math.gl/core';
+import {TextureImageProperties} from '../../i3s-attributes-worker';
+
+const EXT_MESH_FEATURES = 'EXT_mesh_features';
+const EXT_FEATURE_METADATA = 'EXT_feature_metadata';
+
+/**
+ * Getting batchIds from 3DTilesNext extensions.
+ * @param attributes - gltf accessors
+ * @param primitive - gltf primitive data
+ * @param images - gltf texture images
+ */
+export function handleBatchIdsExtensions(
+  attributes: {
+    [key: string]: GLTFAccessorPostprocessed;
+  },
+  primitive: GLTFMeshPrimitivePostprocessed,
+  images: (TextureImageProperties | null)[]
+): NumericArray {
+  const extensions = primitive?.extensions;
+
+  if (!extensions) {
+    return [];
+  }
+
+  for (const [extensionName, extensionData] of Object.entries(extensions || {})) {
+    switch (extensionName) {
+      case EXT_FEATURE_METADATA:
+        return handleExtFeatureMetadataExtension(
+          attributes,
+          extensionData as GLTF_EXT_feature_metadata_primitive,
+          images
+        );
+      case EXT_MESH_FEATURES:
+        console.warn('EXT_mesh_features extension is not supported yet');
+        return [];
+      default:
+        return [];
+    }
+  }
+
+  return [];
+}
+
+/**
+ * Get batchIds from EXT_feature_metadata extension.
+ * Docs - https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_feature_metadata
+ * @param attributes
+ * @param extFeatureMetadata
+ * @param textures
+ */
+function handleExtFeatureMetadataExtension(
+  attributes: {
+    [key: string]: GLTFAccessorPostprocessed;
+  },
+  extFeatureMetadata: GLTF_EXT_feature_metadata_primitive,
+  images: (TextureImageProperties | null)[]
+): NumericArray {
+  // Take only first extension object to get batchIds attribute name.
+  const featureIdAttribute = extFeatureMetadata?.featureIdAttributes?.[0];
+
+  if (featureIdAttribute?.featureIds?.attribute) {
+    const batchIdsAttribute = attributes[featureIdAttribute.featureIds.attribute];
+    return batchIdsAttribute.value;
+  }
+
+  if (
+    featureIdAttribute?.featureIds?.hasOwnProperty('constant') &&
+    featureIdAttribute?.featureIds?.hasOwnProperty('divisor')
+  ) {
+    const featuresCount = attributes?.POSITIONS?.value.length / 3 || 0;
+    return generateImplicitFeatureIds(
+      featuresCount,
+      featureIdAttribute.featureIds.constant,
+      featureIdAttribute.featureIds.divisor
+    );
+  }
+
+  // Take only first extension object to get batchIds attribute name.
+  const featureIdTexture =
+    extFeatureMetadata?.featureIdTextures && extFeatureMetadata?.featureIdTextures[0];
+
+  if (featureIdTexture) {
+    const textureAttributeIndex = featureIdTexture?.featureIds?.texture?.texCoord || 0;
+    const textCoordAttribute = `TEXCOORD_${textureAttributeIndex}`;
+    const textureCoordinates = attributes[textCoordAttribute].value;
+    return generateBatchIdsFromTexture(featureIdTexture, textureCoordinates, images);
+  }
+
+  // Take only first extension texture to get batchIds from the root EXT_feature_metadata object.
+  const featureTexture =
+    extFeatureMetadata?.featureTextures && extFeatureMetadata?.featureTextures[0];
+
+  if (featureTexture) {
+    const batchIdsAttribute = attributes[featureTexture];
+    return batchIdsAttribute.value;
+  }
+
+  return [];
+}
+
+/**
+ * Generates implicit feature ids
+ * Spec - https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_feature_metadata#implicit-feature-ids
+ * @param featuresCount
+ * @param constant
+ * @param devisor
+ */
+function generateImplicitFeatureIds(
+  featuresCount: number,
+  constant: number = 0,
+  divisor: number = 0
+): number[] {
+  let featureIds: number[] = [];
+
+  if (divisor > 0) {
+    let currentValue = constant;
+    let devisorCounter = divisor;
+
+    for (let index = 0; index < featuresCount; index++) {
+      featureIds.push(currentValue);
+
+      devisorCounter -= 1;
+
+      if (devisorCounter === 0) {
+        currentValue++;
+        devisorCounter = divisor;
+      }
+    }
+  } else {
+    featureIds = Array<number>(featuresCount).fill(constant, 0, featuresCount);
+  }
+
+  return featureIds;
+}
+
+/**
+ * Get batchIds from texture.
+ * @param primitive
+ * @param featureIdTextures
+ */
+function generateBatchIdsFromTexture(
+  featureIdTexture: GLTF_EXT_feature_metadata_attribute,
+  textureCoordinates: TypedArray,
+  images: (TextureImageProperties | null)[]
+) {
+  if (!images?.length) {
+    return [];
+  }
+
+  const CHANNELS_MAP = {
+    r: 0,
+    g: 1,
+    b: 2,
+    a: 3
+  };
+
+  const textureIndex = featureIdTexture?.featureIds?.texture?.index;
+  const featureChannel = featureIdTexture?.featureIds?.channels;
+
+  if (!featureChannel || textureIndex === undefined) {
+    return [];
+  }
+
+  const image = images[textureIndex];
+  const batchIds: number[] = [];
+  const channels = CHANNELS_MAP[featureChannel];
+
+  if (image && image?.width && image?.height && image?.components) {
+    for (let index = 0; index < textureCoordinates.length; index += 2) {
+      const u = textureCoordinates[index];
+      const v = textureCoordinates[index + 1];
+
+      const tx = Math.min((emod(u) * image.width) | 0, image.width - 1);
+      const ty = Math.min((emod(v) * image.height) | 0, image.height - 1);
+
+      const offset = (ty * image.width + tx) * image.components + channels;
+      const batchId = new Uint8Array(image.data)[offset];
+
+      batchIds.push(batchId);
+    }
+  } else {
+    console.warn(`Can't get batch Ids from ${image?.mimeType || ''} compressed texture`);
+  }
+
+  return batchIds;
+}
+
+/**
+ * Handle UVs if they are out of range [0,1].
+ * @param n
+ * @param m
+ */
+function emod(n: number): number {
+  return ((n % 1) + 1) % 1;
+}

package/src/i3s-converter/helpers/coordinate-converter.ts

@@ -1,40 +1,42 @@
-import type {BoundingVolumes, Extent, Mbs, Obb} from '@loaders.gl/i3s';
+import type {BoundingVolumes, FullExtent, Mbs, Obb} from '@loaders.gl/i3s';
 
 import {Matrix3, Quaternion, Vector3} from '@math.gl/core';
 import {Ellipsoid} from '@math.gl/geospatial';
-import {OrientedBoundingBox} from '@math.gl/culling';
-import TileHeader from '@loaders.gl/tiles/src/tileset/tile-3d';
-import {GeoidHeightModel} from '../../lib/geoid-height-model';
-import {Tileset3D} from '@loaders.gl/tiles';
+import {
+  OrientedBoundingBox,
+  makeOrientedBoundingBoxFromPoints,
+  makeBoundingSphereFromPoints,
+  BoundingSphere
+} from '@math.gl/culling';
+import {Geoid} from '@math.gl/geoid';
 
 /**
  * Create bounding volumes object from tile and geoid height model.
- * @param tile
- * @param geoidHeightModel
+ * @param sourceBoundingVolume - initialized bounding volume of the source tile
+ * @param geoidHeightModel - instance of Geoid class that converts elevation from geoidal to ellipsoidal and back
  * @returns - Bounding volumes object
  */
 export function createBoundingVolumes(
-  tile: TileHeader,
-  geoidHeightModel: GeoidHeightModel
+  sourceBoundingVolume: OrientedBoundingBox | BoundingSphere,
+  geoidHeightModel: Geoid
 ): BoundingVolumes {
   let radius;
   let halfSize;
   let quaternion;
 
-  const boundingVolume = tile.boundingVolume;
   const cartographicCenter = Ellipsoid.WGS84.cartesianToCartographic(
-    boundingVolume.center,
+    sourceBoundingVolume.center,
     new Vector3()
   );
   cartographicCenter[2] =
     cartographicCenter[2] -
     geoidHeightModel.getHeight(cartographicCenter[1], cartographicCenter[0]);
-  if (boundingVolume instanceof OrientedBoundingBox) {
-    halfSize = boundingVolume.halfSize;
+  if (sourceBoundingVolume instanceof OrientedBoundingBox) {
+    halfSize = sourceBoundingVolume.halfSize;
     radius = new Vector3(halfSize[0], halfSize[1], halfSize[2]).len();
-    quaternion = boundingVolume.quaternion;
+    quaternion = sourceBoundingVolume.quaternion;
   } else {
-    radius = tile.boundingVolume.radius;
+    radius = sourceBoundingVolume.radius;
     halfSize = [radius, radius, radius];
     quaternion = new Quaternion()
       .fromMatrix3(new Matrix3([halfSize[0], 0, 0, 0, halfSize[1], 0, 0, 0, halfSize[2]]))
@@ -52,36 +54,94 @@ export function createBoundingVolumes(
 }
 
 /**
- * Convert common coordinate to extent coordinate
- * @param tileset
- * @returns - Extent
+ * Generates bounding volumes from geometry positions
+ * @param cartesianPositions
+ * @param geoidHeightModel
  */
-export function convertCommonToI3SExtentCoordinate(tileset: Tileset3D): Extent {
-  const cartesianCenter = tileset.cartesianCenter;
-  const radius = tileset.lodMetricValue;
-  const rightTop = Ellipsoid.WGS84.cartesianToCartographic(
-    new Vector3(cartesianCenter[0] + radius, cartesianCenter[1] + radius, cartesianCenter[2]),
+export function createBoundingVolumesFromGeometry(
+  cartesianPositions: Float32Array,
+  geoidHeightModel: Geoid
+): {mbs: Mbs; obb: Obb} {
+  const positionVectors = convertPositionsToVectors(cartesianPositions);
+
+  const geometryObb = makeOrientedBoundingBoxFromPoints(positionVectors);
+  const geometryMbs = makeBoundingSphereFromPoints(positionVectors);
+
+  let mbsCenter = Ellipsoid.WGS84.cartesianToCartographic(geometryMbs.center, new Vector3());
+  let obbCenter = Ellipsoid.WGS84.cartesianToCartographic(geometryObb.center, new Vector3());
+
+  mbsCenter[2] = mbsCenter[2] - geoidHeightModel.getHeight(mbsCenter[1], mbsCenter[0]);
+  obbCenter[2] = obbCenter[2] - geoidHeightModel.getHeight(obbCenter[1], obbCenter[0]);
+
+  return {
+    mbs: [mbsCenter[0], mbsCenter[1], mbsCenter[2], geometryMbs.radius],
+    obb: {
+      center: obbCenter,
+      halfSize: geometryObb.halfSize,
+      quaternion: geometryObb.quaternion
+    }
+  };
+}
+
+/**
+ * Create array of posisitons where each vertex is vector
+ * @param {array} positions
+ * @returns {Vector3[]}
+ */
+export function convertPositionsToVectors(positions: Float32Array): Vector3[] {
+  const result: Vector3[] = [];
+
+  for (let i = 0; i < positions.length; i += 3) {
+    // TODO: (perf) new Vector3 is not optimal but required in `makeOrientedBoundingBoxFromPoints`.
+    // modify `makeOrientedBoundingBoxFromPoints` to use scratch vectors
+    const positionVector = new Vector3([positions[i], positions[i + 1], positions[i + 2]]);
+    result.push(positionVector);
+  }
+
+  return result;
+}
+
+/**
+ * Convert common coordinate to fullExtent https://github.com/Esri/i3s-spec/blob/master/docs/1.8/fullExtent.cmn.md
+ * @param boundingVolume
+ * @returns - fullExtent object
+ */
+export function convertBoundingVolumeToI3SFullExtent(
+  boundingVolume: OrientedBoundingBox | BoundingSphere
+): FullExtent {
+  let sphere: BoundingSphere;
+  if (boundingVolume instanceof BoundingSphere) {
+    sphere = boundingVolume;
+  } else {
+    sphere = boundingVolume.getBoundingSphere();
+  }
+  const center: Vector3 = sphere.center;
+  const radius: number = sphere.radius;
+  const vertexMax = Ellipsoid.WGS84.cartesianToCartographic(
+    new Vector3(center[0] + radius, center[1] + radius, center[2] + radius),
     new Vector3()
   );
-  const leftBottom = Ellipsoid.WGS84.cartesianToCartographic(
-    new Vector3(cartesianCenter[0] - radius, cartesianCenter[1] - radius, cartesianCenter[2]),
+  const vertexMin = Ellipsoid.WGS84.cartesianToCartographic(
+    new Vector3(center[0] - radius, center[1] - radius, center[2] - radius),
     new Vector3()
   );
-  const isFirstRight = rightTop[0] < leftBottom[0];
-  const isFirstTop = rightTop[1] < leftBottom[1];
 
-  return [
-    isFirstRight ? rightTop[0] : leftBottom[0],
-    isFirstTop ? rightTop[1] : leftBottom[1],
-    isFirstRight ? leftBottom[0] : rightTop[0],
-    isFirstTop ? leftBottom[1] : rightTop[1]
-  ];
+  // Converter sometimes returns min values that are bigger then max,
+  // so we should check and take bigger value from max and smaller for nim
+  return {
+    xmin: Math.min(vertexMin[0], vertexMax[0]),
+    xmax: Math.max(vertexMin[0], vertexMax[0]),
+    ymin: Math.min(vertexMin[1], vertexMax[1]),
+    ymax: Math.max(vertexMin[1], vertexMax[1]),
+    zmin: Math.min(vertexMin[2], vertexMax[2]),
+    zmax: Math.max(vertexMin[2], vertexMax[2])
+  };
 }
 
 /**
  * Creates oriented boundinb box from mbs.
  * @param mbs - Minimum Bounding Sphere
- * @returns - Oriented BOunding Box
+ * @returns - Oriented Bounding Box
  */
 export function createObbFromMbs(mbs: Mbs): Obb {
   const radius = mbs[3];

package/src/i3s-converter/helpers/create-scene-server-path.ts

@@ -0,0 +1,29 @@
+import type {SceneLayer3D} from '@loaders.gl/i3s';
+import {v4 as uuidv4} from 'uuid';
+import transform from 'json-map-transform';
+import {join} from 'path';
+
+import {SCENE_SERVER as sceneServerTemplate} from '../json-templates/scene-server';
+import {writeFile} from '../../lib/utils/file-utils';
+
+/**
+ * Form and save sceneServer meta data into a file
+ * @param layerName - layer name to display
+ * @param layers0 - layer object embedded into sceneServer meta data
+ * @param rootPath - root path of new converted tileset
+ */
+export async function createSceneServerPath(
+  layerName: string,
+  layers0: SceneLayer3D,
+  rootPath: string
+): Promise<void> {
+  const sceneServerData = {
+    serviceItemId: uuidv4().replace(/-/gi, ''),
+    layerName,
+    layers0
+  };
+
+  const sceneServer = transform(sceneServerData, sceneServerTemplate());
+  const nodePagePath = join(rootPath, 'SceneServer');
+  await writeFile(nodePagePath, JSON.stringify(sceneServer));
+}