@loaders.gl/tile-converter 4.0.0-alpha.4 → 4.0.0-alpha.6

package/src/i3s-converter/helpers/gltf-attributes.ts

@@ -0,0 +1,103 @@
+import type {B3DMContent} from '@loaders.gl/3d-tiles';
+import type {GLTFAccessorPostprocessed, GLTFNodePostprocessed} from '@loaders.gl/gltf';
+import type {B3DMAttributesData} from '../../i3s-attributes-worker';
+
+type AttributesObject = {
+  [k: string]: GLTFAccessorPostprocessed;
+};
+
+/**
+ * Keep only values for B3DM attributes to pass data to worker thread.
+ * @param attributes
+ */
+function getB3DMAttributesWithoutBufferView(attributes: AttributesObject): AttributesObject {
+  const attributesWithoutBufferView = {};
+
+  for (const attributeName in attributes) {
+    attributesWithoutBufferView[attributeName] = {
+      value: attributes[attributeName].value
+    };
+  }
+
+  return attributesWithoutBufferView;
+}
+
+/**
+ * Prepare attributes for conversion to avoid binary data breaking in worker thread.
+ * @param tileContent
+ * @returns
+ */
+export function prepareDataForAttributesConversion(tileContent: B3DMContent): B3DMAttributesData {
+  let nodes =
+    tileContent.gltf?.scene?.nodes ||
+    tileContent.gltf?.scenes?.[0]?.nodes ||
+    tileContent.gltf?.nodes ||
+    [];
+
+  const images =
+    tileContent.gltf?.images?.map((imageObject) => {
+      // Need data only for uncompressed images because we can't get batchIds from compressed textures.
+      if (imageObject?.image?.compressed) {
+        return {
+          data: null,
+          compressed: true
+        };
+      } else {
+        const data = imageObject?.image?.data;
+        const dataCopy = new Uint8Array(data.length);
+        dataCopy.set(data);
+        return {
+          data: dataCopy,
+          compressed: false,
+          height: imageObject.image.height,
+          width: imageObject.image.width,
+          components: imageObject.image.components,
+          mimeType: imageObject.mimeType
+        };
+      }
+    }) || [];
+
+  prepareNodes(nodes);
+
+  const cartographicOrigin = tileContent.cartographicOrigin;
+  const cartesianModelMatrix = tileContent.cartesianModelMatrix;
+
+  return {
+    nodes,
+    images,
+    cartographicOrigin,
+    cartesianModelMatrix
+  };
+}
+
+/**
+ * Traverse all nodes to replace all sensible data with copy to avoid data corruption in worker.
+ * @param nodes
+ */
+function prepareNodes(nodes: GLTFNodePostprocessed[]): void {
+  for (let index = 0; index < nodes.length; index++) {
+    const node = nodes[index] as any;
+
+    if (node.mesh) {
+      nodes[index] = {
+        ...node,
+        mesh: {
+          ...node.mesh,
+          primitives: node.mesh?.primitives.map((primitive) => ({
+            ...primitive,
+            indices: {value: primitive?.indices?.value},
+            attributes: getB3DMAttributesWithoutBufferView(primitive.attributes),
+            material: {
+              id: primitive?.material?.id,
+              uniqueId: primitive?.material?.uniqueId
+            }
+          }))
+        }
+      };
+    }
+
+    if (node.children) {
+      prepareNodes(node.children);
+    }
+  }
+}

package/src/i3s-converter/helpers/node-debug.ts

@@ -0,0 +1,146 @@
+import type {Mbs, Node3DIndexDocument, Obb} from '@loaders.gl/i3s';
+import {OrientedBoundingBox, BoundingSphere} from '@math.gl/culling';
+import {Vector3} from '@math.gl/core';
+import {Ellipsoid} from '@math.gl/geospatial';
+
+// prettier-ignore
+const CUBE_POSITIONS = new Float32Array([
+  -1,  -1,  1, 1,  -1,  1,  1,  1,  1,  -1,  1,  1,
+  -1,  -1,  -1,  -1,  1,  -1,  1,  1,  -1,  1,  -1,  -1,
+  -1,  1,  -1,  -1,  1,  1,  1,  1,  1,  1,  1,  -1,
+  -1,  -1,  -1,  1,  -1,  -1,  1,  -1,  1,  -1,  -1,  1,
+  1,  -1,  -1,  1,  1,  -1,  1,  1,  1,  1,  -1,  1,
+  -1,  -1,  -1,  -1,  -1,  1,  -1,  1,  1,  -1,  1,  -1
+]);
+
+// TODO Unite Tile validation logic in i3s-17-and-debug with this code.
+
+/**
+ * Do validation of bounding volumes for particular node.
+ * Generates special warnings if there are some issues.
+ * @param node
+ */
+export function validateNodeBoundingVolumes(node: Node3DIndexDocument): string[] {
+  if (!node?.parentNode?.obb || !node?.parentNode?.mbs) {
+    return [];
+  }
+
+  const tileWarnings: string[] = [];
+
+  validateObb(tileWarnings, node);
+  validateMbs(tileWarnings, node);
+
+  return tileWarnings;
+}
+
+/**
+ * Check if child Obb fit into parent Obb.
+ * @param tileWarnings
+ * @param node
+ */
+function validateObb(tileWarnings: string[], node: Node3DIndexDocument): void {
+  // @ts-expect-error
+  const parentObb = createBoundingBoxFromTileObb(node.parentNode.obb);
+  const tileVertices = getTileObbVertices(node);
+  const isTileObbInsideParentObb = isAllVerticesInsideBoundingVolume(parentObb, tileVertices);
+
+  if (isTileObbInsideParentObb) {
+    return;
+  }
+
+  const title = `OBB of Tile (${node.id}) doesn't fit into Parent (${node.parentNode?.id}) tile OBB`;
+  tileWarnings.push(title);
+}
+
+/**
+ * Check if child Mbs fit into parent Mbs.
+ * @param tileWarnings
+ * @param node
+ */
+function validateMbs(tileWarnings: string[], node: Node3DIndexDocument): void {
+  // @ts-expect-error
+  const tileMbs = createBoundingSphereFromTileMbs(node.mbs);
+  // @ts-expect-error
+  const parentMbs = createBoundingSphereFromTileMbs(node.parentNode.mbs);
+  const distanceBetweenCenters = tileMbs.center.distanceTo(parentMbs.center);
+
+  if (distanceBetweenCenters + tileMbs.radius > parentMbs.radius) {
+    const title = `MBS of Tile (${node.id}) doesn't fit into Parent (${node.parentNode?.id}) tile MBS`;
+    tileWarnings.push(title);
+  }
+}
+
+/**
+ * Generates bounding sphere from mbs
+ * @param mbs
+ */
+function createBoundingSphereFromTileMbs(mbs: Mbs): BoundingSphere {
+  return new BoundingSphere([mbs[0], mbs[1], mbs[2]], mbs[3]);
+}
+
+/**
+ * Generates oriented bounding box from tile obb
+ * @param obb
+ * @returns
+ */
+function createBoundingBoxFromTileObb(obb: Obb): OrientedBoundingBox {
+  const {center, halfSize, quaternion} = obb;
+  return new OrientedBoundingBox().fromCenterHalfSizeQuaternion(center, halfSize, quaternion);
+}
+
+/**
+ * Get vertices fromnode obb
+ * TODO check if Obb generates properly
+ * @param node
+ */
+function getTileObbVertices(node: Node3DIndexDocument): number[] {
+  // @ts-expect-error
+  const halfSize = node.obb.halfSize;
+  const positions = CUBE_POSITIONS;
+  // @ts-expect-error
+  const obbCenterCartesian = Ellipsoid.WGS84.cartographicToCartesian(node.obb.center);
+
+  let vertices = [];
+
+  for (let i = 0; i < positions.length; i += 3) {
+    const positionsVector = new Vector3(
+      (positions[i] *= halfSize[0]),
+      (positions[i + 1] *= halfSize[1]),
+      (positions[i + 2] *= halfSize[2])
+    );
+    const rotatedPositions = positionsVector
+      // @ts-expect-error
+      .transformByQuaternion(node.obb.quaternion)
+      .add(obbCenterCartesian);
+    // @ts-expect-error
+    vertices = vertices.concat(rotatedPositions);
+  }
+
+  return vertices;
+}
+
+/**
+ * Check if all vertices inside bounding volume
+ * @param boundingVolume
+ * @param positions
+ */
+function isAllVerticesInsideBoundingVolume(
+  boundingVolume: OrientedBoundingBox,
+  positions: number[]
+): boolean {
+  let isVerticesInsideObb = true;
+
+  for (let index = 0; index < positions.length / 3; index += 3) {
+    const point = [positions[index], positions[index + 1], positions[index + 2]];
+    const cartographicPoint = Ellipsoid.WGS84.cartesianToCartographic(point);
+
+    const distance = boundingVolume.distanceTo(cartographicPoint);
+
+    if (distance > 0) {
+      isVerticesInsideObb = false;
+      break;
+    }
+  }
+
+  return isVerticesInsideObb;
+}

package/src/i3s-converter/helpers/node-index-document.ts

@@ -0,0 +1,315 @@
+import {join} from 'path';
+import {
+  BoundingVolumes,
+  LodSelection,
+  Node3DIndexDocument,
+  NodeInPage,
+  NodeReference
+} from '@loaders.gl/i3s';
+import transform from 'json-map-transform';
+import {v4 as uuidv4} from 'uuid';
+import {openJson, writeFile, writeFileForSlpk} from '../../lib/utils/file-utils';
+import I3SConverter from '../i3s-converter';
+import {NODE as nodeTemplate} from '../json-templates/node';
+import {I3SConvertedResources} from '../types';
+
+/**
+ * Wrapper for https://github.com/Esri/i3s-spec/blob/master/docs/1.7/3DNodeIndexDocument.cmn.md data
+ * The class allows working with 3DNodeIndexDocument in 2 modes:
+ * in memory: the data is stored in `data` field
+ * on disk: the data is written on disk in a file. The file can be rewritten when new childrend or neighbors have to be added
+ */
+export class NodeIndexDocument {
+  /** Node id */
+  public id: string;
+  /** Id in node pages */
+  public inPageId: number;
+  /** 3DNodeIndexDocument data */
+  public data: Node3DIndexDocument | null = null;
+  /** children */
+  public children: NodeIndexDocument[] = [];
+  /** converter instance */
+  private converter: I3SConverter;
+
+  /**
+   * Constructor
+   * @param id - id of the node in node pages
+   * @param converter - converter instance
+   */
+  constructor(id: number, converter: I3SConverter) {
+    this.inPageId = id;
+    this.id = id === 0 ? 'root' : id.toString();
+    this.converter = converter;
+  }
+
+  /**
+   * Add Node3DIndexDocument data to the node
+   * @param data Node3DIndexDocument data
+   * @returns this NodeIndexDocument instance (to recurring with constructor)
+   */
+  public async addData(data: Node3DIndexDocument): Promise<NodeIndexDocument> {
+    if (this.converter.options.instantNodeWriting) {
+      await this.write(data);
+    } else {
+      this.data = data;
+    }
+    return this;
+  }
+
+  /**
+   * Add child node references
+   * @param childNodes - child NodeIndexDocument instances
+   */
+  public async addChildren(childNodes: NodeIndexDocument[]): Promise<void> {
+    const newChildren: NodeReference[] = [];
+    for (const node of childNodes) {
+      const nodeData = await node.load();
+      newChildren.push({
+        id: node.id,
+        href: `../${node.id}`,
+        obb: nodeData.obb,
+        mbs: nodeData.mbs
+      });
+    }
+    this.children = this.children.concat(childNodes);
+
+    let data: Node3DIndexDocument | null = this.data;
+    if (this.converter.options.instantNodeWriting) {
+      data = (await this.load()) as Node3DIndexDocument;
+    }
+    if (data) {
+      data.children = data.children ?? [];
+      data.children = data.children.concat(newChildren);
+    }
+    if (this.converter.options.instantNodeWriting && data) {
+      await this.write(data);
+    }
+  }
+
+  /**
+   * Add neighbors to child nodes of this node
+   */
+  public async addNeighbors(): Promise<void> {
+    const nodeData = await this.load();
+    for (const childNode of this.children) {
+      const childNodeData = await childNode.load();
+      childNodeData.neighbors = childNodeData.neighbors ?? [];
+
+      // Don't do large amount of "neightbors" to avoid big memory consumption
+      if (Number(nodeData?.children?.length) < 1000) {
+        for (const neighbor of nodeData.children || []) {
+          if (childNode.id === neighbor.id) {
+            continue; // eslint-disable-line
+          }
+
+          childNodeData.neighbors.push({...neighbor});
+        }
+      } else {
+        // eslint-disable-next-line no-console, no-undef
+        console.warn(
+          `Node ${childNode.id}: neighbors attribute is omited because of large number of neigbors`
+        );
+        delete childNodeData.neighbors;
+      }
+
+      if (this.converter.options.instantNodeWriting && childNodeData) {
+        await childNode.write(childNodeData);
+      }
+      await childNode.save();
+      // The save after adding neighbors is the last one. Flush the the node
+      childNode.flush();
+    }
+  }
+
+  /** Save 3DNodeIndexDocument in file on disk */
+  public async save(): Promise<void> {
+    if (this.data) {
+      await this.write(this.data);
+    }
+  }
+
+  /**
+   * Write 3DNodeIndexDocument https://github.com/Esri/i3s-spec/blob/master/docs/1.7/3DNodeIndexDocument.cmn.md
+   * @param node - Node3DIndexDocument object
+   */
+  private async write(node: Node3DIndexDocument): Promise<void> {
+    const path = join(this.converter.layers0Path, 'nodes', this.id);
+    if (this.converter.options.slpk) {
+      await this.converter.writeQueue.enqueue(
+        {
+          archiveKey: `nodes/${this.id}/3dNodeIndexDocument.json.gz`,
+          writePromise: () =>
+            writeFileForSlpk(
+              path,
+              JSON.stringify(node),
+              '3dNodeIndexDocument.json',
+              true,
+              this.converter.compressList
+            )
+        },
+        true
+      );
+    } else {
+      await this.converter.writeQueue.enqueue(
+        {writePromise: () => writeFile(path, JSON.stringify(node))},
+        true
+      );
+    }
+  }
+
+  /**
+   * Load 3DNodeIndexDocument data from file on disk
+   * @returns 3DNodeIndexDocument object
+   */
+  private async load(): Promise<Node3DIndexDocument> {
+    if (this.data) {
+      return this.data;
+    }
+    const path = this.id;
+    const parentNodePath = join(this.converter.layers0Path, 'nodes', path);
+    let parentNodeFileName = 'index.json';
+    if (this.converter.options.slpk) {
+      parentNodeFileName = '3dNodeIndexDocument.json';
+    }
+    return (await openJson(parentNodePath, parentNodeFileName)) as Node3DIndexDocument;
+  }
+
+  /**
+   * Unload the Node data
+   */
+  private flush(): void {
+    this.data = null;
+  }
+
+  /**
+   * Create root node of the tree
+   * @param boundingVolumes - MBS and OOB bounding volumes data
+   * @param converter - I3SConverter instance
+   * @returns instance of NodeIndexDocument
+   */
+  static async createRootNode(
+    boundingVolumes: BoundingVolumes,
+    converter: I3SConverter
+  ): Promise<NodeIndexDocument> {
+    const rootData = NodeIndexDocument.createRootNodeIndexDocument(boundingVolumes);
+    const rootNode = await new NodeIndexDocument(0, converter).addData(rootData);
+    return rootNode;
+  }
+
+  /**
+   * Create NodeIndexDocument instance
+   * @param parentNode - parent NodeIndexDocument
+   * @param boundingVolumes - MBS and OOB bounding volumes data
+   * @param lodSelection - LOD metrics data
+   * @param nodeInPage - node data in node pages
+   * @param resources - resources extracted from gltf/b3dm file
+   * @param converter - I3SConverter instance
+   * @returns NodeIndexDocument instance
+   */
+  static async createNode(
+    parentNode: NodeIndexDocument,
+    boundingVolumes: BoundingVolumes,
+    lodSelection: LodSelection[],
+    nodeInPage: NodeInPage,
+    resources: I3SConvertedResources,
+    converter: I3SConverter
+  ): Promise<NodeIndexDocument> {
+    const data = await NodeIndexDocument.createNodeIndexDocument(
+      parentNode,
+      boundingVolumes,
+      lodSelection,
+      nodeInPage,
+      resources
+    );
+    const node = await new NodeIndexDocument(nodeInPage.index, converter).addData(data);
+    return node;
+  }
+
+  /**
+   * Form 3DNodeIndexDocument data for the root node
+   * @param boundingVolumes - mbs and obb data about node's bounding volume
+   * @return 3DNodeIndexDocument data https://github.com/Esri/i3s-spec/blob/master/docs/1.7/3DNodeIndexDocument.cmn.md
+   */
+  static createRootNodeIndexDocument(boundingVolumes: BoundingVolumes): Node3DIndexDocument {
+    const root0data = {
+      version: `{${uuidv4().toUpperCase()}}`,
+      id: 'root',
+      level: 0,
+      lodSelection: [
+        {
+          metricType: 'maxScreenThresholdSQ',
+          maxError: 0
+        },
+        {
+          metricType: 'maxScreenThreshold',
+          maxError: 0
+        }
+      ],
+      ...boundingVolumes,
+      children: []
+    };
+    return transform(root0data, nodeTemplate());
+  }
+
+  /**
+   * Create a new Node3DIndexDocument
+   * @param parentNode - 3DNodeIndexDocument https://github.com/Esri/i3s-spec/blob/master/docs/1.7/3DNodeIndexDocument.cmn.md object of the parent node
+   * @param boundingVolumes - Bounding volumes
+   * @param lodSelection - Level of Details (LOD) metrics
+   * @param nodeInPage - corresponding node object in a node page
+   * @param resources - the node resources data
+   * @param resources.texture - texture image
+   * @param resources.attributes - feature attributes
+   * @return 3DNodeIndexDocument https://github.com/Esri/i3s-spec/blob/master/docs/1.7/3DNodeIndexDocument.cmn.md object
+   */
+  static async createNodeIndexDocument(
+    parentNode: NodeIndexDocument,
+    boundingVolumes: BoundingVolumes,
+    lodSelection: LodSelection[],
+    nodeInPage: NodeInPage,
+    resources: I3SConvertedResources
+  ): Promise<Node3DIndexDocument> {
+    const {texture, attributes} = resources;
+    const nodeId = nodeInPage.index!;
+    const parentNodeData = await parentNode.load();
+    const nodeData = {
+      version: parentNodeData.version,
+      id: nodeId.toString(),
+      level: parentNodeData.level! + 1,
+      ...boundingVolumes,
+      lodSelection,
+      parentNode: {
+        id: parentNode.id,
+        href: `../${parentNode.id}`,
+        mbs: parentNodeData.mbs,
+        obb: parentNodeData.obb
+      },
+      children: [],
+      neighbors: []
+    };
+    const node = transform(nodeData, nodeTemplate());
+
+    if (nodeInPage.mesh) {
+      node.geometryData = [{href: './geometries/0'}];
+      node.sharedResource = {href: './shared'};
+
+      if (texture) {
+        node.textureData = [{href: './textures/0'}, {href: './textures/1'}];
+      }
+
+      if (
+        attributes &&
+        attributes.length &&
+        parentNode.converter.layers0?.attributeStorageInfo?.length
+      ) {
+        node.attributeData = [];
+        for (let index = 0; index < attributes.length; index++) {
+          const folderName = parentNode.converter.layers0.attributeStorageInfo[index].key;
+          node.attributeData.push({href: `./attributes/${folderName}/0`});
+        }
+      }
+    }
+
+    return node;
+  }
+}