@loaders.gl/tile-converter 4.0.0-alpha.1 → 4.0.0-alpha.11

package/src/i3s-converter/helpers/node-pages.ts

@@ -0,0 +1,344 @@
+import {join} from 'path';
+import transform from 'json-map-transform';
+import {METADATA as metadataTemplate} from '../json-templates/metadata';
+import {NodeInPage} from '@loaders.gl/i3s';
+import {isFileExists, openJson} from '../../lib/utils/file-utils';
+import I3SConverter from '../i3s-converter';
+
+/**
+ * class NodePages - wrapper of nodePages array
+ *
+ * @example
+ * import {writeFile} from './helpers/write-file';
+ *
+ * // create an instance of the class
+ * const nodePages = new NodePages(writeFile, HARDCODED_NODES_PER_PAGE);
+ * ...
+ * // push root node
+ * const parent = await nodePages.push({
+      lodThreshold: HARDCODED_MAX_SCREEN_THRESHOLD_SQ,
+      obb: coordinates.obb,
+      children: []
+    });
+ * ...
+ * // push node with parent relation
+ * const nodeInPage = {
+      lodThreshold: HARDCODED_MAX_SCREEN_THRESHOLD_SQ,
+      obb: coordinates.obb,
+      children: [],
+      mesh: {
+        geometry: {
+          definition: 0
+        }
+      }
+    };
+ * const node = await this.nodePages.push(nodeInPage, parent.index);
+ * ...
+ * // save all the nodePages in the end of pushing all the nodes
+ * await this.nodePages.save(layers0path);
+ */
+export default class NodePages {
+  readonly nodesPerPage: number;
+  nodesCounter: number;
+  writeFile: (...args) => Promise<null | string>;
+  converter: I3SConverter;
+  readonly nodePages: {nodes: NodeInPage[]}[];
+  readonly length: number = 0;
+
+  /**
+   * @constructs
+   * Create a nodePages instance.
+   * @param writeFileFunc - function to save one nodePage into a file
+   * @param nodesPerPage - length limit for one nodePage. An additional nodePage is created when this limit is met
+   */
+  constructor(writeFileFunc, nodesPerPage, converter: I3SConverter) {
+    this.nodesPerPage = nodesPerPage;
+    this.nodesCounter = 0;
+    // @ts-expect-error
+    this.nodePages = [{}];
+    this.nodePages[0].nodes = [];
+    this.writeFile = writeFileFunc;
+    this.converter = converter;
+    this.length = 0;
+  }
+
+  /**
+   * Setup function to save node pages
+   * @param func - function which should be used to save node pages
+   */
+  useWriteFunction(func: (...args) => Promise<null | string>): void {
+    this.writeFile = func;
+  }
+
+  /**
+   * Get file path and file name of the node page with the particular id
+   * @param nodePageId - node page id
+   * @returns file path and file name
+   */
+  private getNodePageFileName(nodePageId): {filePath: string; fileName: string} {
+    let filePath;
+    let fileName;
+    if (this.converter.options.slpk) {
+      filePath = join(this.converter.layers0Path, 'nodepages');
+      fileName = `${nodePageId.toString()}.json`;
+    } else {
+      filePath = join(this.converter.layers0Path, 'nodepages', nodePageId.toString());
+      fileName = 'index.json';
+    }
+    return {filePath, fileName};
+  }
+
+  /**
+   * Load node page from a file on the disk
+   * @param nodePageId - node page id
+   * @returns - node page data
+   */
+  private async loadNodePage(nodePageId: number): Promise<{nodes: NodeInPage[]}> {
+    const {filePath, fileName} = this.getNodePageFileName(nodePageId);
+    const fullName = join(filePath, fileName);
+    if (await isFileExists(fullName)) {
+      console.log(`load ${fullName}.`); // eslint-disable-line
+      return (await openJson(filePath, fileName)) as {nodes: NodeInPage[]};
+    } else {
+      return {nodes: []};
+    }
+  }
+
+  /**
+   * Get nodepage id by node id
+   * @param id node id
+   * @returns node page id
+   */
+  private getPageIndexByNodeId(id: number): number {
+    return Math.floor(id / this.nodesPerPage);
+  }
+
+  /**
+   * Get node page data by node id
+   * @param id node id
+   * @returns node page data
+   */
+  private async getPageByNodeId(id: number): Promise<{nodes: NodeInPage[]}> {
+    const pageIndex = this.getPageIndexByNodeId(id);
+    if (this.converter.options.instantNodeWriting) {
+      return await this.loadNodePage(pageIndex);
+    }
+    return this.nodePages[pageIndex];
+  }
+
+  /**
+   * Get the node by its end-to-end index
+   * @param id - end-to-end index of the node
+   * @return the node object
+   */
+  async getNodeById(id: number, nodePage?: {nodes: NodeInPage[]}): Promise<NodeInPage> {
+    const nodeIndex = id % this.nodesPerPage;
+    nodePage = nodePage || (await this.getPageByNodeId(id));
+    return nodePage.nodes[nodeIndex];
+  }
+
+  /**
+   * Add a child id into the parent node.children array
+   * @param parentId - end-to-end parent node index
+   * @param childId - end-to-end child node index
+   */
+  private async addChildRelation(parentId: number | undefined, childId: number): Promise<void> {
+    if (parentId === null || parentId === undefined) {
+      return;
+    }
+    const parentNode = await this.getNodeById(parentId);
+    parentNode.children?.push(childId);
+    await this.saveNode(parentNode);
+  }
+
+  /**
+   * Put new node in nodePages array
+   * @param node - node object
+   * @param parentId - index of parent node
+   * @return
+   */
+  async push(node: NodeInPage, parentId?: number): Promise<NodeInPage> {
+    node.index = this.nodesCounter++;
+    if (!this.converter.options.instantNodeWriting) {
+      let currentNodePage = this.nodePages[this.nodePages.length - 1];
+      if (currentNodePage.nodes.length === this.nodesPerPage) {
+        currentNodePage = {nodes: []};
+        this.nodePages.push(currentNodePage);
+      }
+      currentNodePage.nodes.push(node);
+    }
+    await this.addChildRelation(parentId, node.index);
+    NodePages.updateResourceInMesh(node);
+    await this.saveNode(node);
+    return node;
+  }
+
+  /**
+   * Save node to the file on the disk
+   * @param node - node data
+   */
+  async saveNode(node: NodeInPage): Promise<void> {
+    if (!this.converter.options.instantNodeWriting) {
+      return;
+    } else {
+      const nodePageIndex = this.getPageIndexByNodeId(node.index);
+      const nodePage = await this.getPageByNodeId(node.index);
+      const {filePath, fileName} = this.getNodePageFileName(nodePageIndex);
+      const nodeToUpdate = await this.getNodeById(node.index, nodePage);
+      if (nodeToUpdate) {
+        NodePages.updateAll(nodeToUpdate, node);
+      } else {
+        nodePage.nodes.push(node);
+      }
+      const nodePageStr = JSON.stringify(nodePage);
+      if (this.converter.options.slpk) {
+        await this.converter.writeQueue.enqueue(
+          {
+            archiveKey: `nodePages/${nodePageIndex.toString()}.json.gz`,
+            writePromise: () =>
+              this.writeFile(filePath, nodePageStr, fileName, true, this.converter.compressList)
+          },
+          true
+        );
+      } else {
+        await this.converter.writeQueue.enqueue(
+          {
+            writePromise: () => this.writeFile(filePath, nodePageStr)
+          },
+          true
+        );
+      }
+    }
+  }
+
+  /**
+   * Save metadata file (for slpk only)
+   */
+  async saveMetadata(): Promise<void> {
+    const metadata = transform({nodeCount: this.nodesCounter}, metadataTemplate());
+    const compress = false;
+    await this.converter.writeQueue.enqueue({
+      archiveKey: 'metadata.json',
+      writePromise: () =>
+        this.writeFile(
+          this.converter.layers0Path,
+          JSON.stringify(metadata),
+          'metadata.json',
+          compress
+        )
+    });
+  }
+
+  /**
+   * Save all the node pages
+   * Run this method when all nodes is pushed in nodePages
+   */
+  async save(): Promise<void> {
+    if (this.converter.options.instantNodeWriting) {
+      await this.saveMetadata();
+      return;
+    }
+    if (this.converter.options.slpk) {
+      for (const [index, nodePage] of this.nodePages.entries()) {
+        const nodePageStr = JSON.stringify(nodePage);
+        const slpkPath = join(this.converter.layers0Path, 'nodepages');
+        await this.converter.writeQueue.enqueue({
+          archiveKey: `nodePages/${index.toString()}.json.gz`,
+          writePromise: () => this.writeFile(slpkPath, nodePageStr, `${index.toString()}.json`)
+        });
+      }
+      await this.saveMetadata();
+    } else {
+      for (const [index, nodePage] of this.nodePages.entries()) {
+        const nodePageStr = JSON.stringify(nodePage);
+        const nodePagePath = join(this.converter.layers0Path, 'nodepages', index.toString());
+        await this.converter.writeQueue.enqueue({
+          writePromise: () => this.writeFile(nodePagePath, nodePageStr)
+        });
+      }
+    }
+  }
+
+  /**
+   * Update resource index in node.mesh object
+   * @param node - node object
+   */
+  static updateResourceInMesh(node: NodeInPage): void {
+    if (node.mesh && isFinite(node.index)) {
+      node.mesh.geometry.resource = node.index;
+    }
+  }
+
+  /**
+   * Update all fields in the node excluding id
+   * @param node - node object
+   * @param data - NodeInPage data to replace original data
+   */
+  static updateAll(node: NodeInPage, data: NodeInPage): NodeInPage {
+    Object.assign(node, data, {index: node.index});
+    NodePages.updateResourceInMesh(node);
+    return node;
+  }
+
+  /**
+   * Update material in node.mesh object by node id
+   * @param id - end-to-end index of the node
+   * @param materialId - id from scene layer materialDefinitions
+   */
+  static updateMaterialByNodeId(node: NodeInPage, materialId: number): void {
+    if (!node.mesh) {
+      return;
+    }
+    node.mesh.material = {
+      definition: materialId,
+      resource: node.index
+    };
+  }
+
+  /**
+   * Update vertexCount in node.mesh.geometry object by node id
+   * @param id - end-to-end index of the node
+   * @param vertexCount - vertex count for particular node
+   */
+  static updateVertexCountByNodeId(node: NodeInPage, vertexCount: number): void {
+    if (!node.mesh) {
+      return;
+    }
+    node.mesh.geometry.vertexCount = vertexCount;
+  }
+
+  /**
+   * Update resource in node.mesh.attribute object by node id
+   * @param node - node object
+   */
+  static updateNodeAttributeByNodeId(node: NodeInPage): void {
+    if (!node.mesh || !node.index) {
+      return;
+    }
+    node.mesh.attribute.resource = node.index;
+  }
+
+  /**
+   * Update featureCount in node.mesh.geometry object by node id
+   * @param node - node object
+   * @param featureCount - features count of the node
+   */
+  static updateFeatureCountByNodeId(node: NodeInPage, featureCount: number): void {
+    if (!node.mesh) {
+      return;
+    }
+    node.mesh.geometry.featureCount = featureCount;
+  }
+
+  /**
+   * Update texelCountHint in node.mesh.material object by node id
+   * @param node - node object
+   * @param texelCountHint - texelCountHint of particular node
+   */
+  static updateTexelCountHintByNodeId(node: NodeInPage, texelCountHint: number): void {
+    if (!node.mesh || !node.mesh.material) {
+      return;
+    }
+    node.mesh.material.texelCountHint = texelCountHint;
+  }
+}

package/src/i3s-converter/helpers/preprocess-3d-tiles.ts

@@ -0,0 +1,79 @@
+import {Tiles3DTileContent} from '@loaders.gl/3d-tiles';
+import {GltfPrimitiveModeString, PreprocessData} from '../types';
+import {GLTF, GLTFLoader} from '@loaders.gl/gltf';
+import {parse} from '@loaders.gl/core';
+
+/**
+ * glTF primitive modes
+ * @see https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#_mesh_primitive_mode
+ */
+export const GLTF_PRIMITIVE_MODES = [
+  GltfPrimitiveModeString.POINTS, // 0
+  GltfPrimitiveModeString.LINES, // 1
+  GltfPrimitiveModeString.LINE_LOOP, // 2
+  GltfPrimitiveModeString.LINE_STRIP, // 3
+  GltfPrimitiveModeString.TRIANGLES, // 4
+  GltfPrimitiveModeString.TRIANGLE_STRIP, // 5
+  GltfPrimitiveModeString.TRIANGLE_FAN // 6
+];
+
+/**
+ * Analyze tile content. This function is used during preprocess stage of
+ * conversion
+ * @param tileContent - 3DTiles tile content ArrayBuffer
+ * @returns
+ */
+export const analyzeTileContent = async (
+  tileContent: Tiles3DTileContent | null
+): Promise<PreprocessData> => {
+  const result: PreprocessData = {
+    meshTopologyTypes: new Set()
+  };
+  if (!tileContent?.gltfArrayBuffer) {
+    return result;
+  }
+
+  const gltfData = await parse(tileContent.gltfArrayBuffer, GLTFLoader, {
+    gltf: {normalize: false, loadBuffers: false, loadImages: false, decompressMeshes: false}
+  });
+  const gltf = gltfData.json;
+
+  if (!gltf) {
+    return result;
+  }
+  const meshTypes = getMeshTypesFromGltf(gltf);
+  result.meshTopologyTypes = meshTypes;
+  return result;
+};
+
+/**
+ * Get mesh topology types that the glb content has
+ * @param gltfJson - JSON part of GLB content
+ * @returns array of mesh types found
+ */
+const getMeshTypesFromGltf = (gltfJson: GLTF): Set<GltfPrimitiveModeString> => {
+  const result: Set<GltfPrimitiveModeString> = new Set();
+  for (const mesh of gltfJson.meshes || []) {
+    for (const primitive of mesh.primitives) {
+      let {mode} = primitive;
+      if (typeof mode !== 'number') {
+        mode = 4; // Default is 4 - TRIANGLES
+      }
+      result.add(GLTF_PRIMITIVE_MODES[mode]);
+    }
+  }
+  return result;
+};
+
+/**
+ * Merge object2 into object1
+ * @param object1
+ * @param object2
+ * @returns nothing
+ */
+export const mergePreprocessData = (object1: PreprocessData, object2: PreprocessData): void => {
+  // Merge topology mesh types info
+  for (const type of object2.meshTopologyTypes) {
+    object1.meshTopologyTypes.add(type);
+  }
+};

package/src/i3s-converter/helpers/tileset-traversal.ts

@@ -0,0 +1,51 @@
+import {Tiles3DTileJSONPostprocessed} from '@loaders.gl/3d-tiles';
+import {NodeIndexDocument} from './node-index-document';
+import {Matrix4} from '@math.gl/core';
+
+/** Traversal props for the conversion stage */
+export type TraversalConversionProps = {
+  /** Transformation matrix for the specific tile */
+  transform: Matrix4;
+  /** Parent nodes of the converted tile. Multiple nodes can be if one tile is converted to multiple nodes*/
+  parentNodes: NodeIndexDocument[];
+};
+
+/**
+ * Travesal of 3DTile tiles tree with making specific actions with each tile
+ * @param tile - 3DTiles tile JSON metadata
+ * @param traversalProps - traversal props used to pass data through recursive calls
+ * @param processTile - callback to make some actions with the current tile
+ * @param postprocessTile - callback to make some action after processing of the current tile and all the subtree
+ * @param maxDepth - max recursive calls number the travesal function will do. If not set, the traversal function will
+ *                   go through all the tree.
+ *                   This value is used to limit the convertion with only partial number of levels of the tileset
+ * @param level - counter to keep recursive calls number of the tiles tree. This value used to be able to break
+ *                traversal at the some level of the tree
+ * @returns void
+ */
+export const traverseDatasetWith = async <TProps>(
+  tile: Tiles3DTileJSONPostprocessed,
+  traversalProps: TProps,
+  processTile: (tile: Tiles3DTileJSONPostprocessed, traversalProps: TProps) => Promise<TProps>,
+  postprocessTile?: (processResults: TProps[], currentTraversalProps: TProps) => Promise<void>,
+  maxDepth?: number,
+  level = 0
+): Promise<void> => {
+  if (maxDepth && level > maxDepth) {
+    return;
+  }
+  const processResults: TProps[] = [];
+  const newTraversalProps: TProps = await processTile(tile, traversalProps);
+  processResults.push(newTraversalProps);
+  for (const childTile of tile.children) {
+    await traverseDatasetWith(
+      childTile,
+      newTraversalProps,
+      processTile,
+      postprocessTile,
+      maxDepth,
+      level + 1
+    );
+  }
+  postprocessTile && (await postprocessTile(processResults, traversalProps));
+};