@loaders.gl/tile-converter 3.2.12 → 3.3.0-alpha.10

package/src/i3s-converter/helpers/node-pages.ts

@@ -1,12 +1,10 @@
-import type {WriteQueueItem} from '../../lib/utils/write-queue';
-
 import {join} from 'path';
 import transform from 'json-map-transform';
 import {METADATA as metadataTemplate} from '../json-templates/metadata';
 import {NodeInPage} from '@loaders.gl/i3s';
-import WriteQueue from '../../lib/utils/write-queue';
+import {isFileExists, openJson} from '../../lib/utils/file-utils';
+import I3SConverter from '../i3s-converter';
 
-// @ts-nocheck
 /**
  * class NodePages - wrapper of nodePages array
  *
@@ -17,7 +15,7 @@ import WriteQueue from '../../lib/utils/write-queue';
  * const nodePages = new NodePages(writeFile, HARDCODED_NODES_PER_PAGE);
  * ...
  * // push root node
- * const parentId = nodePages.push({
+ * const parent = await nodePages.push({
       lodThreshold: HARDCODED_MAX_SCREEN_THRESHOLD_SQ,
       obb: coordinates.obb,
       children: []
@@ -34,7 +32,7 @@ import WriteQueue from '../../lib/utils/write-queue';
         }
       }
     };
- * const nodeId = this.nodePages.push(nodeInPage, parentId);
+ * const node = await this.nodePages.push(nodeInPage, parent.index);
  * ...
  * // save all the nodePages in the end of pushing all the nodes
  * await this.nodePages.save(layers0path);
@@ -42,8 +40,10 @@ import WriteQueue from '../../lib/utils/write-queue';
 export default class NodePages {
   readonly nodesPerPage: number;
   nodesCounter: number;
-  writeFile: Function;
+  writeFile: (...args) => Promise<null | string>;
+  converter: I3SConverter;
   readonly nodePages: {nodes: NodeInPage[]}[];
+  readonly length: number = 0;
 
   /**
    * @constructs
@@ -51,99 +51,90 @@ export default class NodePages {
    * @param writeFileFunc - function to save one nodePage into a file
    * @param nodesPerPage - length limit for one nodePage. An additional nodePage is created when this limit is met
    */
-  constructor(writeFileFunc, nodesPerPage) {
+  constructor(writeFileFunc, nodesPerPage, converter: I3SConverter) {
     this.nodesPerPage = nodesPerPage;
     this.nodesCounter = 0;
     // @ts-expect-error
     this.nodePages = [{}];
     this.nodePages[0].nodes = [];
     this.writeFile = writeFileFunc;
+    this.converter = converter;
+    this.length = 0;
   }
 
   /**
    * Setup function to save node pages
    * @param func - function which should be used to save node pages
    */
-  useWriteFunction(func: Function): void {
+  useWriteFunction(func: (...args) => Promise<null | string>): void {
     this.writeFile = func;
   }
 
   /**
-   * Get the node by its end-to-end index
-   * @param id - end-to-end index of the node
-   * @return the node object
+   * Get file path and file name of the node page with the particular id
+   * @param nodePageId - node page id
+   * @returns file path and file name
    */
-  getNodeById(id: number): NodeInPage {
-    const pageIndex = Math.floor(id / this.nodesPerPage);
-    const nodeIndex = id % this.nodesPerPage;
-    return this.nodePages[pageIndex].nodes[nodeIndex];
-  }
-
-  /**
-   * Update material in node.mesh object by node id
-   * @param id - end-to-end index of the node
-   * @param materialId - id from scene layer materialDefinitions
-   */
-  updateMaterialByNodeId(id: number, materialId: number): void {
-    const node = this.getNodeById(id);
-    if (!node.mesh) {
-      return;
+  private getNodePageFileName(nodePageId): {filePath: string; fileName: string} {
+    let filePath;
+    let fileName;
+    if (this.converter.options.slpk) {
+      filePath = join(this.converter.layers0Path, 'nodepages');
+      fileName = `${nodePageId.toString()}.json`;
+    } else {
+      filePath = join(this.converter.layers0Path, 'nodepages', nodePageId.toString());
+      fileName = 'index.json';
     }
-    node.mesh.material = {
-      definition: materialId,
-      resource: node.index
-    };
+    return {filePath, fileName};
   }
 
   /**
-   * Update vertexCount in node.mesh.geometry object by node id
-   * @param id - end-to-end index of the node
-   * @param vertexCount - vertex count for particular node
+   * Load node page from a file on the disk
+   * @param nodePageId - node page id
+   * @returns - node page data
    */
-  updateVertexCountByNodeId(id: number, vertexCount: number): void {
-    const node = this.getNodeById(id);
-    if (!node.mesh) {
-      return;
+  private async loadNodePage(nodePageId: number): Promise<{nodes: NodeInPage[]}> {
+    const {filePath, fileName} = this.getNodePageFileName(nodePageId);
+    const fullName = join(filePath, fileName);
+    if (await isFileExists(fullName)) {
+      console.log(`load ${fullName}.`); // eslint-disable-line
+      return (await openJson(filePath, fileName)) as {nodes: NodeInPage[]};
+    } else {
+      return {nodes: []};
     }
-    node.mesh.geometry.vertexCount = vertexCount;
   }
 
   /**
-   * Update resource in node.mesh.attribute object by node id
-   * @param id - end-to-end index of the node
+   * Get nodepage id by node id
+   * @param id node id
+   * @returns node page id
    */
-  updateNodeAttributeByNodeId(id: number): void {
-    const node = this.getNodeById(id);
-    if (!node.mesh) {
-      return;
-    }
-    node.mesh.attribute.resource = node.index;
+  private getPageIndexByNodeId(id: number): number {
+    return Math.floor(id / this.nodesPerPage);
   }
 
   /**
-   * Update featureCount in node.mesh.geometry object by node id
-   * @param id - end-to-end index of the node
-   * @param featureCount - features count of the node
+   * Get node page data by node id
+   * @param id node id
+   * @returns node page data
    */
-  updateFeatureCountByNodeId(id: number, featureCount: number): void {
-    const node = this.getNodeById(id);
-    if (!node.mesh) {
-      return;
+  private async getPageByNodeId(id: number): Promise<{nodes: NodeInPage[]}> {
+    const pageIndex = this.getPageIndexByNodeId(id);
+    if (this.converter.options.instantNodeWriting) {
+      return await this.loadNodePage(pageIndex);
     }
-    node.mesh.geometry.featureCount = featureCount;
+    return this.nodePages[pageIndex];
   }
 
   /**
-   * Update texelCountHint in node.mesh.material object by node id
+   * Get the node by its end-to-end index
    * @param id - end-to-end index of the node
-   * @param texelCountHint - texelCountHint of particular node
+   * @return the node object
    */
-  updateTexelCountHintByNodeId(id: number, texelCountHint: number): void {
-    const node = this.getNodeById(id);
-    if (!node.mesh || !node.mesh.material) {
-      return;
-    }
-    node.mesh.material.texelCountHint = texelCountHint;
+  async getNodeById(id: number, nodePage?: {nodes: NodeInPage[]}): Promise<NodeInPage> {
+    const nodeIndex = id % this.nodesPerPage;
+    nodePage = nodePage || (await this.getPageByNodeId(id));
+    return nodePage.nodes[nodeIndex];
   }
 
   /**
@@ -151,81 +142,203 @@ export default class NodePages {
    * @param parentId - end-to-end parent node index
    * @param childId - end-to-end child node index
    */
-  addChildRelation(parentId: number | undefined, childId: number): void {
+  private async addChildRelation(parentId: number | undefined, childId: number): Promise<void> {
     if (parentId === null || parentId === undefined) {
       return;
     }
-    const parentNode = this.getNodeById(parentId);
+    const parentNode = await this.getNodeById(parentId);
     parentNode.children?.push(childId);
+    await this.saveNode(parentNode);
   }
 
   /**
-   * Update resource index in node.mesh object
+   * Put new node in nodePages array
    * @param node - node object
+   * @param parentId - index of parent node
+   * @return
    */
-  updateResourceInMesh(node: NodeInPage): void {
-    if (node.mesh) {
-      node.mesh.geometry.resource = node.index;
+  async push(node: NodeInPage, parentId?: number): Promise<NodeInPage> {
+    node.index = this.nodesCounter++;
+    if (!this.converter.options.instantNodeWriting) {
+      let currentNodePage = this.nodePages[this.nodePages.length - 1];
+      if (currentNodePage.nodes.length === this.nodesPerPage) {
+        currentNodePage = {nodes: []};
+        this.nodePages.push(currentNodePage);
+      }
+      currentNodePage.nodes.push(node);
     }
+    await this.addChildRelation(parentId, node.index);
+    NodePages.updateResourceInMesh(node);
+    await this.saveNode(node);
+    return node;
   }
 
   /**
-   * Put new node in nodePages array
-   * @param node - node object
-   * @param parentId - index of parent node
-   * @return
+   * Save node to the file on the disk
+   * @param node - node data
    */
-  push(node: NodeInPage, parentId?: number): number {
-    let currentNodePage = this.nodePages[this.nodePages.length - 1];
-    if (currentNodePage.nodes.length === this.nodesPerPage) {
-      currentNodePage = {nodes: []};
-      this.nodePages.push(currentNodePage);
+  async saveNode(node: NodeInPage): Promise<void> {
+    if (!this.converter.options.instantNodeWriting) {
+      return;
+    } else {
+      const nodePageIndex = this.getPageIndexByNodeId(node.index);
+      const nodePage = await this.getPageByNodeId(node.index);
+      const {filePath, fileName} = this.getNodePageFileName(nodePageIndex);
+      const nodeToUpdate = await this.getNodeById(node.index, nodePage);
+      if (nodeToUpdate) {
+        NodePages.updateAll(nodeToUpdate, node);
+      } else {
+        nodePage.nodes.push(node);
+      }
+      const nodePageStr = JSON.stringify(nodePage);
+      if (this.converter.options.slpk) {
+        await this.converter.writeQueue.enqueue(
+          {
+            archiveKey: `nodePages/${nodePageIndex.toString()}.json.gz`,
+            writePromise: () =>
+              this.writeFile(filePath, nodePageStr, fileName, true, this.converter.compressList)
+          },
+          true
+        );
+      } else {
+        await this.converter.writeQueue.enqueue(
+          {
+            writePromise: () => this.writeFile(filePath, nodePageStr)
+          },
+          true
+        );
+      }
     }
-    node.index = this.nodesCounter++;
-    currentNodePage.nodes.push(node);
-    this.addChildRelation(parentId, node.index);
-    this.updateResourceInMesh(node);
-    return node.index;
+  }
+
+  /**
+   * Save metadata file (for slpk only)
+   */
+  async saveMetadata(): Promise<void> {
+    const metadata = transform({nodeCount: this.nodesCounter}, metadataTemplate());
+    const compress = false;
+    await this.converter.writeQueue.enqueue({
+      archiveKey: 'metadata.json',
+      writePromise: () =>
+        this.writeFile(
+          this.converter.layers0Path,
+          JSON.stringify(metadata),
+          'metadata.json',
+          compress
+        )
+    });
   }
 
   /**
    * Save all the node pages
    * Run this method when all nodes is pushed in nodePages
-   * @param layers0Path - path of layer
-   * @param writeQueue - write queue that controlls files write concurrency
-   * @param slpk
-   */
-  async save(
-    layers0Path: string,
-    writeQueue: WriteQueue<WriteQueueItem>,
-    slpk: boolean = false
-  ): Promise<void> {
-    if (slpk) {
+   */
+  async save(): Promise<void> {
+    if (this.converter.options.instantNodeWriting) {
+      await this.saveMetadata();
+      return;
+    }
+    if (this.converter.options.slpk) {
       for (const [index, nodePage] of this.nodePages.entries()) {
         const nodePageStr = JSON.stringify(nodePage);
-        const slpkPath = join(layers0Path, 'nodepages');
-        await writeQueue.enqueue({
+        const slpkPath = join(this.converter.layers0Path, 'nodepages');
+        await this.converter.writeQueue.enqueue({
           archiveKey: `nodePages/${index.toString()}.json.gz`,
-          writePromise: this.writeFile(slpkPath, nodePageStr, `${index.toString()}.json`)
+          writePromise: () => this.writeFile(slpkPath, nodePageStr, `${index.toString()}.json`)
         });
       }
-      const metadata = transform({nodeCount: this.nodesCounter}, metadataTemplate());
-      const compress = false;
-      await writeQueue.enqueue({
-        archiveKey: 'metadata.json',
-        writePromise: this.writeFile(
-          layers0Path,
-          JSON.stringify(metadata),
-          'metadata.json',
-          compress
-        )
-      });
+      await this.saveMetadata();
     } else {
       for (const [index, nodePage] of this.nodePages.entries()) {
         const nodePageStr = JSON.stringify(nodePage);
-        const nodePagePath = join(layers0Path, 'nodepages', index.toString());
-        await writeQueue.enqueue({writePromise: this.writeFile(nodePagePath, nodePageStr)});
+        const nodePagePath = join(this.converter.layers0Path, 'nodepages', index.toString());
+        await this.converter.writeQueue.enqueue({
+          writePromise: () => this.writeFile(nodePagePath, nodePageStr)
+        });
       }
     }
   }
+
+  /**
+   * Update resource index in node.mesh object
+   * @param node - node object
+   */
+  static updateResourceInMesh(node: NodeInPage): void {
+    if (node.mesh && isFinite(node.index)) {
+      node.mesh.geometry.resource = node.index;
+    }
+  }
+
+  /**
+   * Update all fields in the node excluding id
+   * @param node - node object
+   * @param data - NodeInPage data to replace original data
+   */
+  static updateAll(node: NodeInPage, data: NodeInPage): NodeInPage {
+    Object.assign(node, data, {index: node.index});
+    NodePages.updateResourceInMesh(node);
+    return node;
+  }
+
+  /**
+   * Update material in node.mesh object by node id
+   * @param id - end-to-end index of the node
+   * @param materialId - id from scene layer materialDefinitions
+   */
+  static updateMaterialByNodeId(node: NodeInPage, materialId: number): void {
+    if (!node.mesh) {
+      return;
+    }
+    node.mesh.material = {
+      definition: materialId,
+      resource: node.index
+    };
+  }
+
+  /**
+   * Update vertexCount in node.mesh.geometry object by node id
+   * @param id - end-to-end index of the node
+   * @param vertexCount - vertex count for particular node
+   */
+  static updateVertexCountByNodeId(node: NodeInPage, vertexCount: number): void {
+    if (!node.mesh) {
+      return;
+    }
+    node.mesh.geometry.vertexCount = vertexCount;
+  }
+
+  /**
+   * Update resource in node.mesh.attribute object by node id
+   * @param node - node object
+   */
+  static updateNodeAttributeByNodeId(node: NodeInPage): void {
+    if (!node.mesh || !node.index) {
+      return;
+    }
+    node.mesh.attribute.resource = node.index;
+  }
+
+  /**
+   * Update featureCount in node.mesh.geometry object by node id
+   * @param node - node object
+   * @param featureCount - features count of the node
+   */
+  static updateFeatureCountByNodeId(node: NodeInPage, featureCount: number): void {
+    if (!node.mesh) {
+      return;
+    }
+    node.mesh.geometry.featureCount = featureCount;
+  }
+
+  /**
+   * Update texelCountHint in node.mesh.material object by node id
+   * @param node - node object
+   * @param texelCountHint - texelCountHint of particular node
+   */
+  static updateTexelCountHintByNodeId(node: NodeInPage, texelCountHint: number): void {
+    if (!node.mesh || !node.mesh.material) {
+      return;
+    }
+    node.mesh.material.texelCountHint = texelCountHint;
+  }
 }