@loaders.gl/tile-converter 3.3.0-alpha.7 → 3.3.0

package/src/i3s-converter/helpers/node-pages.ts

@@ -1,12 +1,10 @@
-import type {WriteQueueItem} from '../../lib/utils/write-queue';
-
 import {join} from 'path';
 import transform from 'json-map-transform';
 import {METADATA as metadataTemplate} from '../json-templates/metadata';
 import {NodeInPage} from '@loaders.gl/i3s';
-import WriteQueue from '../../lib/utils/write-queue';
+import {isFileExists, openJson} from '../../lib/utils/file-utils';
+import I3SConverter from '../i3s-converter';
 
-// @ts-nocheck
 /**
  * class NodePages - wrapper of nodePages array
  *
@@ -17,7 +15,7 @@ import WriteQueue from '../../lib/utils/write-queue';
  * const nodePages = new NodePages(writeFile, HARDCODED_NODES_PER_PAGE);
  * ...
  * // push root node
- * const parentId = nodePages.push({
+ * const parent = await nodePages.push({
       lodThreshold: HARDCODED_MAX_SCREEN_THRESHOLD_SQ,
       obb: coordinates.obb,
       children: []
@@ -34,7 +32,7 @@ import WriteQueue from '../../lib/utils/write-queue';
         }
       }
     };
- * const nodeId = this.nodePages.push(nodeInPage, parentId);
+ * const node = await this.nodePages.push(nodeInPage, parent.index);
  * ...
  * // save all the nodePages in the end of pushing all the nodes
  * await this.nodePages.save(layers0path);
@@ -42,8 +40,10 @@ import WriteQueue from '../../lib/utils/write-queue';
 export default class NodePages {
   readonly nodesPerPage: number;
   nodesCounter: number;
-  writeFile: Function;
+  writeFile: (...args) => Promise<null | string>;
+  converter: I3SConverter;
   readonly nodePages: {nodes: NodeInPage[]}[];
+  readonly length: number = 0;
 
   /**
    * @constructs
@@ -51,40 +51,233 @@ export default class NodePages {
    * @param writeFileFunc - function to save one nodePage into a file
    * @param nodesPerPage - length limit for one nodePage. An additional nodePage is created when this limit is met
    */
-  constructor(writeFileFunc, nodesPerPage) {
+  constructor(writeFileFunc, nodesPerPage, converter: I3SConverter) {
     this.nodesPerPage = nodesPerPage;
     this.nodesCounter = 0;
     // @ts-expect-error
     this.nodePages = [{}];
     this.nodePages[0].nodes = [];
     this.writeFile = writeFileFunc;
+    this.converter = converter;
+    this.length = 0;
   }
 
   /**
    * Setup function to save node pages
    * @param func - function which should be used to save node pages
    */
-  useWriteFunction(func: Function): void {
+  useWriteFunction(func: (...args) => Promise<null | string>): void {
     this.writeFile = func;
   }
 
+  /**
+   * Get file path and file name of the node page with the particular id
+   * @param nodePageId - node page id
+   * @returns file path and file name
+   */
+  private getNodePageFileName(nodePageId): {filePath: string; fileName: string} {
+    let filePath;
+    let fileName;
+    if (this.converter.options.slpk) {
+      filePath = join(this.converter.layers0Path, 'nodepages');
+      fileName = `${nodePageId.toString()}.json`;
+    } else {
+      filePath = join(this.converter.layers0Path, 'nodepages', nodePageId.toString());
+      fileName = 'index.json';
+    }
+    return {filePath, fileName};
+  }
+
+  /**
+   * Load node page from a file on the disk
+   * @param nodePageId - node page id
+   * @returns - node page data
+   */
+  private async loadNodePage(nodePageId: number): Promise<{nodes: NodeInPage[]}> {
+    const {filePath, fileName} = this.getNodePageFileName(nodePageId);
+    const fullName = join(filePath, fileName);
+    if (await isFileExists(fullName)) {
+      console.log(`load ${fullName}.`); // eslint-disable-line
+      return (await openJson(filePath, fileName)) as {nodes: NodeInPage[]};
+    } else {
+      return {nodes: []};
+    }
+  }
+
+  /**
+   * Get nodepage id by node id
+   * @param id node id
+   * @returns node page id
+   */
+  private getPageIndexByNodeId(id: number): number {
+    return Math.floor(id / this.nodesPerPage);
+  }
+
+  /**
+   * Get node page data by node id
+   * @param id node id
+   * @returns node page data
+   */
+  private async getPageByNodeId(id: number): Promise<{nodes: NodeInPage[]}> {
+    const pageIndex = this.getPageIndexByNodeId(id);
+    if (this.converter.options.instantNodeWriting) {
+      return await this.loadNodePage(pageIndex);
+    }
+    return this.nodePages[pageIndex];
+  }
+
   /**
    * Get the node by its end-to-end index
    * @param id - end-to-end index of the node
    * @return the node object
    */
-  getNodeById(id: number): NodeInPage {
-    const pageIndex = Math.floor(id / this.nodesPerPage);
+  async getNodeById(id: number, nodePage?: {nodes: NodeInPage[]}): Promise<NodeInPage> {
     const nodeIndex = id % this.nodesPerPage;
-    return this.nodePages[pageIndex].nodes[nodeIndex];
+    nodePage = nodePage || (await this.getPageByNodeId(id));
+    return nodePage.nodes[nodeIndex];
+  }
+
+  /**
+   * Add a child id into the parent node.children array
+   * @param parentId - end-to-end parent node index
+   * @param childId - end-to-end child node index
+   */
+  private async addChildRelation(parentId: number | undefined, childId: number): Promise<void> {
+    if (parentId === null || parentId === undefined) {
+      return;
+    }
+    const parentNode = await this.getNodeById(parentId);
+    parentNode.children?.push(childId);
+    await this.saveNode(parentNode);
+  }
+
+  /**
+   * Put new node in nodePages array
+   * @param node - node object
+   * @param parentId - index of parent node
+   * @return
+   */
+  async push(node: NodeInPage, parentId?: number): Promise<NodeInPage> {
+    node.index = this.nodesCounter++;
+    if (!this.converter.options.instantNodeWriting) {
+      let currentNodePage = this.nodePages[this.nodePages.length - 1];
+      if (currentNodePage.nodes.length === this.nodesPerPage) {
+        currentNodePage = {nodes: []};
+        this.nodePages.push(currentNodePage);
+      }
+      currentNodePage.nodes.push(node);
+    }
+    await this.addChildRelation(parentId, node.index);
+    NodePages.updateResourceInMesh(node);
+    await this.saveNode(node);
+    return node;
+  }
+
+  /**
+   * Save node to the file on the disk
+   * @param node - node data
+   */
+  async saveNode(node: NodeInPage): Promise<void> {
+    if (!this.converter.options.instantNodeWriting) {
+      return;
+    } else {
+      const nodePageIndex = this.getPageIndexByNodeId(node.index);
+      const nodePage = await this.getPageByNodeId(node.index);
+      const {filePath, fileName} = this.getNodePageFileName(nodePageIndex);
+      const nodeToUpdate = await this.getNodeById(node.index, nodePage);
+      if (nodeToUpdate) {
+        NodePages.updateAll(nodeToUpdate, node);
+      } else {
+        nodePage.nodes.push(node);
+      }
+      const nodePageStr = JSON.stringify(nodePage);
+      if (this.converter.options.slpk) {
+        await this.converter.writeQueue.enqueue(
+          {
+            archiveKey: `nodePages/${nodePageIndex.toString()}.json.gz`,
+            writePromise: () =>
+              this.writeFile(filePath, nodePageStr, fileName, true, this.converter.compressList)
+          },
+          true
+        );
+      } else {
+        await this.converter.writeQueue.enqueue(
+          {
+            writePromise: () => this.writeFile(filePath, nodePageStr)
+          },
+          true
+        );
+      }
+    }
+  }
+
+  /**
+   * Save metadata file (for slpk only)
+   */
+  async saveMetadata(): Promise<void> {
+    const metadata = transform({nodeCount: this.nodesCounter}, metadataTemplate());
+    const compress = false;
+    await this.converter.writeQueue.enqueue({
+      archiveKey: 'metadata.json',
+      writePromise: () =>
+        this.writeFile(
+          this.converter.layers0Path,
+          JSON.stringify(metadata),
+          'metadata.json',
+          compress
+        )
+    });
+  }
+
+  /**
+   * Save all the node pages
+   * Run this method when all nodes is pushed in nodePages
+   */
+  async save(): Promise<void> {
+    if (this.converter.options.instantNodeWriting) {
+      await this.saveMetadata();
+      return;
+    }
+    if (this.converter.options.slpk) {
+      for (const [index, nodePage] of this.nodePages.entries()) {
+        const nodePageStr = JSON.stringify(nodePage);
+        const slpkPath = join(this.converter.layers0Path, 'nodepages');
+        await this.converter.writeQueue.enqueue({
+          archiveKey: `nodePages/${index.toString()}.json.gz`,
+          writePromise: () => this.writeFile(slpkPath, nodePageStr, `${index.toString()}.json`)
+        });
+      }
+      await this.saveMetadata();
+    } else {
+      for (const [index, nodePage] of this.nodePages.entries()) {
+        const nodePageStr = JSON.stringify(nodePage);
+        const nodePagePath = join(this.converter.layers0Path, 'nodepages', index.toString());
+        await this.converter.writeQueue.enqueue({
+          writePromise: () => this.writeFile(nodePagePath, nodePageStr)
+        });
+      }
+    }
+  }
+
+  /**
+   * Update resource index in node.mesh object
+   * @param node - node object
+   */
+  static updateResourceInMesh(node: NodeInPage): void {
+    if (node.mesh && isFinite(node.index)) {
+      node.mesh.geometry.resource = node.index;
+    }
   }
 
   /**
    * Update all fields in the node excluding id
+   * @param node - node object
+   * @param data - NodeInPage data to replace original data
    */
-  updateAll(id: number, data: NodeInPage): void {
-    const node = this.getNodeById(id);
+  static updateAll(node: NodeInPage, data: NodeInPage): NodeInPage {
     Object.assign(node, data, {index: node.index});
+    NodePages.updateResourceInMesh(node);
+    return node;
   }
 
   /**
@@ -92,8 +285,7 @@ export default class NodePages {
    * @param id - end-to-end index of the node
    * @param materialId - id from scene layer materialDefinitions
    */
-  updateMaterialByNodeId(id: number, materialId: number): void {
-    const node = this.getNodeById(id);
+  static updateMaterialByNodeId(node: NodeInPage, materialId: number): void {
     if (!node.mesh) {
       return;
     }
@@ -108,8 +300,7 @@ export default class NodePages {
    * @param id - end-to-end index of the node
    * @param vertexCount - vertex count for particular node
    */
-  updateVertexCountByNodeId(id: number, vertexCount: number): void {
-    const node = this.getNodeById(id);
+  static updateVertexCountByNodeId(node: NodeInPage, vertexCount: number): void {
     if (!node.mesh) {
       return;
     }
@@ -118,10 +309,9 @@ export default class NodePages {
 
   /**
    * Update resource in node.mesh.attribute object by node id
-   * @param id - end-to-end index of the node
+   * @param node - node object
    */
-  updateNodeAttributeByNodeId(id: number): void {
-    const node = this.getNodeById(id);
+  static updateNodeAttributeByNodeId(node: NodeInPage): void {
     if (!node.mesh || !node.index) {
       return;
     }
@@ -130,11 +320,10 @@ export default class NodePages {
 
   /**
    * Update featureCount in node.mesh.geometry object by node id
-   * @param id - end-to-end index of the node
+   * @param node - node object
    * @param featureCount - features count of the node
    */
-  updateFeatureCountByNodeId(id: number, featureCount: number): void {
-    const node = this.getNodeById(id);
+  static updateFeatureCountByNodeId(node: NodeInPage, featureCount: number): void {
     if (!node.mesh) {
       return;
     }
@@ -143,97 +332,13 @@ export default class NodePages {
 
   /**
    * Update texelCountHint in node.mesh.material object by node id
-   * @param id - end-to-end index of the node
+   * @param node - node object
    * @param texelCountHint - texelCountHint of particular node
    */
-  updateTexelCountHintByNodeId(id: number, texelCountHint: number): void {
-    const node = this.getNodeById(id);
+  static updateTexelCountHintByNodeId(node: NodeInPage, texelCountHint: number): void {
     if (!node.mesh || !node.mesh.material) {
       return;
     }
     node.mesh.material.texelCountHint = texelCountHint;
   }
-
-  /**
-   * Add a child id into the parent node.children array
-   * @param parentId - end-to-end parent node index
-   * @param childId - end-to-end child node index
-   */
-  addChildRelation(parentId: number | undefined, childId: number): void {
-    if (parentId === null || parentId === undefined) {
-      return;
-    }
-    const parentNode = this.getNodeById(parentId);
-    parentNode.children?.push(childId);
-  }
-
-  /**
-   * Update resource index in node.mesh object
-   * @param node - node object
-   */
-  updateResourceInMesh(node: NodeInPage): void {
-    if (node.mesh && isFinite(node.index)) {
-      node.mesh.geometry.resource = node.index;
-    }
-  }
-
-  /**
-   * Put new node in nodePages array
-   * @param node - node object
-   * @param parentId - index of parent node
-   * @return
-   */
-  push(node: NodeInPage, parentId?: number): number {
-    let currentNodePage = this.nodePages[this.nodePages.length - 1];
-    if (currentNodePage.nodes.length === this.nodesPerPage) {
-      currentNodePage = {nodes: []};
-      this.nodePages.push(currentNodePage);
-    }
-    node.index = this.nodesCounter++;
-    currentNodePage.nodes.push(node);
-    this.addChildRelation(parentId, node.index);
-    this.updateResourceInMesh(node);
-    return node.index;
-  }
-
-  /**
-   * Save all the node pages
-   * Run this method when all nodes is pushed in nodePages
-   * @param layers0Path - path of layer
-   * @param writeQueue - write queue that controlls files write concurrency
-   * @param slpk
-   */
-  async save(
-    layers0Path: string,
-    writeQueue: WriteQueue<WriteQueueItem>,
-    slpk: boolean = false
-  ): Promise<void> {
-    if (slpk) {
-      for (const [index, nodePage] of this.nodePages.entries()) {
-        const nodePageStr = JSON.stringify(nodePage);
-        const slpkPath = join(layers0Path, 'nodepages');
-        await writeQueue.enqueue({
-          archiveKey: `nodePages/${index.toString()}.json.gz`,
-          writePromise: this.writeFile(slpkPath, nodePageStr, `${index.toString()}.json`)
-        });
-      }
-      const metadata = transform({nodeCount: this.nodesCounter}, metadataTemplate());
-      const compress = false;
-      await writeQueue.enqueue({
-        archiveKey: 'metadata.json',
-        writePromise: this.writeFile(
-          layers0Path,
-          JSON.stringify(metadata),
-          'metadata.json',
-          compress
-        )
-      });
-    } else {
-      for (const [index, nodePage] of this.nodePages.entries()) {
-        const nodePageStr = JSON.stringify(nodePage);
-        const nodePagePath = join(layers0Path, 'nodepages', index.toString());
-        await writeQueue.enqueue({writePromise: this.writeFile(nodePagePath, nodePageStr)});
-      }
-    }
-  }
 }