@loaders.gl/tile-converter 4.0.0-alpha.1 → 4.0.0-alpha.10

package/dist/i3s-converter/helpers/node-pages.js

@@ -1,136 +1,316 @@
-import { join } from 'path';
-import transform from 'json-map-transform';
-import { METADATA as metadataTemplate } from '../json-templates/metadata';
-export default class NodePages {
-  constructor(writeFileFunc, nodesPerPage) {
-    this.nodesPerPage = nodesPerPage;
-    this.nodesCounter = 0;
-    this.nodePages = [{}];
-    this.nodePages[0].nodes = [];
-    this.writeFile = writeFileFunc;
-  }
-
-  useWriteFunction(func) {
-    this.writeFile = func;
-  }
-
-  getNodeById(id) {
-    const pageIndex = Math.floor(id / this.nodesPerPage);
-    const nodeIndex = id % this.nodesPerPage;
-    return this.nodePages[pageIndex].nodes[nodeIndex];
-  }
-
-  updateMaterialByNodeId(id, materialId) {
-    const node = this.getNodeById(id);
-
-    if (!node.mesh) {
-      return;
-    }
-
-    node.mesh.material = {
-      definition: materialId,
-      resource: node.index
-    };
-  }
-
-  updateVertexCountByNodeId(id, vertexCount) {
-    const node = this.getNodeById(id);
-
-    if (!node.mesh) {
-      return;
-    }
-
-    node.mesh.geometry.vertexCount = vertexCount;
-  }
-
-  updateNodeAttributeByNodeId(id) {
-    const node = this.getNodeById(id);
-
-    if (!node.mesh) {
-      return;
-    }
-
-    node.mesh.attribute.resource = node.index;
-  }
-
-  updateFeatureCountByNodeId(id, featureCount) {
-    const node = this.getNodeById(id);
-
-    if (!node.mesh) {
-      return;
-    }
-
-    node.mesh.geometry.featureCount = featureCount;
-  }
-
-  updateTexelCountHintByNodeId(id, texelCountHint) {
-    const node = this.getNodeById(id);
-
-    if (!node.mesh || !node.mesh.material) {
-      return;
-    }
-
-    node.mesh.material.texelCountHint = texelCountHint;
-  }
-
-  addChildRelation(parentId, childId) {
-    if (parentId === null) {
-      return;
-    }
-
-    const parentNode = this.getNodeById(parentId);
-    parentNode.children.push(childId);
-  }
-
-  updateResourceInMesh(node) {
-    if (node.mesh) {
-      node.mesh.geometry.resource = node.index;
-    }
-  }
-
-  push(node, parentId = null) {
-    let currentNodePage = this.nodePages[this.nodePages.length - 1];
-
-    if (currentNodePage.nodes.length === this.nodesPerPage) {
-      currentNodePage = {
-        nodes: []
-      };
-      this.nodePages.push(currentNodePage);
-    }
-
-    node.index = this.nodesCounter++;
-    currentNodePage.nodes.push(node);
-    this.addChildRelation(parentId, node.index);
-    this.updateResourceInMesh(node);
-    return node.index;
-  }
-
-  async save(layers0Path, fileMap, slpk = false) {
-    const promises = [];
-
-    if (slpk) {
-      for (const [index, nodePage] of this.nodePages.entries()) {
-        const nodePageStr = JSON.stringify(nodePage);
-        const slpkPath = join(layers0Path, 'nodepages');
-        promises.push(this.writeFile(slpkPath, nodePageStr, "".concat(index.toString(), ".json")));
-        fileMap["nodePages/".concat(index.toString(), ".json.gz")] = "".concat(slpkPath, ".json.gz");
-      }
-
-      const metadata = transform({
-        nodeCount: this.nodesCounter
-      }, metadataTemplate);
-      const compress = false;
-      fileMap['metadata.json'] = await this.writeFile(layers0Path, JSON.stringify(metadata), 'metadata.json', compress);
-    } else {
-      for (const [index, nodePage] of this.nodePages.entries()) {
-        const nodePageStr = JSON.stringify(nodePage);
-        const nodePagePath = join(layers0Path, 'nodepages', index.toString());
-        promises.push(this.writeFile(nodePagePath, nodePageStr));
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path_1 = require("path");
+const json_map_transform_1 = __importDefault(require("json-map-transform"));
+const metadata_1 = require("../json-templates/metadata");
+const file_utils_1 = require("../../lib/utils/file-utils");
+/**
+ * class NodePages - wrapper of nodePages array
+ *
+ * @example
+ * import {writeFile} from './helpers/write-file';
+ *
+ * // create an instance of the class
+ * const nodePages = new NodePages(writeFile, HARDCODED_NODES_PER_PAGE);
+ * ...
+ * // push root node
+ * const parent = await nodePages.push({
+      lodThreshold: HARDCODED_MAX_SCREEN_THRESHOLD_SQ,
+      obb: coordinates.obb,
+      children: []
+    });
+ * ...
+ * // push node with parent relation
+ * const nodeInPage = {
+      lodThreshold: HARDCODED_MAX_SCREEN_THRESHOLD_SQ,
+      obb: coordinates.obb,
+      children: [],
+      mesh: {
+        geometry: {
+          definition: 0
+        }
       }
+    };
+ * const node = await this.nodePages.push(nodeInPage, parent.index);
+ * ...
+ * // save all the nodePages in the end of pushing all the nodes
+ * await this.nodePages.save(layers0path);
+ */
+class NodePages {
+    /**
+     * @constructs
+     * Create a nodePages instance.
+     * @param writeFileFunc - function to save one nodePage into a file
+     * @param nodesPerPage - length limit for one nodePage. An additional nodePage is created when this limit is met
+     */
+    constructor(writeFileFunc, nodesPerPage, converter) {
+        this.length = 0;
+        this.nodesPerPage = nodesPerPage;
+        this.nodesCounter = 0;
+        // @ts-expect-error
+        this.nodePages = [{}];
+        this.nodePages[0].nodes = [];
+        this.writeFile = writeFileFunc;
+        this.converter = converter;
+        this.length = 0;
+    }
+    /**
+     * Setup function to save node pages
+     * @param func - function which should be used to save node pages
+     */
+    useWriteFunction(func) {
+        this.writeFile = func;
+    }
+    /**
+     * Get file path and file name of the node page with the particular id
+     * @param nodePageId - node page id
+     * @returns file path and file name
+     */
+    getNodePageFileName(nodePageId) {
+        let filePath;
+        let fileName;
+        if (this.converter.options.slpk) {
+            filePath = (0, path_1.join)(this.converter.layers0Path, 'nodepages');
+            fileName = `${nodePageId.toString()}.json`;
+        }
+        else {
+            filePath = (0, path_1.join)(this.converter.layers0Path, 'nodepages', nodePageId.toString());
+            fileName = 'index.json';
+        }
+        return { filePath, fileName };
+    }
+    /**
+     * Load node page from a file on the disk
+     * @param nodePageId - node page id
+     * @returns - node page data
+     */
+    async loadNodePage(nodePageId) {
+        const { filePath, fileName } = this.getNodePageFileName(nodePageId);
+        const fullName = (0, path_1.join)(filePath, fileName);
+        if (await (0, file_utils_1.isFileExists)(fullName)) {
+            console.log(`load ${fullName}.`); // eslint-disable-line
+            return (await (0, file_utils_1.openJson)(filePath, fileName));
+        }
+        else {
+            return { nodes: [] };
+        }
+    }
+    /**
+     * Get nodepage id by node id
+     * @param id node id
+     * @returns node page id
+     */
+    getPageIndexByNodeId(id) {
+        return Math.floor(id / this.nodesPerPage);
+    }
+    /**
+     * Get node page data by node id
+     * @param id node id
+     * @returns node page data
+     */
+    async getPageByNodeId(id) {
+        const pageIndex = this.getPageIndexByNodeId(id);
+        if (this.converter.options.instantNodeWriting) {
+            return await this.loadNodePage(pageIndex);
+        }
+        return this.nodePages[pageIndex];
+    }
+    /**
+     * Get the node by its end-to-end index
+     * @param id - end-to-end index of the node
+     * @return the node object
+     */
+    async getNodeById(id, nodePage) {
+        const nodeIndex = id % this.nodesPerPage;
+        nodePage = nodePage || (await this.getPageByNodeId(id));
+        return nodePage.nodes[nodeIndex];
+    }
+    /**
+     * Add a child id into the parent node.children array
+     * @param parentId - end-to-end parent node index
+     * @param childId - end-to-end child node index
+     */
+    async addChildRelation(parentId, childId) {
+        if (parentId === null || parentId === undefined) {
+            return;
+        }
+        const parentNode = await this.getNodeById(parentId);
+        parentNode.children?.push(childId);
+        await this.saveNode(parentNode);
+    }
+    /**
+     * Put new node in nodePages array
+     * @param node - node object
+     * @param parentId - index of parent node
+     * @return
+     */
+    async push(node, parentId) {
+        node.index = this.nodesCounter++;
+        if (!this.converter.options.instantNodeWriting) {
+            let currentNodePage = this.nodePages[this.nodePages.length - 1];
+            if (currentNodePage.nodes.length === this.nodesPerPage) {
+                currentNodePage = { nodes: [] };
+                this.nodePages.push(currentNodePage);
+            }
+            currentNodePage.nodes.push(node);
+        }
+        await this.addChildRelation(parentId, node.index);
+        NodePages.updateResourceInMesh(node);
+        await this.saveNode(node);
+        return node;
+    }
+    /**
+     * Save node to the file on the disk
+     * @param node - node data
+     */
+    async saveNode(node) {
+        if (!this.converter.options.instantNodeWriting) {
+            return;
+        }
+        else {
+            const nodePageIndex = this.getPageIndexByNodeId(node.index);
+            const nodePage = await this.getPageByNodeId(node.index);
+            const { filePath, fileName } = this.getNodePageFileName(nodePageIndex);
+            const nodeToUpdate = await this.getNodeById(node.index, nodePage);
+            if (nodeToUpdate) {
+                NodePages.updateAll(nodeToUpdate, node);
+            }
+            else {
+                nodePage.nodes.push(node);
+            }
+            const nodePageStr = JSON.stringify(nodePage);
+            if (this.converter.options.slpk) {
+                await this.converter.writeQueue.enqueue({
+                    archiveKey: `nodePages/${nodePageIndex.toString()}.json.gz`,
+                    writePromise: () => this.writeFile(filePath, nodePageStr, fileName, true, this.converter.compressList)
+                }, true);
+            }
+            else {
+                await this.converter.writeQueue.enqueue({
+                    writePromise: () => this.writeFile(filePath, nodePageStr)
+                }, true);
+            }
+        }
+    }
+    /**
+     * Save metadata file (for slpk only)
+     */
+    async saveMetadata() {
+        const metadata = (0, json_map_transform_1.default)({ nodeCount: this.nodesCounter }, (0, metadata_1.METADATA)());
+        const compress = false;
+        await this.converter.writeQueue.enqueue({
+            archiveKey: 'metadata.json',
+            writePromise: () => this.writeFile(this.converter.layers0Path, JSON.stringify(metadata), 'metadata.json', compress)
+        });
+    }
+    /**
+     * Save all the node pages
+     * Run this method when all nodes is pushed in nodePages
+     */
+    async save() {
+        if (this.converter.options.instantNodeWriting) {
+            await this.saveMetadata();
+            return;
+        }
+        if (this.converter.options.slpk) {
+            for (const [index, nodePage] of this.nodePages.entries()) {
+                const nodePageStr = JSON.stringify(nodePage);
+                const slpkPath = (0, path_1.join)(this.converter.layers0Path, 'nodepages');
+                await this.converter.writeQueue.enqueue({
+                    archiveKey: `nodePages/${index.toString()}.json.gz`,
+                    writePromise: () => this.writeFile(slpkPath, nodePageStr, `${index.toString()}.json`)
+                });
+            }
+            await this.saveMetadata();
+        }
+        else {
+            for (const [index, nodePage] of this.nodePages.entries()) {
+                const nodePageStr = JSON.stringify(nodePage);
+                const nodePagePath = (0, path_1.join)(this.converter.layers0Path, 'nodepages', index.toString());
+                await this.converter.writeQueue.enqueue({
+                    writePromise: () => this.writeFile(nodePagePath, nodePageStr)
+                });
+            }
+        }
+    }
+    /**
+     * Update resource index in node.mesh object
+     * @param node - node object
+     */
+    static updateResourceInMesh(node) {
+        if (node.mesh && isFinite(node.index)) {
+            node.mesh.geometry.resource = node.index;
+        }
+    }
+    /**
+     * Update all fields in the node excluding id
+     * @param node - node object
+     * @param data - NodeInPage data to replace original data
+     */
+    static updateAll(node, data) {
+        Object.assign(node, data, { index: node.index });
+        NodePages.updateResourceInMesh(node);
+        return node;
+    }
+    /**
+     * Update material in node.mesh object by node id
+     * @param id - end-to-end index of the node
+     * @param materialId - id from scene layer materialDefinitions
+     */
+    static updateMaterialByNodeId(node, materialId) {
+        if (!node.mesh) {
+            return;
+        }
+        node.mesh.material = {
+            definition: materialId,
+            resource: node.index
+        };
+    }
+    /**
+     * Update vertexCount in node.mesh.geometry object by node id
+     * @param id - end-to-end index of the node
+     * @param vertexCount - vertex count for particular node
+     */
+    static updateVertexCountByNodeId(node, vertexCount) {
+        if (!node.mesh) {
+            return;
+        }
+        node.mesh.geometry.vertexCount = vertexCount;
+    }
+    /**
+     * Update resource in node.mesh.attribute object by node id
+     * @param node - node object
+     */
+    static updateNodeAttributeByNodeId(node) {
+        if (!node.mesh || !node.index) {
+            return;
+        }
+        node.mesh.attribute.resource = node.index;
+    }
+    /**
+     * Update featureCount in node.mesh.geometry object by node id
+     * @param node - node object
+     * @param featureCount - features count of the node
+     */
+    static updateFeatureCountByNodeId(node, featureCount) {
+        if (!node.mesh) {
+            return;
+        }
+        node.mesh.geometry.featureCount = featureCount;
+    }
+    /**
+     * Update texelCountHint in node.mesh.material object by node id
+     * @param node - node object
+     * @param texelCountHint - texelCountHint of particular node
+     */
+    static updateTexelCountHintByNodeId(node, texelCountHint) {
+        if (!node.mesh || !node.mesh.material) {
+            return;
+        }
+        node.mesh.material.texelCountHint = texelCountHint;
     }
-
-    await Promise.all(promises);
-  }
-
 }
-//# sourceMappingURL=node-pages.js.map
+exports.default = NodePages;

package/dist/i3s-converter/helpers/preprocess-3d-tiles.d.ts

@@ -0,0 +1,23 @@
+import { Tiles3DTileContent, Tiles3DTileJSONPostprocessed } from '@loaders.gl/3d-tiles';
+import { GltfPrimitiveModeString, PreprocessData } from '../types';
+/**
+ * glTF primitive modes
+ * @see https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#_mesh_primitive_mode
+ */
+export declare const GLTF_PRIMITIVE_MODES: GltfPrimitiveModeString[];
+/**
+ * Analyze tile content. This function is used during preprocess stage of
+ * conversion
+ * @param tile - 3DTiles tile JSON metadata
+ * @param tileContent - 3DTiles tile content ArrayBuffer
+ * @returns
+ */
+export declare const analyzeTileContent: (tile: Tiles3DTileJSONPostprocessed, tileContent: Tiles3DTileContent | null) => Promise<PreprocessData>;
+/**
+ * Merge object2 into object1
+ * @param object1
+ * @param object2
+ * @returns nothing
+ */
+export declare const mergePreprocessData: (object1: PreprocessData, object2: PreprocessData) => void;
+//# sourceMappingURL=preprocess-3d-tiles.d.ts.map

package/dist/i3s-converter/helpers/preprocess-3d-tiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"preprocess-3d-tiles.d.ts","sourceRoot":"","sources":["../../../src/i3s-converter/helpers/preprocess-3d-tiles.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,kBAAkB,EAAE,4BAA4B,EAAC,MAAM,sBAAsB,CAAC;AACtF,OAAO,EAAC,uBAAuB,EAAE,cAAc,EAAC,MAAM,UAAU,CAAC;AAIjE;;;GAGG;AACH,eAAO,MAAM,oBAAoB,2BAQhC,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,kBAAkB,SACvB,4BAA4B,eACrB,kBAAkB,GAAG,IAAI,KACrC,QAAQ,cAAc,CAmBxB,CAAC;AAqBF;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB,YAAa,cAAc,WAAW,cAAc,KAAG,IAKtF,CAAC"}

package/dist/i3s-converter/helpers/preprocess-3d-tiles.js

@@ -0,0 +1,76 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mergePreprocessData = exports.analyzeTileContent = exports.GLTF_PRIMITIVE_MODES = void 0;
+const types_1 = require("../types");
+const gltf_1 = require("@loaders.gl/gltf");
+const core_1 = require("@loaders.gl/core");
+/**
+ * glTF primitive modes
+ * @see https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#_mesh_primitive_mode
+ */
+exports.GLTF_PRIMITIVE_MODES = [
+    types_1.GltfPrimitiveModeString.POINTS,
+    types_1.GltfPrimitiveModeString.LINES,
+    types_1.GltfPrimitiveModeString.LINE_LOOP,
+    types_1.GltfPrimitiveModeString.LINE_STRIP,
+    types_1.GltfPrimitiveModeString.TRIANGLES,
+    types_1.GltfPrimitiveModeString.TRIANGLE_STRIP,
+    types_1.GltfPrimitiveModeString.TRIANGLE_FAN // 6
+];
+/**
+ * Analyze tile content. This function is used during preprocess stage of
+ * conversion
+ * @param tile - 3DTiles tile JSON metadata
+ * @param tileContent - 3DTiles tile content ArrayBuffer
+ * @returns
+ */
+const analyzeTileContent = async (tile, tileContent) => {
+    const result = {
+        meshTopologyTypes: new Set()
+    };
+    if (!tileContent?.gltfArrayBuffer) {
+        return result;
+    }
+    const gltfData = await (0, core_1.parse)(tileContent.gltfArrayBuffer, gltf_1.GLTFLoader, {
+        gltf: { normalize: false, loadBuffers: false, loadImages: false, decompressMeshes: false }
+    });
+    const gltf = gltfData.json;
+    if (!gltf) {
+        return result;
+    }
+    const meshTypes = getMeshTypesFromGltf(gltf);
+    result.meshTopologyTypes = meshTypes;
+    return result;
+};
+exports.analyzeTileContent = analyzeTileContent;
+/**
+ * Get mesh topology types that the glb content has
+ * @param gltfJson - JSON part of GLB content
+ * @returns array of mesh types found
+ */
+const getMeshTypesFromGltf = (gltfJson) => {
+    const result = new Set();
+    for (const mesh of gltfJson.meshes || []) {
+        for (const primitive of mesh.primitives) {
+            let { mode } = primitive;
+            if (typeof mode !== 'number') {
+                mode = 4; // Default is 4 - TRIANGLES
+            }
+            result.add(exports.GLTF_PRIMITIVE_MODES[mode]);
+        }
+    }
+    return result;
+};
+/**
+ * Merge object2 into object1
+ * @param object1
+ * @param object2
+ * @returns nothing
+ */
+const mergePreprocessData = (object1, object2) => {
+    // Merge topology mesh types info
+    for (const type of object2.meshTopologyTypes) {
+        object1.meshTopologyTypes.add(type);
+    }
+};
+exports.mergePreprocessData = mergePreprocessData;

package/dist/i3s-converter/helpers/tileset-traversal.d.ts

@@ -0,0 +1,25 @@
+import { Tiles3DTileJSONPostprocessed } from '@loaders.gl/3d-tiles';
+import { NodeIndexDocument } from './node-index-document';
+import { Matrix4 } from '@math.gl/core';
+/** Traversal props for the conversion stage */
+export type TraversalConversionProps = {
+    /** Transformation matrix for the specific tile */
+    transform: Matrix4;
+    /** Parent nodes of the converted tile. Multiple nodes can be if one tile is converted to multiple nodes*/
+    parentNodes: NodeIndexDocument[];
+};
+/**
+ * Travesal of 3DTile tiles tree with making specific actions with each tile
+ * @param tile - 3DTiles tile JSON metadata
+ * @param traversalProps - traversal props used to pass data through recursive calls
+ * @param processTile - callback to make some actions with the current tile
+ * @param postprocessTile - callback to make some action after processing of the current tile and all the subtree
+ * @param maxDepth - max recursive calls number the travesal function will do. If not set, the traversal function will
+ *                   go through all the tree.
+ *                   This value is used to limit the convertion with only partial number of levels of the tileset
+ * @param level - counter to keep recursive calls number of the tiles tree. This value used to be able to break
+ *                traversal at the some level of the tree
+ * @returns void
+ */
+export declare const traverseDatasetWith: <TProps>(tile: Tiles3DTileJSONPostprocessed, traversalProps: TProps, processTile: (tile: Tiles3DTileJSONPostprocessed, traversalProps: TProps) => Promise<TProps>, postprocessTile?: ((processResults: TProps[], currentTraversalProps: TProps) => Promise<void>) | undefined, maxDepth?: number, level?: number) => Promise<void>;
+//# sourceMappingURL=tileset-traversal.d.ts.map

package/dist/i3s-converter/helpers/tileset-traversal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tileset-traversal.d.ts","sourceRoot":"","sources":["../../../src/i3s-converter/helpers/tileset-traversal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,4BAA4B,EAAC,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAC,iBAAiB,EAAC,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAEtC,+CAA+C;AAC/C,MAAM,MAAM,wBAAwB,GAAG;IACrC,kDAAkD;IAClD,SAAS,EAAE,OAAO,CAAC;IACnB,0GAA0G;IAC1G,WAAW,EAAE,iBAAiB,EAAE,CAAC;CAClC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,mBAAmB,iBACxB,4BAA4B,8CAEd,4BAA4B,8HAC+B,QAAQ,IAAI,CAAC,0BACjF,MAAM,qBAEhB,QAAQ,IAAI,CAkBd,CAAC"}

package/dist/i3s-converter/helpers/tileset-traversal.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.traverseDatasetWith = void 0;
+/**
+ * Travesal of 3DTile tiles tree with making specific actions with each tile
+ * @param tile - 3DTiles tile JSON metadata
+ * @param traversalProps - traversal props used to pass data through recursive calls
+ * @param processTile - callback to make some actions with the current tile
+ * @param postprocessTile - callback to make some action after processing of the current tile and all the subtree
+ * @param maxDepth - max recursive calls number the travesal function will do. If not set, the traversal function will
+ *                   go through all the tree.
+ *                   This value is used to limit the convertion with only partial number of levels of the tileset
+ * @param level - counter to keep recursive calls number of the tiles tree. This value used to be able to break
+ *                traversal at the some level of the tree
+ * @returns void
+ */
+const traverseDatasetWith = async (tile, traversalProps, processTile, postprocessTile, maxDepth, level = 0) => {
+    if (maxDepth && level > maxDepth) {
+        return;
+    }
+    const processResults = [];
+    const newTraversalProps = await processTile(tile, traversalProps);
+    processResults.push(newTraversalProps);
+    for (const childTile of tile.children) {
+        await (0, exports.traverseDatasetWith)(childTile, newTraversalProps, processTile, postprocessTile, maxDepth, level + 1);
+    }
+    postprocessTile && (await postprocessTile(processResults, traversalProps));
+};
+exports.traverseDatasetWith = traverseDatasetWith;