@loaders.gl/tile-converter 4.0.0-beta.8 → 4.0.1

package/src/i3s-converter/i3s-converter.ts

@@ -1,6 +1,8 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
+import {AttributeMetadataInfo} from './helpers/attribute-metadata-info';
+
 import type {
   FeatureTableJson,
   Tiles3DLoaderOptions,
@@ -62,12 +64,8 @@ import {WorkerFarm} from '@loaders.gl/worker-utils';
 import WriteQueue from '../lib/utils/write-queue';
 import {BROWSER_ERROR_MESSAGE} from '../constants';
 import {
-  getAttributeTypesFromPropertyTable,
-  getAttributeTypesFromSchema,
-  createdStorageAttribute,
-  getFieldAttributeType,
-  createFieldAttribute,
-  createPopupInfo
+  getAttributeTypesMapFromPropertyTable,
+  getAttributeTypesMapFromSchema
 } from './helpers/feature-attributes';
 import {NodeIndexDocument} from './helpers/node-index-document';
 import {
@@ -94,6 +92,7 @@ const CESIUM_DATASET_PREFIX = 'https://';
  * Converter from 3d-tiles tileset to i3s layer
  */
 export default class I3SConverter {
+  attributeMetadataInfo: AttributeMetadataInfo;
   nodePages: NodePages;
   options: any;
   layers0Path: string;
@@ -139,8 +138,18 @@ export default class I3SConverter {
     meshTopologyTypes: new Set(),
     metadataClasses: new Set()
   };
+  /** Total count of tiles in tileset */
+  tileCountTotal: number = 0;
+  /** Count of tiles already converted plus one (refers to the tile currently being converted) */
+  tileCountCurrentlyConverting: number = 0;
+  /**
+   * The number of digits to appear after decimal point in the string representation of the tile count.
+   * It's calculated based on the total count of tiles.
+   */
+  numberOfDigitsInPercentage: number = 0;
 
   constructor() {
+    this.attributeMetadataInfo = new AttributeMetadataInfo();
     this.nodePages = new NodePages(writeFile, HARDCODED_NODES_PER_PAGE, this);
     this.options = {};
     this.layers0Path = '';
@@ -301,8 +310,12 @@ export default class I3SConverter {
     );
     const {meshTopologyTypes, metadataClasses} = this.preprocessData;
 
+    this.numberOfDigitsInPercentage =
+      this.tileCountTotal > 100 ? Math.ceil(Math.log10(this.tileCountTotal)) - 2 : 0;
+
     console.log(`------------------------------------------------`);
     console.log(`Preprocess results:`);
+    console.log(`Tile count: ${this.tileCountTotal}`);
     console.log(`glTF mesh topology types: ${Array.from(meshTopologyTypes).join(', ')}`);
 
     if (metadataClasses.size) {
@@ -344,6 +357,7 @@ export default class I3SConverter {
       return null;
     }
     if (sourceTile.id) {
+      this.tileCountTotal++;
       console.log(`[analyze]: ${sourceTile.id}`); // eslint-disable-line
     }
 
@@ -450,6 +464,14 @@ export default class I3SConverter {
       this.options.maxDepth
     );
 
+    this.layers0!.attributeStorageInfo = this.attributeMetadataInfo.attributeStorageInfo;
+    this.layers0!.fields = this.attributeMetadataInfo.fields;
+    this.layers0!.popupInfo = this.attributeMetadataInfo.popupInfo;
+
+    if (this.attributeMetadataInfo.attributeStorageInfo.length) {
+      this.layers0!.layerType = _3D_OBJECT_LAYER_TYPE;
+    }
+
     this.layers0!.materialDefinitions = this.materialDefinitions;
     // @ts-ignore
     this.layers0.geometryDefinitions = transform(
@@ -596,7 +618,14 @@ export default class I3SConverter {
       return traversalProps;
     }
     if (sourceTile.id) {
-      console.log(`[convert]: ${sourceTile.id}`); // eslint-disable-line
+      // Print the tile number that is currently being converted.
+      this.tileCountCurrentlyConverting++;
+      let percentString = '';
+      if (this.tileCountTotal) {
+        const percent = (this.tileCountCurrentlyConverting / this.tileCountTotal) * 100;
+        percentString = ' ' + percent.toFixed(this.numberOfDigitsInPercentage);
+      }
+      console.log(`[convert${percentString}%]: ${sourceTile.id}`); // eslint-disable-line
     }
 
     const {parentNodes, transform} = traversalProps;
@@ -775,7 +804,7 @@ export default class I3SConverter {
       async () => (await this.nodePages.push({index: 0, obb: draftObb}, parentId)).index,
       propertyTable,
       this.featuresHashArray,
-      this.layers0?.attributeStorageInfo,
+      this.attributeMetadataInfo.attributeStorageInfo,
       this.options.draco,
       this.generateBoundingVolumes,
       this.options.mergeMaterials,
@@ -1081,14 +1110,14 @@ export default class I3SConverter {
     childPath: string,
     slpkChildPath: string
   ): Promise<void> {
-    if (attributes?.length && this.layers0?.attributeStorageInfo?.length) {
+    if (attributes?.length && this.attributeMetadataInfo.attributeStorageInfo.length) {
       const minimumLength =
-        attributes.length < this.layers0.attributeStorageInfo.length
+        attributes.length < this.attributeMetadataInfo.attributeStorageInfo.length
           ? attributes.length
-          : this.layers0.attributeStorageInfo.length;
+          : this.attributeMetadataInfo.attributeStorageInfo.length;
 
       for (let index = 0; index < minimumLength; index++) {
-        const folderName = this.layers0.attributeStorageInfo[index].key;
+        const folderName = this.attributeMetadataInfo.attributeStorageInfo[index].key;
         const fileBuffer = new Uint8Array(attributes[index]);
 
         if (this.options.slpk) {
@@ -1174,67 +1203,19 @@ export default class I3SConverter {
     */
     let attributeTypesMap: Record<string, Attribute> | null = null;
     if (this.options.metadataClass) {
-      if (!this.layers0!.attributeStorageInfo!.length && tileContent?.gltf) {
-        attributeTypesMap = getAttributeTypesFromSchema(
+      if (!this.attributeMetadataInfo.attributeStorageInfo.length && tileContent?.gltf) {
+        attributeTypesMap = getAttributeTypesMapFromSchema(
           tileContent.gltf,
           this.options.metadataClass
         );
       }
     } else if (propertyTable) {
-      attributeTypesMap = getAttributeTypesFromPropertyTable(propertyTable);
+      attributeTypesMap = getAttributeTypesMapFromPropertyTable(propertyTable);
     }
 
     if (attributeTypesMap) {
-      this.createStorageAttributes(attributeTypesMap);
-    }
-  }
-
-  /**
-   * Creates Attribute Storage Info objects based on attribute's types
-   * @param attributeTypesMap - set of attribute's types
-   */
-  private createStorageAttributes(attributeTypesMap: Record<string, Attribute>): void {
-    if (!Object.keys(attributeTypesMap).length) {
-      return;
-    }
-    const attributeTypes: Record<string, Attribute> = {
-      OBJECTID: 'OBJECTID',
-      ...attributeTypesMap
-    };
-
-    let isUpdated = false;
-    let attributeIndex = this.layers0!.attributeStorageInfo!.length;
-    for (const key in attributeTypes) {
-      /*
-      We will append a new attribute only in case it has not been added to the attribute storage info yet.
-      */
-      const elementFound = this.layers0!.attributeStorageInfo!.find(
-        (element) => element.name === key
-      );
-      if (!elementFound) {
-        const attributeType = attributeTypes[key];
-
-        const storageAttribute = createdStorageAttribute(attributeIndex, key, attributeType);
-        const fieldAttributeType = getFieldAttributeType(attributeType);
-        const fieldAttribute = createFieldAttribute(key, fieldAttributeType);
-
-        this.layers0!.attributeStorageInfo!.push(storageAttribute);
-        this.layers0!.fields!.push(fieldAttribute);
-        attributeIndex += 1;
-        isUpdated = true;
-      }
-    }
-    if (isUpdated) {
-      /*
-      The attributeStorageInfo is updated. So, popupInfo should be recreated.
-      Use attributeStorageInfo as a source of attribute names to create the popupInfo.
-    */
-      const attributeNames: string[] = [];
-      for (let info of this.layers0!.attributeStorageInfo!) {
-        attributeNames.push(info.name);
-      }
-      this.layers0!.popupInfo = createPopupInfo(attributeNames);
-      this.layers0!.layerType = _3D_OBJECT_LAYER_TYPE;
+      // Add new storage attributes, fields and create popupInfo
+      this.attributeMetadataInfo.addMetadataInfo(attributeTypesMap);
     }
   }
 

package/src/i3s-converter/types.ts

@@ -225,3 +225,19 @@ export type GLTFAttributesData = {
   /** Model matrix to convert coordinate system of POSITION and NORMAL attributes from METER_OFFSETS to CARTESIAN  */
   cartesianModelMatrix: Matrix4;
 };
+
+/**
+ * I3S' types difine the following:
+ *   type Attribute = 'OBJECTID' | 'string' | 'double' | 'Int32' | string;
+ * The AttributeType contains the string values of the Attribute type.
+ */
+export const AttributeType = {
+  /** Type of attribute that is linked with feature ids */
+  OBJECT_ID_TYPE: 'OBJECTID',
+  /** String data type name for feature attributes */
+  STRING_TYPE: 'string',
+  /** Double data type name for feature attributes */
+  DOUBLE_TYPE: 'double',
+  /** Integer data type name for feature attributes */
+  SHORT_INT_TYPE: 'Int32'
+} as const;

package/src/i3s-server/app.ts

@@ -2,13 +2,18 @@ import express from 'express';
 import path from 'path';
 import logger from 'morgan';
 import cors from 'cors';
+// For local debug
+// import {fileURLToPath} from 'url';
 import {loadArchive} from './controllers/slpk-controller';
+import {router as indexRouter} from './routes';
+import {sceneServerRouter, router} from './routes/slpk-router';
+
+// For local debug
+// const __filename = fileURLToPath(import.meta.url);
+// const __dirname = path.dirname(__filename);
 
 const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef
 const FULL_LAYER_PATH = path.join(process.cwd(), I3S_LAYER_PATH); // eslint-disable-line no-undef
-loadArchive(FULL_LAYER_PATH);
-
-const indexRouter = require('./routes/index');
 
 export const app = express();
 
@@ -19,7 +24,7 @@ app.use(express.static(path.join(__dirname, 'public')));
 app.use(cors());
 
 if (/\.slpk$/.test(I3S_LAYER_PATH)) {
-  const {sceneServerRouter, router} = require('./routes/slpk-router');
+  loadArchive(FULL_LAYER_PATH);
   app.use('/SceneServer/layers/0', router);
   app.use('/SceneServer', sceneServerRouter);
 } else {

package/src/i3s-server/bin/www.ts

@@ -5,8 +5,14 @@ import https from 'https';
 import http from 'http';
 import fs from 'fs';
 import path from 'path';
+// For local debug
+// import {fileURLToPath} from 'url';
 import {formErrorHandler, formListeningHandler, normalizePort} from '../utils/server-utils';
 
+// For local debug
+// const __filename = fileURLToPath(import.meta.url);
+// const __dirname = path.dirname(__filename);
+
 /** Get port from environment and store in Express. */
 const httpPort = normalizePort(process.env.PORT || '80');
 if (httpPort === false) {

package/src/i3s-server/routes/index.ts

@@ -1,7 +1,7 @@
 import express from 'express';
+import {getFileNameByUrl} from '../controllers/index-controller';
 
-const router = express.Router();
-const {getFileNameByUrl} = require('../controllers/index-controller');
+export const router = express.Router();
 
 /* GET home page. */
 router.get('*', async function (req, res, next) {
@@ -13,5 +13,3 @@ router.get('*', async function (req, res, next) {
     res.send('File not found');
   }
 });
-
-module.exports = router;

package/src/i3s-server/routes/slpk-router.ts

@@ -2,11 +2,13 @@ import express from 'express';
 import {getFileByUrl} from '../controllers/slpk-controller';
 import {createSceneServer} from '../utils/create-scene-server';
 
+const textDecoder = new TextDecoder();
+
 export const sceneServerRouter = express.Router();
 sceneServerRouter.get('*', async function (req, res, next) {
   const file = await getFileByUrl('/');
   if (file) {
-    const layer = JSON.parse(file.toString());
+    const layer = JSON.parse(textDecoder.decode(file));
     const sceneServerResponse = createSceneServer(layer.name, layer);
     res.send(sceneServerResponse);
   } else {
@@ -17,10 +19,9 @@ sceneServerRouter.get('*', async function (req, res, next) {
 
 export const router = express.Router();
 router.get('*', async function (req, res, next) {
-  console.log(req.path);
   const file = await getFileByUrl(req.path);
   if (file) {
-    res.send(file);
+    res.send(Buffer.from(file));
   } else {
     res.status(404);
     res.send('File not found');