@loaders.gl/tile-converter 4.0.0-alpha.9 → 4.0.0-beta.1

package/src/i3s-converter/types.ts

@@ -1,10 +1,11 @@
-import {GLTFImagePostprocessed} from '@loaders.gl/gltf';
+import {GLTFImagePostprocessed, GLTFNodePostprocessed} from '@loaders.gl/gltf';
 import {
   BoundingVolumes,
   I3SMaterialDefinition,
   MaterialDefinitionInfo,
   TextureDefinitionInfo
 } from '@loaders.gl/i3s';
+import {Matrix4, Vector3} from '@math.gl/core';
 
 /** Converted resources for specific node */
 export type I3SConvertedResources = {
@@ -105,10 +106,8 @@ export type GeometryAttributes = {
   featureCount: number;
 };
 
-/** Geometry attributes specific for the particular feature */
-export type GroupedByFeatureIdAttributes = {
-  /** Feature Id */
-  featureId: number;
+/** Geometry attributes applicable for reordering by featureId */
+export type GroupedAttributes = {
   /** POSITION attribute value */
   positions: Float32Array;
   /** NORMAL attribute value */
@@ -121,6 +120,12 @@ export type GroupedByFeatureIdAttributes = {
   texCoords: Float32Array;
 };
 
+/** Geometry attributes specific for the particular feature */
+export type GroupedByFeatureIdAttributes = GroupedAttributes & {
+  /** Feature Id */
+  featureId: number;
+};
+
 /** Shared resources made from GLTF material */
 export type SharedResourcesArrays = {
   /** material definitions list https://github.com/Esri/i3s-spec/blob/master/docs/1.8/materialDefinitionInfo.cmn.md */
@@ -168,7 +173,7 @@ export type TypedArrayConstructor =
  * glTF primitive modes (mesh topology types)
  * @see https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#_mesh_primitive_mode
  */
-export enum GltfPrimitiveModeString {
+export enum GLTFPrimitiveModeString {
   POINTS = 'POINTS',
   LINES = 'LINES',
   LINE_LOOP = 'LINE_LOOP',
@@ -181,5 +186,42 @@ export enum GltfPrimitiveModeString {
 /** Preprocessed data gathered from child tiles binary content */
 export type PreprocessData = {
   /** Mesh topology types used in gltf primitives of the tileset */
-  meshTopologyTypes: Set<GltfPrimitiveModeString>;
+  meshTopologyTypes: Set<GLTFPrimitiveModeString>;
+  /**
+   * Featrue metadata classes found in glTF extensions
+   * The tileset might contain multiple metadata classes provided by EXT_feature_metadata and EXT_structural_metadata extensions.
+   * Every class is a set of properties. But I3S can consume only one set of properties.
+   * On the pre-process we collect all classes from the tileset in order to show the prompt to select one class for conversion to I3S.
+   */
+  metadataClasses: Set<string>;
+};
+
+/** Texture image properties required for conversion */
+export type TextureImageProperties = {
+  /** Array with image data */
+  data: Uint8Array;
+  /** Is the texture compressed */
+  compressed?: boolean;
+  /** Height of the texture's image */
+  height?: number;
+  /** Width of the texture's image */
+  width?: number;
+  /** Number of components (3 for RGB, 4 for RGBA) */
+  components?: number;
+  /** Mime type of the texture's image */
+  mimeType?: string;
+};
+
+/** glTF attributes data, prepared for conversion */
+export type GLTFAttributesData = {
+  /** glTF PBR materials (only id is required) */
+  gltfMaterials?: {id: string}[];
+  /** glTF geometry nodes */
+  nodes: GLTFNodePostprocessed[];
+  /** glTF texture images (set to null for compressed textures) */
+  images: (null | TextureImageProperties)[];
+  /** Source tile origin coordinates in cartographic coordinate system */
+  cartographicOrigin: Vector3;
+  /** Model matrix to convert coordinate system of POSITION and NORMAL attributes from METER_OFFSETS to CARTESIAN  */
+  cartesianModelMatrix: Matrix4;
 };

package/src/i3s-server/README.md

@@ -8,12 +8,56 @@ The server provides I3S Rest endpoints per specification https://github.com/Esri
 ### Serve 3DTiles to I3S converted dataset
 
 - Convert data set from 3DTiles to I3S without `--slpk` option
-- Serve output folder `I3sLayerPath="./data/BatchedTextured" DEBUG=i3s-server:* npx i3s-server`
+- Serve output folder
+
+Example for path `./data/BatchTextured/SceneServer/layers/0/...`:
+
+#### Start the server
+
+```bash
+PORT=8080 HTTPS_PORT=4443 I3sLayerPath="./data" DEBUG=i3s-server:* npx i3s-server
+```
+#### Check the layer availability
+
+The layer should be available on URLs:
+
+- `http://localhost:8080/BatchTextured/SceneServer/layers/0/...`
+- `https://localhost:4443/BatchTextured/SceneServer/layers/0/...`
+
+#### Open in ArcGIS
+
+`https://www.arcgis.com/home/webscene/viewer.html?url=http://localhost:8080/BatchTextured/SceneServer/layers/0/`
+
+#### Open in I3S Explorer
+
+`https://i3s.loaders.gl/viewer?tileset=http://localhost:8080/BatchTextured/SceneServer/layers/0`
 
 ### Serve SLPK
 
-- Serve slpk file `I3sLayerPath="../datasets/Rancho_Mesh_mesh_v17_1.slpk" DEBUG=i3s-server:* npx i3s-server`
+Example for path `../datasets/Rancho_Mesh_mesh_v17_1.slpk`:
+
+#### Start the server
+
+```bash
+PORT=8080 HTTPS_PORT=4443 I3sLayerPath="../datasets/Rancho_Mesh_mesh_v17_1.slpk" DEBUG=i3s-server:* npx i3s-server
+```
+#### Check the layer availability
+
+The layer should be available on URLs:
+
+- `http://localhost:8080/SceneServer/layers/0/...`
+- `https://localhost:4443/SceneServer/layers/0/...`
+
+#### Open in ArcGIS
+
+`https://www.arcgis.com/home/webscene/viewer.html?url=http://localhost:8080/SceneServer`
+
+#### Open in I3S Explorer
+
+`https://i3s.loaders.gl/viewer?tileset=http://localhost:8080/SceneServer/layers/0`
 
 ## ENV variables
 
 - `I3sLayerPath` - path to converted data or SLPK file.
+- `PORT` - HTTP port. Eg for `PORT = 8080 npx i3s-server` the server will work on host `http://localhost:8080/...`. Default value is `80`;
+- `HTTPS_PORT` - HTTPS port. Eg for `PORT = 4443 npx i3s-server` the server will work on host `https://localhost:4443/...`. Default value is `443`

package/src/i3s-server/{app.js → app.ts}

@@ -1,13 +1,16 @@
-const express = require('express');
-const path = require('path');
-const logger = require('morgan');
-const cors = require('cors');
+import express from 'express';
+import path from 'path';
+import logger from 'morgan';
+import cors from 'cors';
+import {loadArchive} from './controllers/slpk-controller';
+
+const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef
+const FULL_LAYER_PATH = path.join(process.cwd(), I3S_LAYER_PATH); // eslint-disable-line no-undef
+loadArchive(FULL_LAYER_PATH);
 
 const indexRouter = require('./routes/index');
-const {sceneServerRouter, router} = require('./routes/slpk-router');
 
-const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef
-const app = express();
+export const app = express();
 
 app.use(logger('dev'));
 app.use(express.json());
@@ -16,10 +19,9 @@ app.use(express.static(path.join(__dirname, 'public')));
 app.use(cors());
 
 if (/\.slpk$/.test(I3S_LAYER_PATH)) {
+  const {sceneServerRouter, router} = require('./routes/slpk-router');
   app.use('/SceneServer/layers/0', router);
   app.use('/SceneServer', sceneServerRouter);
 } else {
   app.use('/', indexRouter);
 }
-
-module.exports = app;

package/src/i3s-server/bin/www.ts

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+
+import {app} from '../app';
+import https from 'https';
+import http from 'http';
+import fs from 'fs';
+import path from 'path';
+import {formErrorHandler, formListeningHandler, normalizePort} from '../utils/server-utils';
+
+/** Get port from environment and store in Express. */
+const httpPort = normalizePort(process.env.PORT || '80');
+if (httpPort === false) {
+  console.error(`Incorrect HTTP port`);
+  process.exit(1);
+}
+const httpsPort = normalizePort(process.env.HTTPS_PORT || '443');
+if (httpsPort === false) {
+  console.error(`Incorrect HTTPs port`);
+  process.exit(1);
+}
+
+/** Create HTTP server. */
+const options = {
+  key: fs.readFileSync(path.join(__dirname, '../certs/key.pem')),
+  cert: fs.readFileSync(path.join(__dirname, '../certs/cert.pem'))
+};
+
+const httpServer = http.createServer(app);
+const httpsServer = https.createServer(options, app);
+
+/** Listen on provided port, on all network interfaces. */
+httpServer.listen(httpPort);
+httpServer.on('error', formErrorHandler(httpPort));
+httpServer.on('listening', formListeningHandler(httpServer));
+
+httpsServer.listen(httpsPort);
+httpsServer.on('error', formErrorHandler(httpsPort));
+httpsServer.on('listening', formListeningHandler(httpsServer));

package/src/i3s-server/controllers/{index-controller.js → index-controller.ts}

@@ -1,13 +1,18 @@
-const path = require('path');
-const fs = require('fs');
+import path from 'path';
+import fs from 'fs';
 
 const {promises} = fs;
 
-const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef
-const FULL_LAYER_PATH = path.join(process.cwd(), I3S_LAYER_PATH); // eslint-disable-line no-undef
+const I3S_LAYER_PATH = process.env.I3sLayerPath || '';
 
-async function getFileNameByUrl(url) {
+/**
+ * Get local file name by input HTTP URL
+ * @param url - I3S HTTP service url
+ * @returns - local file name
+ */
+export async function getFileNameByUrl(url: string): Promise<string | null> {
   const extensions = ['json', 'bin', 'jpg', 'jpeg', 'png', 'bin.dds', 'ktx2'];
+  const FULL_LAYER_PATH = path.join(process.cwd(), I3S_LAYER_PATH);
   for (const ext of extensions) {
     const fileName = `${FULL_LAYER_PATH}${url}/index.${ext}`;
     try {
@@ -19,7 +24,3 @@ async function getFileNameByUrl(url) {
   }
   return null;
 }
-
-module.exports = {
-  getFileNameByUrl
-};

package/src/i3s-server/controllers/slpk-controller.ts

@@ -0,0 +1,32 @@
+import '@loaders.gl/polyfills';
+import {parseSLPK} from '@loaders.gl/i3s';
+import {FileHandleFile} from '@loaders.gl/loader-utils';
+
+let slpkArchive;
+
+/**
+ * Open SLPK file for reading and load HASH file
+ * @param fullLayerPath - full path to SLPK file
+ */
+export const loadArchive = async (fullLayerPath: string): Promise<void> => {
+  slpkArchive = await parseSLPK(await FileHandleFile.from(fullLayerPath), (msg) =>
+    console.log(msg)
+  );
+  console.log('The server is ready to use');
+};
+
+/**
+ * Get a file from SLPK
+ * @param url - I3S HTTP URL
+ * @returns  - file content
+ */
+export async function getFileByUrl(url: string) {
+  const trimmedPath = /^\/?(.*)\/?$/.exec(url);
+  let uncompressedFile: Buffer | null = null;
+  if (trimmedPath) {
+    try {
+      uncompressedFile = Buffer.from(await slpkArchive.getFile(trimmedPath[1], 'http'));
+    } catch (e) {}
+  }
+  return uncompressedFile;
+}

package/src/i3s-server/routes/{index.js → index.ts}

@@ -1,4 +1,5 @@
-const express = require('express');
+import express from 'express';
+
 const router = express.Router();
 const {getFileNameByUrl} = require('../controllers/index-controller');
 

package/src/i3s-server/routes/{slpk-router.js → slpk-router.ts}

@@ -1,8 +1,8 @@
-const express = require('express');
-const {getFileByUrl} = require('../controllers/slpk-controller');
-const createSceneServer = require('../utils/create-scene-server');
+import express from 'express';
+import {getFileByUrl} from '../controllers/slpk-controller';
+import {createSceneServer} from '../utils/create-scene-server';
 
-const sceneServerRouter = express.Router();
+export const sceneServerRouter = express.Router();
 sceneServerRouter.get('*', async function (req, res, next) {
   const file = await getFileByUrl('/');
   if (file) {
@@ -15,7 +15,7 @@ sceneServerRouter.get('*', async function (req, res, next) {
   }
 });
 
-const router = express.Router();
+export const router = express.Router();
 router.get('*', async function (req, res, next) {
   console.log(req.path);
   const file = await getFileByUrl(req.path);
@@ -26,8 +26,3 @@ router.get('*', async function (req, res, next) {
     res.send('File not found');
   }
 });
-
-module.exports = {
-  sceneServerRouter,
-  router
-};

package/src/i3s-server/utils/create-scene-server.ts

@@ -0,0 +1,20 @@
+import {SceneLayer3D} from '@loaders.gl/i3s';
+import {v4 as uuidv4} from 'uuid';
+
+/**
+ * Create `/SceneServer` response
+ * @param name - service name, custom user-friendly name of the service
+ * @param layer - I3S layer JSON
+ * @returns reponse JSON for `/SceneServer` route
+ */
+export const createSceneServer = (name: string, layer: SceneLayer3D) => {
+  return {
+    serviceItemId: uuidv4().replace(/-/gi, ''),
+    serviceName: name,
+    name,
+    currentVersion: '10.7',
+    serviceVersion: '1.8',
+    supportedBindings: ['REST'],
+    layers: [layer]
+  };
+};

package/src/i3s-server/utils/server-utils.ts

@@ -0,0 +1,70 @@
+import type {Server as HttpsServer} from 'https';
+import type {Server as HttpServer} from 'http';
+
+import debugFactory from 'debug';
+const debug = debugFactory('i3s-server:server');
+
+/**
+ * Normalize a port into a number, string, or false.
+ * @param val - port value from env variables
+ * @returns - `number` for port, `string` for a named pipe, or `false` if the port number is not correct
+ */
+export function normalizePort(val: string): number | string | false {
+  const chkPort = parseInt(val, 10);
+
+  if (Number.isNaN(chkPort)) {
+    // named pipe
+    return val;
+  }
+
+  if (chkPort >= 0) {
+    // port number
+    return chkPort;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener creator for HTTP/HTTPS server "error" event.
+ * @param optionalPort - the port/named pipe the server is started on
+ * @return callback to handle server errors
+ */
+export function formErrorHandler(
+  optionalPort: string | number
+): (error: NodeJS.ErrnoException) => void {
+  return function onError(error: NodeJS.ErrnoException) {
+    if (error.syscall !== 'listen') {
+      throw error;
+    }
+
+    const bind = typeof optionalPort === 'string' ? `Pipe ${optionalPort}` : `Port ${optionalPort}`;
+
+    // handle specific listen errors with friendly messages
+    switch (error.code) {
+      case 'EACCES':
+        console.error(`${bind} requires elevated privileges`); // eslint-disable-line no-console, no-undef
+        process.exit(1); // eslint-disable-line no-process-exit, no-undef
+        break;
+      case 'EADDRINUSE':
+        console.error(`${bind} is already in use`); // eslint-disable-line no-console, no-undef
+        process.exit(1); // eslint-disable-line no-process-exit, no-undef
+        break;
+      default:
+        throw error;
+    }
+  };
+}
+
+/**
+ * Event listener for HTTP/HTTPS server "listening" event.
+ * @param optionalServer - http or https NodeJS server
+ * @return callback that is triggered when the server has started
+ */
+export function formListeningHandler(optionalServer: HttpsServer | HttpServer): () => void {
+  return function onListening() {
+    const addr = optionalServer.address();
+    const bind = typeof addr === 'string' ? `pipe ${addr}` : `port ${addr?.port}`;
+    debug(`Listening on ${bind}`);
+  };
+}

package/src/lib/utils/cli-utils.ts

@@ -0,0 +1,78 @@
+/**
+ * Get string option value from cli arguments
+ * @param index - option's name index in the argument's array.
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - string value of the option
+ */
+export function getStringValue(index: number, args: string[]): string {
+  if (index + 1 >= args.length) {
+    return '';
+  }
+  const value = args[index + 1];
+  if (value.indexOf('--') === 0) {
+    return '';
+  }
+  return value;
+}
+
+/**
+ * Modyfy URL path to be compatible with fetch
+ * @param index - option's name index in the argument's array.
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - string value of the option
+ */
+export function getURLValue(index: number, args: string[]): string {
+  const value = getStringValue(index, args);
+  console.log(`Input tileset value: ${value}`);
+  console.log(`Modified tileset value: ${value.replace(/\\/g, '/')}`);
+  return value.replace(/\\/g, '/');
+}
+
+export function validateOptionsWithEqual(args: string[]): string[] {
+  return args.reduce((acc: string[], curr) => {
+    const equalSignIndex = curr.indexOf('=');
+    const beforeEqual = curr.slice(0, equalSignIndex);
+    const afterEqual = curr.slice(equalSignIndex + 1, curr.length);
+    const condition = curr.includes('=') && curr.startsWith('--') && afterEqual;
+    if (condition) {
+      return acc.concat(beforeEqual, afterEqual);
+    }
+    return acc.concat(curr);
+  }, []);
+}
+
+/**
+ * Get integer option value from cli arguments
+ * @param index - option's name index in the argument's array
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - number value of the option
+ */
+export function getIntegerValue(index: number, args: string[]): number {
+  const stringValue: string = getStringValue(index, args);
+  const result: number = Number.parseInt(stringValue);
+  if (isFinite(result)) {
+    return result;
+  }
+  return NaN;
+}
+
+/**
+ * Get boolean option value from cli arguments
+ * @param index - option's name index in the argument's array
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - boolean value of the option
+ */
+export function getBooleanValue(index: number, args: string[]): boolean {
+  const stringValue: string = getStringValue(index, args).toLowerCase().trim();
+  if (['--no-draco', '--split-nodes'].includes(args[index]) && !stringValue) {
+    return false;
+  }
+  if (!stringValue || stringValue === 'true') {
+    return true;
+  }
+  return false;
+}

package/src/lib/utils/compress-util.ts

@@ -206,7 +206,7 @@ export async function generateHash128FromZip(inputZipFile: string, outputFile: s
     const content = zipEntry[_data].compressedContent;
     if (zipEntry.dir) continue; // eslint-disable-line no-continue
     // eslint-disable-next-line no-undef
-    const hash = await new MD5Hash().hash(Buffer.from(relativePath.toLowerCase()));
+    const hash = await new MD5Hash().hash(Buffer.from(relativePath.toLowerCase()), 'base64');
     // eslint-disable-next-line no-undef
     hashTable.push({key: atob(hash), value: content.byteOffset});
   }

package/src/lib/utils/lod-conversion-utils.ts

@@ -1,6 +1,6 @@
 import {Tiles3DTileJSONPostprocessed} from '@loaders.gl/3d-tiles';
 import {BoundingVolumes} from '@loaders.gl/i3s';
-import {Tile3D} from '@loaders.gl/tiles';
+import {I3STileHeader} from '@loaders.gl/i3s/src/types';
 
 // https://cesium.com/docs/cesiumjs-ref-doc/Cesium3DTileset.html
 const DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR = 16;
@@ -64,11 +64,15 @@ export function convertGeometricErrorToScreenThreshold(
  * @param node - i3s node data
  * @returns lod metric in 3d-tiles format
  */
-export function convertScreenThresholdToGeometricError(node: Tile3D): number {
-  const metricData = node.header.lodSelection.maxScreenThreshold || {};
-  let maxError = metricData.maxError;
+export function convertScreenThresholdToGeometricError(node: I3STileHeader): number {
+  const metricData = node.lodSelection?.find(
+    (metric) => metric.metricType === 'maxScreenThreshold'
+  );
+  let maxError = metricData?.maxError;
   if (!maxError) {
-    const sqMetricData = node.header.lodSelection.maxScreenThresholdSQ;
+    const sqMetricData = node.lodSelection?.find(
+      (metric) => metric.metricType === 'maxScreenThresholdSQ'
+    );
     if (sqMetricData) {
       maxError = Math.sqrt(sqMetricData.maxError / (Math.PI * 0.25));
     }
@@ -78,5 +82,5 @@ export function convertScreenThresholdToGeometricError(node: Tile3D): number {
     maxError = DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR;
   }
 
-  return (node.header.mbs[3] * 2 * DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR) / maxError;
+  return (node.mbs[3] * 2 * DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR) / maxError;
 }

package/src/pgm-loader.ts

@@ -1,24 +1,32 @@
-import type {LoaderWithParser} from '@loaders.gl/loader-utils';
-import {parsePGM} from '@math.gl/geoid';
+import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import {Geoid, parsePGM} from '@math.gl/geoid';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
+export {Geoid};
+
+export type PGMLoaderOptions = LoaderOptions & {
+  pgm?: {
+    cubic?: boolean;
+  };
+};
+
 /**
  * Loader for PGM - Netpbm grayscale image format
  */
-export const PGMLoader: LoaderWithParser = {
+export const PGMLoader: LoaderWithParser<Geoid, never, PGMLoaderOptions> = {
   name: 'PGM - Netpbm grayscale image format',
   id: 'pgm',
   module: 'tile-converter',
   version: VERSION,
   mimeTypes: ['image/x-portable-graymap'],
-  // @ts-expect-error LoaderOptions does not have cubic parameter
-  parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options),
+  parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options?.pgm || {}),
   extensions: ['pgm'],
   options: {
-    // TODO - use pgm namespace
-    cubic: false
+    pgm: {
+      cubic: false
+    }
   }
 };