@loaders.gl/tile-converter 4.0.0-alpha.1 → 4.0.0-alpha.10

package/src/i3s-converter/json-templates/geometry-definitions.ts

@@ -0,0 +1,83 @@
+import transform from 'json-map-transform';
+
+const PLAIN_GEOMETRY_DEFINITION = () => ({
+  offset: {
+    default: 8
+  },
+  position: {
+    default: {
+      type: 'Float32',
+      component: 3
+    }
+  },
+  normal: {
+    default: {
+      type: 'Float32',
+      component: 3
+    }
+  },
+  uv0: {
+    path: 'hasTexture',
+    transform: (val) => (val && {type: 'Float32', component: 2}) || false,
+    omitValues: [false]
+  },
+  color: {
+    default: {
+      type: 'UInt8',
+      component: 4
+    }
+  },
+  uvRegion: {
+    path: 'hasUvRegions',
+    transform: (val) => (val && {type: 'UInt16', component: 4}) || false,
+    omitValues: [false]
+  },
+  featureId: {
+    default: {
+      binding: 'per-feature',
+      type: 'UInt64',
+      component: 1
+    }
+  },
+  faceRange: {
+    default: {
+      binding: 'per-feature',
+      type: 'UInt32',
+      component: 2
+    }
+  }
+});
+
+const COMPRESSED_GEOMETRY_DEFINITION = () => ({
+  'compressedAttributes.encoding': {
+    default: 'draco'
+  },
+  'compressedAttributes.attributes': {
+    path: 'geometryConfig',
+    transform: (val) => {
+      const result = ['position', 'normal'];
+      if (val.hasTexture) {
+        result.push('uv0');
+      }
+      result.push('color');
+      if (val.hasUvRegions) {
+        result.push('uv-region');
+      }
+      result.push('feature-index');
+      return result;
+    }
+  }
+});
+
+export const GEOMETRY_DEFINITION = () => ({
+  geometryBuffers: {
+    path: 'geometryConfig',
+    transform: (val) => {
+      const result = [transform(val, PLAIN_GEOMETRY_DEFINITION())];
+      if (val.draco) {
+        result.push(transform({geometryConfig: val}, COMPRESSED_GEOMETRY_DEFINITION()));
+      }
+      return result;
+    }
+  }
+});

package/src/i3s-converter/json-templates/layers.ts

@@ -0,0 +1,137 @@
+import transform from 'json-map-transform';
+import {STORE} from './store';
+
+const SPATIAL_REFERENCE = () => ({
+  wkid: {
+    path: 'wkid',
+    default: 4326
+  },
+  latestWkid: {
+    path: 'latestWkid',
+    default: 4326
+  },
+  vcsWkid: {
+    path: 'vcsWkid',
+    default: 5773
+  },
+  latestVcsWkid: {
+    path: 'latestVcsWkid',
+    default: 5773
+  }
+});
+
+const HEIGHT_MODEL_INFO = () => ({
+  heightModel: {
+    path: 'heightModel',
+    default: 'gravity_related_height'
+  },
+  vertCRS: {
+    path: 'vertCRS',
+    default: 'EGM96_Geoid'
+  },
+  heightUnit: {
+    path: 'heightUnit',
+    default: 'meter'
+  }
+});
+
+const NODE_PAGES = () => ({
+  nodesPerPage: {
+    path: 'nodesPerPage'
+  },
+  lodSelectionMetricType: {
+    path: 'lodSelectionMetricType',
+    default: 'maxScreenThresholdSQ'
+  }
+});
+
+const FULL_EXTENT = () => ({
+  xmin: {
+    path: 'xmin'
+  },
+  ymin: {
+    path: 'ymin'
+  },
+  xmax: {
+    path: 'xmax'
+  },
+  ymax: {
+    path: 'ymax'
+  },
+  zmin: {
+    path: 'zmin'
+  },
+  zmax: {
+    path: 'zmax'
+  }
+});
+
+export const LAYERS = () => ({
+  version: {
+    path: 'version',
+    transform: (val) => val.toUpperCase()
+  },
+  id: {
+    path: 'id',
+    default: 0
+  },
+  name: {
+    path: 'name'
+  },
+  href: {
+    path: 'href',
+    default: './layers/0'
+  },
+  layerType: {
+    path: 'layerType',
+    default: 'IntegratedMesh'
+  },
+  spatialReference: {
+    path: 'spatialReference',
+    transform: (val) => transform(val, SPATIAL_REFERENCE())
+  },
+  capabilities: {
+    path: 'capabilities',
+    default: ['View', 'Query']
+  },
+  store: {
+    path: 'store',
+    transform: (val) => transform(val, STORE)
+  },
+  fullExtent: {
+    path: 'fullExtent',
+    transform: (val) => transform(val, FULL_EXTENT())
+  },
+  heightModelInfo: {
+    path: 'heightModelInfo',
+    transform: (val) => transform(val, HEIGHT_MODEL_INFO())
+  },
+  nodePages: {
+    path: 'nodePages',
+    transform: (val) => transform(val, NODE_PAGES())
+  },
+  materialDefinitions: {
+    path: 'materialDefinitions',
+    default: []
+  },
+  textureSetDefinitions: {
+    path: 'textureSetDefinitions',
+    default: []
+  },
+  geometryDefinitions: {
+    path: 'geometryDefinitions',
+    default: []
+  },
+  attributeStorageInfo: {
+    path: 'attributeStorageInfo',
+    default: []
+  },
+  fields: {
+    path: 'fields',
+    default: []
+  },
+  popupInfo: {
+    path: 'popupInfo',
+    default: null
+  }
+});

package/src/i3s-converter/json-templates/{metadata.js → metadata.ts}

@@ -1,4 +1,4 @@
-export const METADATA = {
+export const METADATA = () => ({
   folderPattern: {
     path: 'folderPattern',
     default: 'BASIC'
@@ -18,4 +18,4 @@ export const METADATA = {
   nodeCount: {
     path: 'nodeCount'
   }
-};
+});

package/src/i3s-converter/json-templates/{node.js → node.ts}

@@ -1,29 +1,29 @@
 import transform from 'json-map-transform';
 
-const COORDINATES = {
+const COORDINATES = () => ({
   mbs: {
     path: 'mbs'
   },
   obb: {
     path: 'obb'
   }
-};
+});
 
-const _href = {
+const HREF = () => ({
   href: {
     path: 'href'
   }
-};
+});
 
-const PARENT_NODE = {
+const PARENT_NODE = () => ({
   id: {
     path: 'id'
   },
-  ..._href,
-  ...COORDINATES
-};
+  ...HREF(),
+  ...COORDINATES()
+});
 
-export const NODE = {
+export const NODE = () => ({
   version: {
     path: 'version'
   },
@@ -36,7 +36,7 @@ export const NODE = {
   level: {
     path: 'level'
   },
-  ...COORDINATES,
+  ...COORDINATES(),
   lodSelection: {
     path: 'lodSelection',
     default: [
@@ -60,7 +60,7 @@ export const NODE = {
   },
   parentNode: {
     path: 'parentNode',
-    transform: (val) => transform(val, PARENT_NODE),
+    transform: (val) => transform(val, PARENT_NODE()),
     default: null
   },
   sharedResource: {
@@ -83,4 +83,4 @@ export const NODE = {
     path: 'attributeData',
     default: null
   }
-};
+});

package/src/i3s-converter/json-templates/{scene-server.js → scene-server.ts}

@@ -1,4 +1,4 @@
-export const SCENE_SERVER_TEMPLATE = {
+export const SCENE_SERVER = () => ({
   serviceItemId: {
     path: 'serviceItemId'
   },
@@ -24,4 +24,4 @@ export const SCENE_SERVER_TEMPLATE = {
     path: 'layers0',
     transform: (layers0) => [layers0]
   }
-};
+});

package/src/i3s-converter/json-templates/{shared-resources.js → shared-resources.ts}

@@ -1,6 +1,6 @@
 import transform from 'json-map-transform';
 
-const MATERIAL_DEFINITION_INFO_PARAMS = {
+const MATERIAL_DEFINITION_INFO_PARAMS = () => ({
   renderMode: {
     path: 'renderMode',
     default: 'solid'
@@ -15,15 +15,15 @@ const MATERIAL_DEFINITION_INFO_PARAMS = {
   },
   ambient: {
     path: 'ambient',
-    default: [1, 1, 1, 1]
+    default: [1, 1, 1]
   },
   diffuse: {
     path: 'diffuse',
-    default: [1, 1, 1, 1]
+    default: [1, 1, 1]
   },
   specular: {
     path: 'specular',
-    default: [0, 0, 0, 0]
+    default: [0, 0, 0]
   },
   useVertexColorAlpha: {
     path: 'useVertexColorAlpha',
@@ -37,9 +37,9 @@ const MATERIAL_DEFINITION_INFO_PARAMS = {
     path: 'vertexColors',
     default: true
   }
-};
+});
 
-const MATERIAL_DEFINITION_INFO = {
+const MATERIAL_DEFINITION_INFO = () => ({
   name: {
     path: 'name',
     default: 'standard'
@@ -51,11 +51,11 @@ const MATERIAL_DEFINITION_INFO = {
   params: {
     path: 'params',
     transform: (val, thisObject, originalObject) =>
-      transform(originalObject, MATERIAL_DEFINITION_INFO_PARAMS)
+      transform(originalObject, MATERIAL_DEFINITION_INFO_PARAMS())
   }
-};
+});
 
-const TEXTURE_DEFINITION_IMAGE = {
+const TEXTURE_DEFINITION_IMAGE = () => ({
   id: {
     path: 'id'
   },
@@ -69,9 +69,9 @@ const TEXTURE_DEFINITION_IMAGE = {
   length: {
     path: 'length'
   }
-};
+});
 
-const TEXTURE_DEFINITION_INFO = {
+const TEXTURE_DEFINITION_INFO = () => ({
   encoding: {
     path: 'encoding'
   },
@@ -94,11 +94,11 @@ const TEXTURE_DEFINITION_INFO = {
   images: {
     path: 'images',
     transform: (val, thisObject, originalObject) =>
-      val.map((image) => transform(image, TEXTURE_DEFINITION_IMAGE))
+      val.map((image) => transform(image, TEXTURE_DEFINITION_IMAGE()))
   }
-};
+});
 
-export const SHARED_RESOURCES_TEMPLATE = {
+export const SHARED_RESOURCES = () => ({
   materialDefinitions: {
     path: 'materialDefinitionInfos',
     transform: transfromMaterialDefinitions
@@ -107,14 +107,14 @@ export const SHARED_RESOURCES_TEMPLATE = {
     path: 'textureDefinitionInfos',
     transform: transfromTextureDefinitions
   }
-};
+});
 
 function transfromMaterialDefinitions(materialDefinitionInfos, thisObject, originalObject) {
   const result = {};
   for (const [index, materialDefinitionInfo] of materialDefinitionInfos.entries()) {
     result[`Mat${originalObject.nodePath}${index}`] = transform(
       materialDefinitionInfo,
-      MATERIAL_DEFINITION_INFO
+      MATERIAL_DEFINITION_INFO()
     );
   }
   return result;
@@ -128,7 +128,7 @@ function transfromTextureDefinitions(textureDefinitionInfos, thisObject, origina
   for (const [index, textureDefinitionInfo] of textureDefinitionInfos.entries()) {
     const imageIndex = `${originalObject.nodePath}${index}`;
     textureDefinitionInfo.imageIndex = imageIndex;
-    result[imageIndex] = transform(textureDefinitionInfo, TEXTURE_DEFINITION_INFO);
+    result[imageIndex] = transform(textureDefinitionInfo, TEXTURE_DEFINITION_INFO());
   }
   return result;
 }

package/src/i3s-converter/types.ts

@@ -0,0 +1,185 @@
+import {GLTFImagePostprocessed} from '@loaders.gl/gltf';
+import {
+  BoundingVolumes,
+  I3SMaterialDefinition,
+  MaterialDefinitionInfo,
+  TextureDefinitionInfo
+} from '@loaders.gl/i3s';
+
+/** Converted resources for specific node */
+export type I3SConvertedResources = {
+  /**
+   * Node id
+   */
+  nodeId?: number;
+  /** Non-compressed geometry buffer that have structure met
+   * https://github.com/Esri/i3s-spec/blob/master/docs/1.8/defaultGeometrySchema.cmn.md
+   * (Geometry buffer)
+   */
+  geometry: ArrayBuffer | null;
+  /**
+   * Draco compressed geometry
+   */
+  compressedGeometry?: Promise<ArrayBuffer> | null;
+  /**
+   * Texture image content
+   */
+  texture: any | null;
+  /**
+   * If the resource has uvRegions geometry attribute
+   */
+  hasUvRegions: boolean;
+  /**
+   * Shared resources built from GLTF material
+   */
+  sharedResources: SharedResourcesArrays | null;
+  /**
+   * Material definition of the node
+   */
+  meshMaterial?: I3SMaterialDefinition | null;
+  /**
+   * Number of vertices in the node
+   */
+  vertexCount: number | null;
+  /**
+   * Feature attributes contents
+   */
+  attributes: ArrayBuffer[] | null;
+  /**
+   * Number of features in the node
+   */
+  featureCount: number | null;
+  /**
+   * MBS and/or OBB bounding volumes of the node
+   */
+  boundingVolumes: BoundingVolumes | null;
+};
+
+/**
+ * Geometry and feature attributes converted from GLTF primitives
+ */
+export type ConvertedAttributes = {
+  /** POSITION attribute value */
+  positions: Float32Array;
+  /** NORMAL attribute value */
+  normals: Float32Array;
+  /** TEXCOORD_0 attribute value */
+  texCoords: Float32Array;
+  /** COLOR_0 attribute value */
+  colors: Uint8Array;
+  /** uvRegion attribute for a texture atlas */
+  uvRegions: Uint16Array;
+  /** Feature indices grouped by ...
+   * converted from "batch ids" of GLTF
+   */
+  featureIndicesGroups?: number[][];
+  /** Feature indices converted from "batch ids" */
+  featureIndices: number[];
+  /**
+   * MBS and/or OBB bounding volumes of the node
+   */
+  boundingVolumes: null | BoundingVolumes;
+  /** merged materials data */
+  mergedMaterials: MergedMaterial[];
+};
+
+/** Postprocessed geometry and feature attributes
+ * https://github.com/Esri/i3s-spec/blob/master/docs/1.8/defaultGeometrySchema.cmn.md
+ */
+export type GeometryAttributes = {
+  /** POSITION attribute value */
+  positions: Float32Array;
+  /** NORMAL attribute value */
+  normals: Float32Array;
+  /** TEXCOORD_0 attribute value */
+  texCoords: Float32Array;
+  /** COLOR_0 attribute value */
+  colors: Uint8Array;
+  /** uvRegion attribute for a texture atlas */
+  uvRegions: Uint16Array;
+  /** faceRanges attribute value  */
+  faceRange: Uint32Array;
+  /** feature Ids attribute value */
+  featureIds: number[];
+  /** number of features in the node */
+  featureCount: number;
+};
+
+/** Geometry attributes specific for the particular feature */
+export type GroupedByFeatureIdAttributes = {
+  /** Feature Id */
+  featureId: number;
+  /** POSITION attribute value */
+  positions: Float32Array;
+  /** NORMAL attribute value */
+  normals: Float32Array;
+  /** COLOR_0 attribute value */
+  colors: Uint8Array;
+  /** uvRegion attribute for a texture atlas */
+  uvRegions: Uint16Array;
+  /** TEXCOORD_0 attribute value */
+  texCoords: Float32Array;
+};
+
+/** Shared resources made from GLTF material */
+export type SharedResourcesArrays = {
+  /** material definitions list https://github.com/Esri/i3s-spec/blob/master/docs/1.8/materialDefinitionInfo.cmn.md */
+  materialDefinitionInfos?: MaterialDefinitionInfo[];
+  /** texture definitions list  https://github.com/Esri/i3s-spec/blob/master/docs/1.8/textureDefinitionInfo.cmn.md*/
+  textureDefinitionInfos?: TextureDefinitionInfo[];
+  /** node id to make unique SharedResource ids */
+  nodePath?: string;
+};
+
+/** I3S material definition and texture content taken from GLTF material */
+export type I3SMaterialWithTexture = {
+  /** Material definition https://github.com/Esri/i3s-spec/blob/master/docs/1.8/materialDefinitions.cmn.md */
+  material: I3SMaterialDefinition;
+  /** Texture content (image) */
+  texture?: GLTFImagePostprocessed;
+  /** Metadata of all merged materials */
+  mergedMaterials: MergedMaterial[];
+};
+
+/** Metadata of some original texture */
+export type MergedMaterial = {
+  /** Gltf material Id */
+  originalMaterialId: string;
+  /** Original texture size */
+  textureSize?: {
+    width: number;
+    height: number;
+  };
+  /** Uint16Array of 4 elements https://github.com/Esri/i3s-spec/blob/master/docs/1.7/geometryUVRegion.cmn.md */
+  uvRegion?: Uint16Array;
+};
+
+export type TypedArrayConstructor =
+  | Uint8ArrayConstructor
+  | Int8ArrayConstructor
+  | Uint16ArrayConstructor
+  | Int16ArrayConstructor
+  | Int32ArrayConstructor
+  | Uint32ArrayConstructor
+  | Float32ArrayConstructor
+  | Float64ArrayConstructor;
+
+/**
+ * glTF primitive modes (mesh topology types)
+ * @see https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#_mesh_primitive_mode
+ */
+export enum GltfPrimitiveModeString {
+  POINTS = 'POINTS',
+  LINES = 'LINES',
+  LINE_LOOP = 'LINE_LOOP',
+  LINE_STRIP = 'LINE_STRIP',
+  TRIANGLES = 'TRIANGLES',
+  TRIANGLE_STRIP = 'TRIANGLE_STRIP',
+  TRIANGLE_FAN = 'TRIANGLE_FAN'
+}
+
+/** Preprocessed data gathered from child tiles binary content */
+export type PreprocessData = {
+  /** Mesh topology types used in gltf primitives of the tileset */
+  meshTopologyTypes: Set<GltfPrimitiveModeString>;
+};

package/src/i3s-server/README.md

@@ -0,0 +1,19 @@
+# I3S server
+
+The http server run on NodeJS ExpressJS framework.
+The server provides I3S Rest endpoints per specification https://github.com/Esri/i3s-spec/blob/master/docs/1.7/3Dobject_ReadMe.md#http-api-overview-17
+
+## Usage
+
+### Serve 3DTiles to I3S converted dataset
+
+- Convert data set from 3DTiles to I3S without `--slpk` option
+- Serve output folder `I3sLayerPath="./data/BatchedTextured" DEBUG=i3s-server:* npx i3s-server`
+
+### Serve SLPK
+
+- Serve slpk file `I3sLayerPath="../datasets/Rancho_Mesh_mesh_v17_1.slpk" DEBUG=i3s-server:* npx i3s-server`
+
+## ENV variables
+
+- `I3sLayerPath` - path to converted data or SLPK file.

package/src/i3s-server/app.js

@@ -4,7 +4,9 @@ const logger = require('morgan');
 const cors = require('cors');
 
 const indexRouter = require('./routes/index');
+const {sceneServerRouter, router} = require('./routes/slpk-router');
 
+const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef
 const app = express();
 
 app.use(logger('dev'));
@@ -13,6 +15,11 @@ app.use(express.urlencoded({extended: false}));
 app.use(express.static(path.join(__dirname, 'public')));
 app.use(cors());
 
-app.use('/', indexRouter);
+if (/\.slpk$/.test(I3S_LAYER_PATH)) {
+  app.use('/SceneServer/layers/0', router);
+  app.use('/SceneServer', sceneServerRouter);
+} else {
+  app.use('/', indexRouter);
+}
 
 module.exports = app;

package/src/i3s-server/controllers/slpk-controller.js

@@ -0,0 +1,38 @@
+require('@loaders.gl/polyfills');
+const {fetchFile, parse} = require('@loaders.gl/core');
+const {SLPKLoader} = require('@loaders.gl/i3s');
+const path = require('path');
+
+let slpkArchive;
+
+const loadArchive = async (fullLayerPath) => {
+  slpkArchive = await (await fetchFile(fullLayerPath)).arrayBuffer();
+};
+
+const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef
+const FULL_LAYER_PATH = path.join(process.cwd(), I3S_LAYER_PATH); // eslint-disable-line no-undef
+
+loadArchive(FULL_LAYER_PATH);
+
+async function getFileByUrl(url) {
+  const trimmedPath = /^\/?(.*)\/?$/.exec(url);
+  let uncompressedFile;
+  if (trimmedPath) {
+    try {
+      uncompressedFile = Buffer.from(
+        await parse(slpkArchive, SLPKLoader, {
+          slpk: {
+            path: trimmedPath[1],
+            pathMode: 'http'
+          }
+        })
+      );
+    } catch (e) {}
+  }
+  return uncompressedFile;
+}
+
+module.exports = {
+  loadArchive,
+  getFileByUrl
+};