@loaders.gl/tile-converter 5.0.0-alpha.11 → 5.0.0-alpha.14

package/src/v4/i3s-converter/helpers/preprocess-3d-tiles.ts

@@ -1,5 +1,5 @@
 /* eslint-disable camelcase */
-import {Tiles3DTileContent} from '@loaders.gl/3d-tiles';
+import {Tiles3DTileContent} from '@loaders.gl/tiles-3d/3d-tiles';
 import {GLTFPrimitiveModeString, PreprocessData} from '../types';
 import {
   EXT_STRUCTURAL_METADATA,
@@ -43,7 +43,7 @@ export const analyzeTileContent = async (
   }
 
   const gltfData = await parse(tileContent.gltfArrayBuffer, GLTFLoader, {
-    gltf: {normalize: false, loadBuffers: false, loadImages: false, decompressMeshes: false}
+    gltf: {normalize: true, loadBuffers: false, loadImages: false, decompressMeshes: false}
   });
   const gltf = gltfData.json;
 

package/src/v4/i3s-converter/helpers/tileset-traversal.ts

@@ -1,4 +1,4 @@
-import {Tiles3DTileJSONPostprocessed} from '@loaders.gl/3d-tiles';
+import {Tiles3DTileJSONPostprocessed} from '@loaders.gl/tiles-3d/3d-tiles';
 import {NodeIndexDocument} from './node-index-document';
 import {Matrix4} from '@math.gl/core';
 

package/src/v4/i3s-converter/i3s-converter.ts

@@ -1,3 +1,4 @@
+import type {ConversionImage} from './helpers/conversion-gltf';
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
@@ -10,7 +11,7 @@ import type {
   Tiles3DTileContent,
   Tiles3DTileJSONPostprocessed,
   Tiles3DTilesetJSONPostprocessed
-} from '@loaders.gl/3d-tiles';
+} from '@loaders.gl/tiles-3d/3d-tiles';
 import type {WriteQueueItem} from '../lib/utils/write-queue';
 import type {
   SceneLayer3D,
@@ -19,15 +20,14 @@ import type {
   NodeInPage,
   Attribute,
   Node3DIndexDocument
-} from '@loaders.gl/i3s';
+} from '@loaders.gl/tiles-3d/i3s';
 import {load, encode, isBrowser} from '@loaders.gl/core';
-import {CesiumIonLoader, Tiles3DLoader} from '@loaders.gl/3d-tiles';
-import {Geoid} from '@math.gl/geoid';
+import {CesiumIonLoader, Tiles3DLoader} from '@loaders.gl/tiles-3d/3d-tiles';
+import {Geoid} from '@math.gl/earth/geoid';
 import {join} from 'path';
-import {v4 as uuidv4} from 'uuid';
+import {randomUUID} from 'node:crypto';
 import process from 'process';
-import transform from 'json-map-transform';
-import md5 from 'md5';
+import {hashString} from '../lib/utils/hash-string';
 
 import NodePages from './helpers/node-pages';
 import {writeFile, removeDir, writeFileForSlpk, removeFile} from '../lib/utils/file-utils';
@@ -42,15 +42,14 @@ import {createSceneServerPath} from './helpers/create-scene-server-path';
 import {convertGeometricErrorToScreenThreshold} from '../lib/utils/lod-conversion-utils';
 import {PGMLoader} from '../pgm-loader';
 
-import {LAYERS as layersTemplate} from './json-templates/layers';
-import {GEOMETRY_DEFINITION as geometryDefinitionTemlate} from './json-templates/geometry-definitions';
-import {SHARED_RESOURCES as sharedResourcesTemplate} from './json-templates/shared-resources';
+import {createLayers} from './json-templates/layers';
+import {createGeometryDefinition} from './json-templates/geometry-definitions';
+import {createSharedResources} from './json-templates/shared-resources';
 import {validateNodeBoundingVolumes} from './helpers/node-debug';
 import {KTX2BasisWriterWorker} from '@loaders.gl/textures';
-import {LoaderWithParser} from '@loaders.gl/loader-utils';
-import {I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/i3s';
+import {LoaderWithParser} from '@loaders.gl/core/loader-utils';
+import {I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/tiles-3d/i3s';
 import {getImageData, ImageWriter} from '@loaders.gl/images';
-import {GLTFImagePostprocessed} from '@loaders.gl/gltf';
 import {
   GLTFPrimitiveModeString,
   I3SConvertedResources,
@@ -58,7 +57,7 @@ import {
   ResourceType,
   SharedResourcesArrays
 } from './types';
-import {WorkerFarm} from '@loaders.gl/worker-utils';
+import {WorkerFarm} from '@loaders.gl/core/worker-utils';
 import WriteQueue from '../lib/utils/write-queue';
 import {BROWSER_ERROR_MESSAGE} from '../constants';
 import {
@@ -73,12 +72,12 @@ import {
   loadFromArchive
 } from './helpers/load-3d-tiles';
 import {Matrix4} from '@math.gl/core';
-import {BoundingSphere, OrientedBoundingBox} from '@math.gl/culling';
-import {TILE_REFINEMENT, createBoundingVolume} from '@loaders.gl/tiles';
+import {BoundingSphere, OrientedBoundingBox} from '@math.gl/geometry/culling';
+import {TILE_REFINEMENT, createBoundingVolume} from '@loaders.gl/tiles-3d';
 import {TraversalConversionProps, traverseDatasetWith} from './helpers/tileset-traversal';
 import {analyzeTileContent, mergePreprocessData} from './helpers/preprocess-3d-tiles';
 import {Progress} from './helpers/progress';
-import {composeHashFile, createZip} from '@loaders.gl/zip';
+import {composeHashFile, createZip} from '@loaders.gl/archives/zip';
 import {ConversionDump, ConversionDumpOptions, DumpMetadata} from '../lib/utils/conversion-dump';
 import {PromptModule} from 'inquirer';
 
@@ -475,7 +474,7 @@ export default class I3SConverter {
 
     if (this.conversionDump.restored && this.conversionDump.materialDefinitions) {
       for (let i = 0; i < this.conversionDump.materialDefinitions.length; i++) {
-        const hash = md5(JSON.stringify(this.conversionDump.materialDefinitions[i]));
+        const hash = hashString(JSON.stringify(this.conversionDump.materialDefinitions[i]));
         this.materialMap.set(hash, i);
       }
       this.materialDefinitions = this.conversionDump.materialDefinitions;
@@ -531,12 +530,11 @@ export default class I3SConverter {
 
     this.layers0!.materialDefinitions = this.materialDefinitions;
     // @ts-ignore
-    this.layers0.geometryDefinitions = transform(
-      this.geometryConfigs.map(config => ({
+    this.layers0.geometryDefinitions = this.geometryConfigs
+      .map(config => ({
         geometryConfig: {...config, draco: this.options.draco}
-      })),
-      geometryDefinitionTemlate()
-    );
+      }))
+      .map(createGeometryDefinition);
 
     if (this.layersHasTexture === false) {
       this.layers0!.store.defaultGeometrySchema.ordering =
@@ -575,12 +573,12 @@ export default class I3SConverter {
     }
     const extent = [fullExtent.xmin, fullExtent.ymin, fullExtent.xmax, fullExtent.ymax];
     const layers0data = {
-      version: `{${uuidv4().toUpperCase()}}`,
+      version: `{${randomUUID().toUpperCase()}}`,
       id: 0,
       name: tilesetName,
       href: './layers/0',
       store: {
-        id: `{${uuidv4().toUpperCase()}}`,
+        id: `{${randomUUID().toUpperCase()}}`,
         extent
       },
       nodePages: {
@@ -589,7 +587,7 @@ export default class I3SConverter {
       compressGeometry: this.options.draco,
       fullExtent
     };
-    this.layers0 = transform(layers0data, layersTemplate());
+    this.layers0 = createLayers(layers0data);
   }
 
   /**
@@ -1210,7 +1208,7 @@ export default class I3SConverter {
       return;
     }
     sharedResources.nodePath = nodePath;
-    const sharedData = transform(sharedResources, sharedResourcesTemplate());
+    const sharedData = createSharedResources(sharedResources);
     const sharedDataStr = JSON.stringify(sharedData);
     this.conversionDump.updateDoneStatus(sourceId, nodeId, ResourceType.SHARED, false);
     const slpkSharedPath = join(childPath, 'shared');
@@ -1233,7 +1231,7 @@ export default class I3SConverter {
    */
   // eslint-disable-next-line max-statements
   private async _writeTexture(
-    texture: GLTFImagePostprocessed,
+    texture: ConversionImage,
     childPath: string,
     slpkChildPath: string,
     sourceId: string,
@@ -1462,7 +1460,7 @@ export default class I3SConverter {
    * @return material id
    */
   private _findOrCreateMaterial(material: I3SMaterialDefinition): number {
-    const hash = md5(JSON.stringify(material));
+    const hash = hashString(JSON.stringify(material));
     if (this.materialMap.has(hash)) {
       return this.materialMap.get(hash) || 0;
     }
@@ -1480,7 +1478,7 @@ export default class I3SConverter {
    */
   private findOrCreateGeometryDefinition(hasTexture: boolean, hasUvRegions: boolean): number {
     const geometryConfig = {hasTexture, hasUvRegions};
-    const hash = md5(JSON.stringify(geometryConfig));
+    const hash = hashString(JSON.stringify(geometryConfig));
     if (this.geometryMap.has(hash)) {
       return this.geometryMap.get(hash) || 0;
     }
@@ -1508,7 +1506,7 @@ export default class I3SConverter {
     if (this.options.metadataClass) {
       if (!this.attributeMetadataInfo.attributeStorageInfo.length && tileContent?.gltf) {
         attributeTypesMap = getAttributeTypesMapFromSchema(
-          tileContent.gltf,
+          tileContent.gltf.json,
           this.options.metadataClass
         );
       }

package/src/v4/i3s-converter/json-templates/geometry-definitions.ts

@@ -1,83 +1,88 @@
-import transform from 'json-map-transform';
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 
-const PLAIN_GEOMETRY_DEFINITION = () => ({
-  offset: {
-    default: 8
-  },
-  position: {
-    default: {
+import type {BuilderInput} from '../../lib/utils/builder-input';
+
+import type {SceneLayer3D} from '@loaders.gl/tiles-3d/i3s';
+/** Legacy geometry layout choices. */
+export type GeometryConfiguration = {
+  /** Whether texture coordinates are present. */
+  hasTexture?: boolean;
+  /** Whether UV regions are present. */
+  hasUvRegions?: boolean;
+  /** Whether Draco is emitted. */
+  draco?: boolean;
+};
+/** Input to one geometry layout builder. */
+export type GeometryDefinitionInput = {
+  /** Geometry layout choices. */
+  geometryConfig: GeometryConfiguration;
+};
+
+/** Builds plain geometry definition with the legacy field selection and defaults. */
+function createPlainGeometryDefinition(
+  input: BuilderInput<GeometryConfiguration> | null | undefined
+) {
+  const source = input ?? {};
+  return {
+    offset: 8,
+    position: {
       type: 'Float32',
       component: 3
-    }
-  },
-  normal: {
-    default: {
+    },
+    normal: {
       type: 'Float32',
       component: 3
-    }
-  },
-  uv0: {
-    path: 'hasTexture',
-    transform: val => (val && {type: 'Float32', component: 2}) || false,
-    omitValues: [false]
-  },
-  color: {
-    default: {
+    },
+    ...(source.hasTexture ? {uv0: {type: 'Float32', component: 2}} : {}),
+    color: {
       type: 'UInt8',
       component: 4
-    }
-  },
-  uvRegion: {
-    path: 'hasUvRegions',
-    transform: val => (val && {type: 'UInt16', component: 4}) || false,
-    omitValues: [false]
-  },
-  featureId: {
-    default: {
+    },
+    ...(source.hasUvRegions ? {uvRegion: {type: 'UInt16', component: 4}} : {}),
+    featureId: {
       binding: 'per-feature',
       type: 'UInt64',
       component: 1
-    }
-  },
-  faceRange: {
-    default: {
+    },
+    faceRange: {
       binding: 'per-feature',
       type: 'UInt32',
       component: 2
     }
-  }
-});
+  };
+}
 
-const COMPRESSED_GEOMETRY_DEFINITION = () => ({
-  'compressedAttributes.encoding': {
-    default: 'draco'
-  },
-  'compressedAttributes.attributes': {
-    path: 'geometryConfig',
-    transform: val => {
-      const result = ['position', 'normal'];
-      if (val.hasTexture) {
-        result.push('uv0');
-      }
-      result.push('color');
-      if (val.hasUvRegions) {
-        result.push('uv-region');
-      }
-      result.push('feature-index');
-      return result;
+/** Builds compressed geometry definition with the legacy field selection and defaults. */
+function createCompressedGeometryDefinition(
+  input: BuilderInput<GeometryDefinitionInput> | null | undefined
+) {
+  const configuration = input!.geometryConfig!;
+  return {
+    compressedAttributes: {
+      encoding: 'draco',
+      attributes: [
+        'position',
+        'normal',
+        ...(configuration.hasTexture ? ['uv0'] : []),
+        'color',
+        ...(configuration.hasUvRegions ? ['uv-region'] : []),
+        'feature-index'
+      ]
     }
-  }
-});
+  };
+}
 
-export const GEOMETRY_DEFINITION = () => ({
-  geometryBuffers: {
-    path: 'geometryConfig',
-    transform: val => {
-      const result = [transform(val, PLAIN_GEOMETRY_DEFINITION())];
-      if (val.draco) {
-        result.push(transform({geometryConfig: val}, COMPRESSED_GEOMETRY_DEFINITION()));
-      }
-      return result;
-    }
-  }
-});
+/** Builds geometry definition with the legacy field selection and defaults. */
+export function createGeometryDefinition(
+  input: BuilderInput<GeometryDefinitionInput> | null | undefined
+): NonNullable<SceneLayer3D['geometryDefinitions']>[number] {
+  const source = input ?? {};
+  return {
+    geometryBuffers: [
+      createPlainGeometryDefinition(source.geometryConfig),
+      ...(source.geometryConfig!.draco ? [createCompressedGeometryDefinition(source)] : [])
+    ]
+  } as NonNullable<SceneLayer3D['geometryDefinitions']>[number];
+}

package/src/v4/i3s-converter/json-templates/layers.ts

@@ -1,137 +1,102 @@
-import transform from 'json-map-transform';
-import {STORE} from './store';
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 
-const SPATIAL_REFERENCE = () => ({
-  wkid: {
-    path: 'wkid',
-    default: 4326
-  },
-  latestWkid: {
-    path: 'latestWkid',
-    default: 4326
-  },
-  vcsWkid: {
-    path: 'vcsWkid',
-    default: 5773
-  },
-  latestVcsWkid: {
-    path: 'latestVcsWkid',
-    default: 5773
-  }
-});
+import type {BuilderInput} from '../../lib/utils/builder-input';
 
-const HEIGHT_MODEL_INFO = () => ({
-  heightModel: {
-    path: 'heightModel',
-    default: 'gravity_related_height'
-  },
-  vertCRS: {
-    path: 'vertCRS',
-    default: 'EGM96_Geoid'
-  },
-  heightUnit: {
-    path: 'heightUnit',
-    default: 'meter'
-  }
-});
+import type {SceneLayer3D} from '@loaders.gl/tiles-3d/i3s';
+import {createStore, type StoreBuilderRecord} from './store';
 
-const NODE_PAGES = () => ({
-  nodesPerPage: {
-    path: 'nodesPerPage'
-  },
-  lodSelectionMetricType: {
-    path: 'lodSelectionMetricType',
-    default: 'maxScreenThresholdSQ'
-  }
-});
+/** Partially specified nested records completed by the legacy layer builder. */
+type LayerBuilderRecord = Omit<
+  SceneLayer3D,
+  'store' | 'nodePages' | 'spatialReference' | 'heightModelInfo' | 'fullExtent'
+> & {
+  /** Store values completed with defaults. */
+  store: BuilderInput<StoreBuilderRecord>;
+  /** Node page settings. */
+  nodePages?: BuilderInput<NonNullable<SceneLayer3D['nodePages']>>;
+  /** Spatial reference identifiers. */
+  spatialReference?: BuilderInput<NonNullable<SceneLayer3D['spatialReference']>>;
+  /** Height model settings. */
+  heightModelInfo?: BuilderInput<NonNullable<SceneLayer3D['heightModelInfo']>>;
+  /** Extent fields retained without defaults. */
+  fullExtent?: BuilderInput<NonNullable<SceneLayer3D['fullExtent']>>;
+};
 
-const FULL_EXTENT = () => ({
-  xmin: {
-    path: 'xmin'
-  },
-  ymin: {
-    path: 'ymin'
-  },
-  xmax: {
-    path: 'xmax'
-  },
-  ymax: {
-    path: 'ymax'
-  },
-  zmin: {
-    path: 'zmin'
-  },
-  zmax: {
-    path: 'zmax'
-  }
-});
+/** Builds spatial reference with the legacy field selection and defaults. */
+function createSpatialReference(
+  input: BuilderInput<NonNullable<SceneLayer3D['spatialReference']>> | null | undefined
+) {
+  const source = input ?? {};
+  return {
+    wkid: source.wkid ?? 4326,
+    latestWkid: source.latestWkid ?? 4326,
+    vcsWkid: source.vcsWkid ?? 5773,
+    latestVcsWkid: source.latestVcsWkid ?? 5773
+  };
+}
 
-export const LAYERS = () => ({
-  version: {
-    path: 'version',
-    transform: val => val.toUpperCase()
-  },
-  id: {
-    path: 'id',
-    default: 0
-  },
-  name: {
-    path: 'name'
-  },
-  href: {
-    path: 'href',
-    default: './layers/0'
-  },
-  layerType: {
-    path: 'layerType',
-    default: 'IntegratedMesh'
-  },
-  spatialReference: {
-    path: 'spatialReference',
-    transform: val => transform(val, SPATIAL_REFERENCE())
-  },
-  capabilities: {
-    path: 'capabilities',
-    default: ['View', 'Query']
-  },
-  store: {
-    path: 'store',
-    transform: val => transform(val, STORE)
-  },
-  fullExtent: {
-    path: 'fullExtent',
-    transform: val => transform(val, FULL_EXTENT())
-  },
-  heightModelInfo: {
-    path: 'heightModelInfo',
-    transform: val => transform(val, HEIGHT_MODEL_INFO())
-  },
-  nodePages: {
-    path: 'nodePages',
-    transform: val => transform(val, NODE_PAGES())
-  },
-  materialDefinitions: {
-    path: 'materialDefinitions',
-    default: []
-  },
-  textureSetDefinitions: {
-    path: 'textureSetDefinitions',
-    default: []
-  },
-  geometryDefinitions: {
-    path: 'geometryDefinitions',
-    default: []
-  },
-  attributeStorageInfo: {
-    path: 'attributeStorageInfo',
-    default: []
-  },
-  fields: {
-    path: 'fields',
-    default: []
-  },
-  popupInfo: {
-    path: 'popupInfo',
-    default: null
-  }
-});
+/** Builds height model info with the legacy field selection and defaults. */
+function createHeightModelInfo(
+  input: BuilderInput<NonNullable<SceneLayer3D['heightModelInfo']>> | null | undefined
+) {
+  const source = input ?? {};
+  return {
+    heightModel: source.heightModel ?? 'gravity_related_height',
+    vertCRS: source.vertCRS ?? 'EGM96_Geoid',
+    heightUnit: source.heightUnit ?? 'meter'
+  };
+}
+
+/** Builds node pages with the legacy field selection and defaults. */
+function createNodePages(
+  input: BuilderInput<NonNullable<SceneLayer3D['nodePages']>> | null | undefined
+) {
+  const source = input ?? {};
+  return {
+    nodesPerPage: source.nodesPerPage ?? undefined,
+    lodSelectionMetricType: source.lodSelectionMetricType ?? 'maxScreenThresholdSQ'
+  };
+}
+
+/** Builds full extent with the legacy field selection and defaults. */
+function createFullExtent(
+  input: BuilderInput<NonNullable<SceneLayer3D['fullExtent']>> | null | undefined
+) {
+  const source = input ?? {};
+  return {
+    xmin: source.xmin ?? undefined,
+    ymin: source.ymin ?? undefined,
+    xmax: source.xmax ?? undefined,
+    ymax: source.ymax ?? undefined,
+    zmin: source.zmin ?? undefined,
+    zmax: source.zmax ?? undefined
+  };
+}
+
+/** Builds layers with the legacy field selection and defaults. */
+export function createLayers(
+  input: BuilderInput<LayerBuilderRecord> | null | undefined
+): SceneLayer3D {
+  const source = input ?? {};
+  return {
+    version: source.version!.toUpperCase(),
+    id: source.id ?? 0,
+    name: source.name ?? undefined,
+    href: source.href ?? './layers/0',
+    layerType: source.layerType ?? 'IntegratedMesh',
+    spatialReference: createSpatialReference(source.spatialReference),
+    capabilities: source.capabilities ?? ['View', 'Query'],
+    store: createStore(source.store),
+    fullExtent: createFullExtent(source.fullExtent),
+    heightModelInfo: createHeightModelInfo(source.heightModelInfo),
+    nodePages: createNodePages(source.nodePages),
+    materialDefinitions: source.materialDefinitions ?? [],
+    textureSetDefinitions: source.textureSetDefinitions ?? [],
+    geometryDefinitions: source.geometryDefinitions ?? [],
+    attributeStorageInfo: source.attributeStorageInfo ?? [],
+    fields: source.fields ?? [],
+    popupInfo: source.popupInfo ?? undefined
+  } as SceneLayer3D;
+}

package/src/v4/i3s-converter/json-templates/metadata.ts

@@ -1,21 +1,31 @@
-export const METADATA = () => ({
-  folderPattern: {
-    path: 'folderPattern',
-    default: 'BASIC'
-  },
-  archiveCompressionType: {
-    path: 'archiveCompressionType',
-    default: 'STORE'
-  },
-  resourceCompressionType: {
-    path: 'resourceCompressionType',
-    default: 'GZIP'
-  },
-  I3SVersion: {
-    path: 'I3SVersion',
-    default: '1.8'
-  },
-  nodeCount: {
-    path: 'nodeCount'
-  }
-});
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {BuilderInput} from '../../lib/utils/builder-input';
+
+/** Serialized legacy archive metadata. */
+export type ConversionMetadata = {
+  /** Folder layout. */
+  folderPattern: string;
+  /** Archive compression. */
+  archiveCompressionType: string;
+  /** Resource compression. */
+  resourceCompressionType: string;
+  /** I3S version. */
+  I3SVersion: string;
+  /** Number of nodes. */
+  nodeCount?: number;
+};
+
+/** Builds metadata with the legacy field selection and defaults. */
+export function createMetadata(input: BuilderInput<ConversionMetadata> | null | undefined) {
+  const source = input ?? {};
+  return {
+    folderPattern: source.folderPattern ?? 'BASIC',
+    archiveCompressionType: source.archiveCompressionType ?? 'STORE',
+    resourceCompressionType: source.resourceCompressionType ?? 'GZIP',
+    I3SVersion: source.I3SVersion ?? '1.8',
+    nodeCount: source.nodeCount ?? undefined
+  };
+}