@loaders.gl/tile-converter 4.4.0-alpha.1 → 4.4.0-alpha.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "4.4.0-alpha.1",
+  "version": "4.4.0-alpha.9",
   "description": "Converter",
   "license": "MIT",
   "type": "module",
@@ -51,20 +51,20 @@
     "copy-certificates": "cp -R src/i3s-server/certs dist/i3s-server"
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "4.4.0-alpha.1",
-    "@loaders.gl/compression": "4.4.0-alpha.1",
-    "@loaders.gl/crypto": "4.4.0-alpha.1",
-    "@loaders.gl/draco": "4.4.0-alpha.1",
-    "@loaders.gl/gltf": "4.4.0-alpha.1",
-    "@loaders.gl/i3s": "4.4.0-alpha.1",
-    "@loaders.gl/images": "4.4.0-alpha.1",
-    "@loaders.gl/loader-utils": "4.4.0-alpha.1",
-    "@loaders.gl/math": "4.4.0-alpha.1",
-    "@loaders.gl/polyfills": "4.4.0-alpha.1",
-    "@loaders.gl/textures": "4.4.0-alpha.1",
-    "@loaders.gl/tiles": "4.4.0-alpha.1",
-    "@loaders.gl/worker-utils": "4.4.0-alpha.1",
-    "@loaders.gl/zip": "4.4.0-alpha.1",
+    "@loaders.gl/3d-tiles": "4.4.0-alpha.9",
+    "@loaders.gl/compression": "4.4.0-alpha.9",
+    "@loaders.gl/crypto": "4.4.0-alpha.9",
+    "@loaders.gl/draco": "4.4.0-alpha.9",
+    "@loaders.gl/gltf": "4.4.0-alpha.9",
+    "@loaders.gl/i3s": "4.4.0-alpha.9",
+    "@loaders.gl/images": "4.4.0-alpha.9",
+    "@loaders.gl/loader-utils": "4.4.0-alpha.9",
+    "@loaders.gl/math": "4.4.0-alpha.9",
+    "@loaders.gl/polyfills": "4.4.0-alpha.9",
+    "@loaders.gl/textures": "4.4.0-alpha.9",
+    "@loaders.gl/tiles": "4.4.0-alpha.9",
+    "@loaders.gl/worker-utils": "4.4.0-alpha.9",
+    "@loaders.gl/zip": "4.4.0-alpha.9",
     "@math.gl/core": "^4.1.0",
     "@math.gl/culling": "^4.1.0",
     "@math.gl/geoid": "^4.1.0",
@@ -91,7 +91,7 @@
     "@types/inquirer": "^9.0.7"
   },
   "peerDependencies": {
-    "@loaders.gl/core": "4.4.0-alpha.0"
+    "@loaders.gl/core": "4.4.0-alpha.1"
   },
-  "gitHead": "f1732de45907bd500bf4eedb4803beca8bf4bfb0"
+  "gitHead": "e9e6710379718c7663e97eba868c76e15de4cb84"
 }

package/src/3d-tiles-converter/3d-tiles-converter.ts

@@ -51,10 +51,12 @@ export default class Tiles3DConverter {
   workerSource: {[key: string]: string} = {};
   slpkFilesystem: ZipFileSystem | null = null;
   loaderOptions: I3SLoaderOptions = {
-    _nodeWorkers: true,
-    reuseWorkers: true,
-    // TODO: converter freezes in the end because of i3s-content-worker
-    worker: false,
+    core: {
+      _nodeWorkers: true,
+      reuseWorkers: true,
+      // TODO: converter freezes in the end because of i3s-content-worker
+      worker: false
+    },
     i3s: {coordinateSystem: COORDINATE_SYSTEM.LNGLAT_OFFSETS, decodeTextures: false},
     // We need to load local fs workers because nodejs can't load workers from the Internet
     'i3s-content': {
@@ -451,9 +453,11 @@ export default class Tiles3DConverter {
       const inputUrl = attributeUrls[index];
       const attribute = attributeStorageInfo[index];
       const options = {
-        attributeName: attribute.name,
-        attributeType: this._getAttributeType(attribute)
-      };
+        i3s: {
+          attributeName: attribute.name,
+          attributeType: this._getAttributeType(attribute)
+        }
+      }; //  as const satisfies I3SLoaderOptions;
 
       promises.push(loadFromArchive(inputUrl, I3SAttributeLoader, options, this.slpkFilesystem));
     }

package/src/3d-tiles-converter/helpers/load-i3s.ts

@@ -7,7 +7,7 @@ import {
   I3SLoaderOptions,
   parseSLPKArchive
 } from '@loaders.gl/i3s';
-import {FileHandleFile} from '@loaders.gl/loader-utils';
+import {NodeFile} from '@loaders.gl/loader-utils';
 import {ZipFileSystem, makeZipCDHeaderIterator} from '@loaders.gl/zip';
 
 export type SLPKUrlParts = {slpkFileName: string; internalFileName: string};
@@ -75,7 +75,7 @@ export async function openSLPK(url: string): Promise<ZipFileSystem | null> {
   const slpkUrlParts = url.split('.slpk');
   if (slpkUrlParts.length === 2) {
     const slpkFileName = `${slpkUrlParts[0]}.slpk`;
-    const fileProvider = new FileHandleFile(slpkFileName);
+    const fileProvider = new NodeFile(slpkFileName);
     const archive = await parseSLPKArchive(fileProvider, undefined, slpkFileName);
     const fileSystem = new ZipFileSystem(archive);
     return fileSystem;
@@ -99,7 +99,7 @@ export async function loadFromArchive(
   if (fileSystem !== null) {
     const content = await load(url, loader, {
       ...loadOptions,
-      fetch: fileSystem.fetch.bind(fileSystem)
+      core: {...loadOptions?.core, fetch: fileSystem.fetch.bind(fileSystem)}
     });
     return content;
   }
@@ -112,11 +112,11 @@ export async function loadFromArchive(
  * @returns number of nodes
  */
 export async function getNodeCount(fileSystem: ZipFileSystem | null): Promise<number> {
-  if (!fileSystem?.fileProvider) {
+  if (!fileSystem?.file) {
     return 0;
   }
   const nodeSet = new Set();
-  const filesIterator = makeZipCDHeaderIterator(fileSystem.fileProvider);
+  const filesIterator = makeZipCDHeaderIterator(fileSystem.file);
   for await (const file of filesIterator) {
     const filename = file.fileName;
     const nodeNumberSearchResult = /^nodes\/(\d+)\//.exec(filename);

package/src/converter-cli.ts

@@ -12,7 +12,7 @@ import {
   validateOptionsWithEqual
 } from './lib/utils/cli-utils';
 import {addOneFile, composeHashFile, makeZipCDHeaderIterator} from '@loaders.gl/zip';
-import {FileHandleFile} from '@loaders.gl/loader-utils';
+import {NodeFile} from '@loaders.gl/loader-utils';
 // @ts-ignore
 import {copyFile} from 'node:fs/promises';
 
@@ -141,7 +141,7 @@ async function main() {
     if (finalPath !== validatedOptions.tileset) {
       await copyFile(validatedOptions.tileset, finalPath);
     }
-    const hashTable = await composeHashFile(makeZipCDHeaderIterator(new FileHandleFile(finalPath)));
+    const hashTable = await composeHashFile(makeZipCDHeaderIterator(new NodeFile(finalPath)));
     await addOneFile(finalPath, hashTable, '@specialIndexFileHASH128@');
 
     return;

package/src/i3s-converter/helpers/geometry-converter.ts

@@ -18,6 +18,11 @@ import type {
 import {Vector3, Matrix4, Vector4} from '@math.gl/core';
 import {Ellipsoid} from '@math.gl/geospatial';
 
+/* @ts-expect-error TS2732: Cannot find module 'node:module' or its corresponding type declarations. */
+import {createRequire} from 'node:module';
+/* @ts-expect-error TS2732: Cannot find module 'node:module' or its corresponding type declarations. */
+import {dirname, join} from 'node:path';
+
 import {DracoWriterWorker} from '@loaders.gl/draco';
 import {assert, encode} from '@loaders.gl/core';
 import {
@@ -1629,12 +1634,23 @@ async function generateCompressedGeometry(
       },
       ['draco-writer']: {
         // We need to load local fs workers because nodejs can't load workers from the Internet
-        workerUrl: './modules/draco/dist/draco-writer-worker-node.js'
+        workerUrl: getLocalDracoWriterWorkerUrl()
       }
     }
   );
 }
 
+function getLocalDracoWriterWorkerUrl(): string {
+  try {
+    const require = createRequire(import.meta.url);
+    const packageJsonPath = require.resolve('@loaders.gl/draco/package.json');
+    return join(dirname(packageJsonPath), 'src/workers/draco-writer-worker-node.cjs');
+  } catch {
+    // Fall back to monorepo-local path
+    return './modules/draco/src/workers/draco-writer-worker-node.cjs';
+  }
+}
+
 /**
  * Generates ordered feature indices based on face range
  * @param featureIndex

package/src/i3s-converter/helpers/load-3d-tiles.ts

@@ -6,14 +6,16 @@ import type {
 } from '@loaders.gl/3d-tiles';
 import {Tiles3DArchive} from '@loaders.gl/3d-tiles';
 import {LoaderWithParser, load} from '@loaders.gl/core';
-import {FileHandleFile, FileProviderInterface} from '@loaders.gl/loader-utils';
+import type {ReadableFile} from '@loaders.gl/loader-utils';
+import {HttpFile, NodeFile, isBrowser} from '@loaders.gl/loader-utils';
 import {
   CD_HEADER_SIGNATURE,
   ZipFileSystem,
   parseHashTable,
   parseZipCDFileHeader,
   parseZipLocalFileHeader,
-  searchFromTheEnd
+  searchFromTheEnd,
+  readRange
 } from '@loaders.gl/zip';
 
 /**
@@ -113,14 +115,19 @@ export async function loadFromArchive(
   }
   if (filename) {
     const tz3Path = `${tz3UrlParts[0]}.3tz`;
-    const fileProvider = new FileHandleFile(tz3Path);
+    const fileProvider = createReadableFile(tz3Path);
     const hashTable = await loadHashTable(fileProvider);
     const archive = new Tiles3DArchive(fileProvider, hashTable, tz3Path);
     const fileSystem = new ZipFileSystem(archive);
-    const content = await load(filename, loader, {
-      ...loadOptions,
-      fetch: fileSystem.fetch.bind(fileSystem)
-    });
+    // const loadOptionsWithoutDeprecatedFetch = {...loadOptions} as Tiles3DLoaderOptions & {
+    //   fetch?: unknown;
+    // };
+    // delete loadOptionsWithoutDeprecatedFetch.fetch;
+    // const mergedOptions = {
+    //   ...loadOptionsWithoutDeprecatedFetch,
+    //   core: {...loadOptions?.core, fetch: fileSystem.fetch.bind(fileSystem)}
+    // }
+    const content = await load(filename, loader, loadOptions);
     await fileSystem.destroy();
     return content;
   }
@@ -136,13 +143,20 @@ export function isNestedTileset(tile: Tiles3DTileJSONPostprocessed) {
   return tile?.type === 'json' || tile?.type === '3tz';
 }
 
+function createReadableFile(path: string): ReadableFile {
+  if (path.startsWith('http://') || path.startsWith('https://')) {
+    return new HttpFile(path);
+  }
+  return isBrowser ? new HttpFile(path) : new NodeFile(path);
+}
+
 /**
  * Load hash file from 3TZ
  * @param fileProvider - binary reader of 3TZ
  * @returns hash table of the 3TZ file content or undefined if the hash file is not presented inside
  */
 async function loadHashTable(
-  fileProvider: FileProviderInterface
+  fileProvider: ReadableFile
 ): Promise<undefined | Record<string, bigint>> {
   let hashTable: undefined | Record<string, bigint>;
 
@@ -163,7 +177,8 @@ async function loadHashTable(
     }
 
     const fileDataOffset = localFileHeader.fileDataOffset;
-    const hashFile = await fileProvider.slice(
+    const hashFile = await readRange(
+      fileProvider,
       fileDataOffset,
       fileDataOffset + localFileHeader.compressedSize
     );

package/src/i3s-converter/i3s-converter.ts

@@ -135,9 +135,11 @@ export default class I3SConverter {
   refreshTokenTime: [number, number] = [0, 0];
   sourceTileset: Tiles3DTilesetJSONPostprocessed | null = null;
   loadOptions: Tiles3DLoaderOptions = {
-    _nodeWorkers: true,
-    reuseWorkers: true,
-    useLocalLibraries: true,
+    core: {
+      _nodeWorkers: true,
+      reuseWorkers: true,
+      useLocalLibraries: true
+    },
     basis: {
       format: 'rgba32',
       // We need to load local fs workers because nodejs can't load workers from the Internet
@@ -145,7 +147,6 @@ export default class I3SConverter {
     },
     // We need to load local fs workers because nodejs can't load workers from the Internet
     draco: {workerUrl: './modules/draco/dist/draco-worker-node.js'},
-    fetch: {},
     modules: {}
   };
   geoidHeightModel: Geoid | null = null;
@@ -266,7 +267,16 @@ export default class I3SConverter {
         tilesetUrl = preloadOptions.url;
       }
       if (preloadOptions.headers) {
-        this.loadOptions.fetch = {headers: preloadOptions.headers};
+        this.loadOptions.core ||= {};
+        const existingFetch = this.loadOptions.core.fetch;
+        if (typeof existingFetch === 'function') {
+          this.loadOptions.core.fetch = (url: string, requestInit?: RequestInit) =>
+            existingFetch(url, {...(requestInit || {}), headers: preloadOptions.headers});
+        } else {
+          const existingRequestInit =
+            existingFetch && typeof existingFetch === 'object' ? existingFetch : {};
+          this.loadOptions.core.fetch = {...existingRequestInit, headers: preloadOptions.headers};
+        }
       }
       this.sourceTileset = await loadFromArchive(tilesetUrl, this.Loader, this.loadOptions);
 
@@ -1570,10 +1580,16 @@ export default class I3SConverter {
 
     const preloadOptions = await this._fetchPreloadOptions();
     if (preloadOptions.headers) {
-      this.loadOptions.fetch = {
-        ...this.loadOptions.fetch,
-        headers: preloadOptions.headers
-      };
+      this.loadOptions.core ||= {};
+      const existingFetch = this.loadOptions.core.fetch;
+      if (typeof existingFetch === 'function') {
+        this.loadOptions.core.fetch = (url: string, requestInit?: RequestInit) =>
+          existingFetch(url, {...(requestInit || {}), headers: preloadOptions.headers});
+      } else {
+        const existingRequestInit =
+          existingFetch && typeof existingFetch === 'object' ? existingFetch : {};
+        this.loadOptions.core.fetch = {...existingRequestInit, headers: preloadOptions.headers};
+      }
       console.log('Authorization Bearer token has been updated'); // eslint-disable-line no-undef, no-console
     }
   }

package/src/i3s-server/controllers/slpk-controller.ts

@@ -1,6 +1,6 @@
 import '@loaders.gl/polyfills';
 import {parseSLPKArchive, SLPKArchive} from '@loaders.gl/i3s';
-import {FileHandleFile} from '@loaders.gl/loader-utils';
+import {NodeFile} from '@loaders.gl/loader-utils';
 
 let slpkArchive: SLPKArchive;
 
@@ -10,7 +10,7 @@ let slpkArchive: SLPKArchive;
  */
 export async function loadArchive(fullLayerPath: string): Promise<void> {
   slpkArchive = await parseSLPKArchive(
-    new FileHandleFile(fullLayerPath),
+    new NodeFile(fullLayerPath),
     (msg) => console.log(msg) // eslint-disable-line no-console
   );
   // eslint-disable-next-line no-console

package/src/lib/utils/compress-util.ts

@@ -22,6 +22,7 @@ export function compressFileWithGzip(pathFile: string): Promise<string> {
       console.log(`${compressedPathFile}: compression error!`); // eslint-disable-line no-undef,no-console
       reject(error);
     });
+    // @ts-ignore Seems typescript upgrade triggered this
     input.pipe(gzip).pipe(output);
   });
 }

package/dist/3d-tiles-converter/3d-tiles-converter.d.ts

@@ -1,115 +0,0 @@
-import type { AttributeStorageInfo, I3STilesetHeader } from '@loaders.gl/i3s';
-import { Geoid } from '@math.gl/geoid';
-import { I3SLoaderOptions } from '@loaders.gl/i3s/src/i3s-loader';
-import { ZipFileSystem } from "../../../zip/src.js";
-import { ConversionDump } from "../lib/utils/conversion-dump.js";
-import { Progress } from "../i3s-converter/helpers/progress.js";
-import { PromptModule } from 'inquirer';
-/**
- * Converter from i3s to 3d-tiles
- */
-export default class Tiles3DConverter {
-    options: any;
-    tilesetPath: string;
-    vertexCounter: number;
-    conversionStartTime: [number, number];
-    geoidHeightModel: Geoid | null;
-    sourceTileset: I3STilesetHeader | null;
-    attributeStorageInfo?: AttributeStorageInfo[] | null;
-    workerSource: {
-        [key: string]: string;
-    };
-    slpkFilesystem: ZipFileSystem | null;
-    loaderOptions: I3SLoaderOptions;
-    conversionDump: ConversionDump;
-    progress: Progress;
-    fileExt: string;
-    constructor();
-    /**
-     * Convert i3s format data to 3dTiles
-     * @param options
-     * @param options.inputUrl the url to read the tileset from
-     * @param options.outputPath the output filename
-     * @param options.outputVersion the version of 3DTiles
-     * @param options.tilesetName the output name of the tileset
-     * @param options.egmFilePath location of *.pgm file to convert heights from ellipsoidal to gravity-related format
-     * @param options.maxDepth The max tree depth of conversion
-     */
-    convert(options: {
-        inputUrl: string;
-        outputPath: string;
-        tilesetName: string;
-        outputVersion?: string;
-        maxDepth?: number;
-        egmFilePath: string;
-        inquirer?: {
-            prompt: PromptModule;
-        };
-        analyze?: boolean;
-    }): Promise<string | undefined>;
-    /**
-     * Preprocess stage of the tile converter. Calculate number of nodes
-     * @returns true - the conversion is possible, false - the tileset's content is not supported
-     */
-    private preprocessConversion;
-    /**
-     * Convert particular I3S Node
-     * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
-     * @param parentNode object in resulting tileset
-     * @param level a current level of a tree depth
-     * @param childNodeInfo child node to convert
-     */
-    private convertChildNode;
-    /**
-     * The recursive function of traversal of a nodes tree
-     * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
-     * @param parentNode object in resulting tileset
-     * @param level a current level of a tree depth
-     */
-    private _addChildren;
-    /**
-     * Load a child node having information from the node header
-     * @param parentNode a parent node tile object (@loaders.gl/tiles/Tile3D)
-     * @param childNodeInfo child information from 3DNodeIndexDocument
-     *   (https://github.com/Esri/i3s-spec/blob/master/docs/1.7/nodeReference.cmn.md)
-     */
-    private _loadChildNode;
-    /**
-     * Create child and child's boundingVolume for the converted node
-     * @param sourceChild
-     * @returns child and child's boundingVolume
-     */
-    private _createChildAndBoundingVolume;
-    /**
-     * Make an url of a resource from its relative url having the base url
-     * @param baseUrl the base url. A resulting url will be related from this url
-     * @param relativeUrl a realtive url of a resource
-     */
-    private _relativeUrlToFullUrl;
-    /**
-     * Do loading all attributes related to particular node.
-     * @param sourceChild
-     * @param attributeStorageInfo
-     * @returns Promise of attributes object.
-     */
-    private _loadChildAttributes;
-    /**
-     * Returns attribute type for loading attributes
-     * @param attribute
-     * Workaround for I3S v1.6. There is no attribute.attributeValues.valueType field in attribute.
-     * There is an 'Oid32' type if attribute has objectIds property.
-     * Doc: https://github.com/Esri/i3s-spec/blob/master/docs/1.6/attributeStorageInfo.cmn.md
-     */
-    private _getAttributeType;
-    /**
-     * Make simple arrays from attribute typed arrays.
-     * @param attributesList
-     */
-    private _replaceNestedArrays;
-    /**
-     * Print statistics in the end of conversion
-     * @param params - output files data
-     */
-    private _finishConversion;
-}
-//# sourceMappingURL=3d-tiles-converter.d.ts.map

package/dist/3d-tiles-converter/3d-tiles-converter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"3d-tiles-converter.d.ts","sourceRoot":"","sources":["../../src/3d-tiles-converter/3d-tiles-converter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,oBAAoB,EAGpB,gBAAgB,EAEjB,MAAM,iBAAiB,CAAC;AAQzB,OAAO,EAAC,KAAK,EAAC,MAAM,gBAAgB,CAAC;AAiBrC,OAAO,EAAC,gBAAgB,EAAC,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,OAAO,EAAC,cAAc,EAAwB,wCAAqC;AACnF,OAAO,EAAC,QAAQ,EAAC,6CAA0C;AAC3D,OAAO,EAAC,YAAY,EAAC,MAAM,UAAU,CAAC;AAItC;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,gBAAgB;IACnC,OAAO,EAAE,GAAG,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,gBAAgB,EAAE,KAAK,GAAG,IAAI,CAAC;IAC/B,aAAa,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACvC,oBAAoB,CAAC,EAAE,oBAAoB,EAAE,GAAG,IAAI,CAAC;IACrD,YAAY,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAM;IAC3C,cAAc,EAAE,aAAa,GAAG,IAAI,CAAQ;IAC5C,aAAa,EAAE,gBAAgB,CAU7B;IACF,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;;IAgBhB;;;;;;;;;OASG;IAEU,OAAO,CAAC,OAAO,EAAE;QAC5B,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,QAAQ,CAAC,EAAE;YAAC,MAAM,EAAE,YAAY,CAAA;SAAC,CAAC;QAClC,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IA8G/B;;;OAGG;YACW,oBAAoB;IA8BlC;;;;;;OAMG;YAEW,gBAAgB;IAiF9B;;;;;OAKG;YACW,YAAY;IAa1B;;;;;OAKG;YACW,cAAc;IA4B5B;;;;OAIG;IACH,OAAO,CAAC,6BAA6B;IAsBrC;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAiB7B;;;;;OAKG;YACW,oBAAoB;IAsBlC;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB;IASzB;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAU5B;;;OAGG;YACW,iBAAiB;CAgBhC"}