@loaders.gl/tile-converter 4.0.0-alpha.14 → 4.0.0-alpha.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "4.0.0-alpha.14",
+  "version": "4.0.0-alpha.15",
   "description": "Converter",
   "license": "MIT",
   "publishConfig": {
@@ -22,7 +22,7 @@
   "bin": {
     "tile-converter": "./bin/converter.js",
     "slpk-extractor": "./bin/slpk-extractor.js",
-    "i3s-server": "./src/i3s-server/bin/www"
+    "i3s-server": "./bin/i3s-server.js"
   },
   "files": [
     "src",
@@ -38,24 +38,25 @@
     "join-images": false
   },
   "scripts": {
-    "pre-build": "npm run build-bundle && npm run build-converter-bundle && npm run build-slpk-extractor-bundle",
+    "pre-build": "npm run build-bundle && npm run build-converter-bundle && npm run build-slpk-extractor-bundle && npm run build-i3s-server-bundle",
     "build-bundle": "esbuild ./src/index.ts --bundle --outfile=dist/dist.min.js --platform=node  --external:join-images",
     "build-converter-bundle": "esbuild src/converter-cli.ts --outfile=dist/converter.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\"",
-    "build-slpk-extractor-bundle": "esbuild src/slpk-extractor-cli.ts --outfile=dist/slpk-extractor.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\""
+    "build-slpk-extractor-bundle": "esbuild src/slpk-extractor-cli.ts --outfile=dist/slpk-extractor.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-i3s-server-bundle": "esbuild src/i3s-server/bin/www.js --outfile=dist/esm/i3s-server/bin/i3s-server.min.js --platform=node --target=esnext,node14 --external:join-images --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "4.0.0-alpha.14",
-    "@loaders.gl/crypto": "4.0.0-alpha.14",
-    "@loaders.gl/draco": "4.0.0-alpha.14",
-    "@loaders.gl/gltf": "4.0.0-alpha.14",
-    "@loaders.gl/i3s": "4.0.0-alpha.14",
-    "@loaders.gl/images": "4.0.0-alpha.14",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.14",
-    "@loaders.gl/polyfills": "4.0.0-alpha.14",
-    "@loaders.gl/textures": "4.0.0-alpha.14",
-    "@loaders.gl/tiles": "4.0.0-alpha.14",
-    "@loaders.gl/worker-utils": "4.0.0-alpha.14",
-    "@loaders.gl/zip": "4.0.0-alpha.14",
+    "@loaders.gl/3d-tiles": "4.0.0-alpha.15",
+    "@loaders.gl/crypto": "4.0.0-alpha.15",
+    "@loaders.gl/draco": "4.0.0-alpha.15",
+    "@loaders.gl/gltf": "4.0.0-alpha.15",
+    "@loaders.gl/i3s": "4.0.0-alpha.15",
+    "@loaders.gl/images": "4.0.0-alpha.15",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.15",
+    "@loaders.gl/polyfills": "4.0.0-alpha.15",
+    "@loaders.gl/textures": "4.0.0-alpha.15",
+    "@loaders.gl/tiles": "4.0.0-alpha.15",
+    "@loaders.gl/worker-utils": "4.0.0-alpha.15",
+    "@loaders.gl/zip": "4.0.0-alpha.15",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
     "@math.gl/geoid": "^3.5.1",
@@ -63,13 +64,13 @@
     "archiver": "^5.0.0",
     "cors": "^2.8.5",
     "crypt": "^0.0.2",
-    "debug": "~2.6.9",
-    "express": "~4.17.3",
+    "debug": "~4.3.4",
+    "express": "~4.18.2",
     "json-map-transform": "^1.2.6",
     "jszip": "^3.5.0",
     "md5": "^2.3.0",
     "morgan": "~1.9.1",
-    "uuid": "^8.1.0"
+    "uuid": "^9.0.0"
   },
   "peerDependencies": {
     "@loaders.gl/core": "^4.0.0-alpha.8"
@@ -78,5 +79,5 @@
     "join-images": "^1.1.3",
     "sharp": "^0.31.3"
   },
-  "gitHead": "dd885592142ad9c26fc38e0eb0a711cf4806345a"
+  "gitHead": "47467ab23c65917e4642730e99b5d005af333c76"
 }

package/src/i3s-converter/i3s-converter.ts

@@ -240,10 +240,14 @@ export default class I3SConverter {
 
     try {
       const preloadOptions = await this._fetchPreloadOptions();
+      let tilesetUrl = inputUrl;
+      if (preloadOptions.url) {
+        tilesetUrl = preloadOptions.url;
+      }
       if (preloadOptions.headers) {
         this.loadOptions.fetch = {headers: preloadOptions.headers};
       }
-      this.sourceTileset = await load(inputUrl, this.Loader, this.loadOptions);
+      this.sourceTileset = await load(tilesetUrl, this.Loader, this.loadOptions);
 
       const preprocessResult = await this.preprocessConversion();
 

package/src/i3s-server/README.md

@@ -8,12 +8,56 @@ The server provides I3S Rest endpoints per specification https://github.com/Esri
 ### Serve 3DTiles to I3S converted dataset
 
 - Convert data set from 3DTiles to I3S without `--slpk` option
-- Serve output folder `I3sLayerPath="./data/BatchedTextured" DEBUG=i3s-server:* npx i3s-server`
+- Serve output folder
+
+Example for path `./data/BatchTextured/SceneServer/layers/0/...`:
+
+#### Start the server
+
+```bash
+PORT=8080 HTTPS_PORT=4443 I3sLayerPath="./data" DEBUG=i3s-server:* npx i3s-server
+```
+#### Check the layer availability
+
+The layer should be available on URLs:
+
+- `http://localhost:8080/BatchTextured/SceneServer/layers/0/...`
+- `https://localhost:4443/BatchTextured/SceneServer/layers/0/...`
+
+#### Open in ArcGIS
+
+`https://www.arcgis.com/home/webscene/viewer.html?url=http://localhost:8080/BatchTextured/SceneServer/layers/0/`
+
+#### Open in I3S Explorer
+
+`https://i3s.loaders.gl/viewer?tileset=http://localhost:8080/BatchTextured/SceneServer/layers/0`
 
 ### Serve SLPK
 
-- Serve slpk file `I3sLayerPath="../datasets/Rancho_Mesh_mesh_v17_1.slpk" DEBUG=i3s-server:* npx i3s-server`
+Example for path `../datasets/Rancho_Mesh_mesh_v17_1.slpk`:
+
+#### Start the server
+
+```bash
+PORT=8080 HTTPS_PORT=4443 I3sLayerPath="../datasets/Rancho_Mesh_mesh_v17_1.slpk" DEBUG=i3s-server:* npx i3s-server
+```
+#### Check the layer availability
+
+The layer should be available on URLs:
+
+- `http://localhost:8080/SceneServer/layers/0/...`
+- `https://localhost:4443/SceneServer/layers/0/...`
+
+#### Open in ArcGIS
+
+`https://www.arcgis.com/home/webscene/viewer.html?url=http://localhost:8080/SceneServer`
+
+#### Open in I3S Explorer
+
+`https://i3s.loaders.gl/viewer?tileset=http://localhost:8080/SceneServer/layers/0`
 
 ## ENV variables
 
 - `I3sLayerPath` - path to converted data or SLPK file.
+- `PORT` - HTTP port. Eg for `PORT = 8080 npx i3s-server` the server will work on host `http://localhost:8080/...`. Default value is `80`;
+- `HTTPS_PORT` - HTTPS port. Eg for `PORT = 4443 npx i3s-server` the server will work on host `https://localhost:4443/...`. Default value is `443`

package/src/i3s-server/app.js

@@ -4,7 +4,6 @@ const logger = require('morgan');
 const cors = require('cors');
 
 const indexRouter = require('./routes/index');
-const {sceneServerRouter, router} = require('./routes/slpk-router');
 
 const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef
 const app = express();
@@ -16,6 +15,7 @@ app.use(express.static(path.join(__dirname, 'public')));
 app.use(cors());
 
 if (/\.slpk$/.test(I3S_LAYER_PATH)) {
+  const {sceneServerRouter, router} = require('./routes/slpk-router');
   app.use('/SceneServer/layers/0', router);
   app.use('/SceneServer', sceneServerRouter);
 } else {

package/{dist/es5/i3s-server/bin/www → src/i3s-server/bin/www.js}

@@ -72,22 +72,22 @@ function formErrorHandler(optionalPort) {
       throw error;
     }
 
-    const bind = typeof port === 'string' ? `Pipe ${optionalPort}`: `Port ${optionalPort}`;
+    const bind = typeof global.port === 'string' ? `Pipe ${optionalPort}` : `Port ${optionalPort}`;
 
     // handle specific listen errors with friendly messages
     switch (error.code) {
       case 'EACCES':
-        console.error(`${bind} requires elevated privileges`);// eslint-disable-line no-console, no-undef
-        process.exit(1);// eslint-disable-line no-process-exit, no-undef
+        console.error(`${bind} requires elevated privileges`); // eslint-disable-line no-console, no-undef
+        process.exit(1); // eslint-disable-line no-process-exit, no-undef
         break;
       case 'EADDRINUSE':
-        console.error(`${bind} is already in use`);// eslint-disable-line no-console, no-undef
-        process.exit(1);// eslint-disable-line no-process-exit, no-undef
+        console.error(`${bind} is already in use`); // eslint-disable-line no-console, no-undef
+        process.exit(1); // eslint-disable-line no-process-exit, no-undef
         break;
       default:
         throw error;
     }
-  }
+  };
 }
 
 /**
@@ -98,5 +98,5 @@ function formListeningHandler(optionalServer) {
     const addr = optionalServer.address();
     const bind = typeof addr === 'string' ? `pipe ${addr}` : `port ${addr.port}`;
     debug(`Listening on ${bind}`);
-  }
+  };
 }

package/src/i3s-server/controllers/slpk-controller.js

@@ -1,12 +1,12 @@
 require('@loaders.gl/polyfills');
-const {fetchFile, parse} = require('@loaders.gl/core');
-const {SLPKLoader} = require('@loaders.gl/i3s');
+const {parseSLPK} = require('@loaders.gl/i3s');
 const path = require('path');
+const {FileHandleProvider} = require('@loaders.gl/tile-converter');
 
 let slpkArchive;
 
 const loadArchive = async (fullLayerPath) => {
-  slpkArchive = await (await fetchFile(fullLayerPath)).arrayBuffer();
+  slpkArchive = await parseSLPK(await FileHandleProvider.from(fullLayerPath));
 };
 
 const I3S_LAYER_PATH = process.env.I3sLayerPath || ''; // eslint-disable-line no-process-env, no-undef
@@ -19,14 +19,7 @@ async function getFileByUrl(url) {
   let uncompressedFile;
   if (trimmedPath) {
     try {
-      uncompressedFile = Buffer.from(
-        await parse(slpkArchive, SLPKLoader, {
-          slpk: {
-            path: trimmedPath[1],
-            pathMode: 'http'
-          }
-        })
-      );
+      uncompressedFile = Buffer.from(await slpkArchive.getFile(trimmedPath[1], 'http'));
     } catch (e) {}
   }
   return uncompressedFile;

package/src/index.ts

@@ -1,2 +1,3 @@
 export {default as I3SConverter} from './i3s-converter/i3s-converter';
 export {default as Tiles3DConverter} from './3d-tiles-converter/3d-tiles-converter';
+export {FileHandleProvider} from './slpk-extractor/helpers/file-handle-provider';

package/src/slpk-extractor/helpers/file-handle-provider.ts

@@ -1,5 +1,5 @@
 import {FileProvider} from '@loaders.gl/i3s';
-import {promises as fsPromises, PathLike} from 'fs';
+import {FileHandle} from './fs-promises';
 
 /**
  * Provides file data using node fs library
@@ -9,31 +9,31 @@ export class FileHandleProvider implements FileProvider {
    * Returns a new copy of FileHandleProvider
    * @param path The path to the file in file system
    */
-  static async from(path: PathLike): Promise<FileHandleProvider> {
-    const fileDescriptor = await fsPromises.open(path);
-    return new FileHandleProvider(fileDescriptor, (await fileDescriptor.stat()).size);
+  static async from(path: string): Promise<FileHandleProvider> {
+    const fileDescriptor = await FileHandle.open(path);
+    return new FileHandleProvider(fileDescriptor, fileDescriptor.stat.size);
   }
 
   /**
    * The FileHandle from which data is provided
    */
-  private fileDescriptor: fsPromises.FileHandle;
+  private fileDescriptor: FileHandle;
 
   /**
    * The file length in bytes
    */
-  private size: number;
+  private size: bigint;
 
-  private constructor(fileDescriptor: fsPromises.FileHandle, size: number) {
+  private constructor(fileDescriptor: FileHandle, size: bigint) {
     this.fileDescriptor = fileDescriptor;
     this.size = size;
   }
 
   /**
-   * Gets an unsigned 8-bit integer (unsigned byte) at the specified byte offset from the start of the file.
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
    * @param offset The offset, in bytes, from the start of the file where to read the data.
    */
-  async getUint8(offset: number): Promise<number> {
+  async getUint8(offset: bigint): Promise<number> {
     const val = new Uint8Array(
       (await this.fileDescriptor.read(Buffer.alloc(1), 0, 1, offset)).buffer.buffer
     ).at(0);
@@ -44,10 +44,10 @@ export class FileHandleProvider implements FileProvider {
   }
 
   /**
-   * Gets an unsigned 16-bit integer (unsigned byte) at the specified byte offset from the start of the file.
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
    * @param offset The offset, in bytes, from the start of the file where to read the data.
    */
-  async getUint16(offset: number): Promise<number> {
+  async getUint16(offset: bigint): Promise<number> {
     const val = new Uint16Array(
       (await this.fileDescriptor.read(Buffer.alloc(2), 0, 2, offset)).buffer.buffer
     ).at(0);
@@ -58,10 +58,10 @@ export class FileHandleProvider implements FileProvider {
   }
 
   /**
-   * Gets an unsigned 32-bit integer (unsigned byte) at the specified byte offset from the start of the file.
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
    * @param offset The offset, in bytes, from the start of the file where to read the data.
    */
-  async getUint32(offset: number): Promise<number> {
+  async getUint32(offset: bigint): Promise<number> {
     const val = new Uint32Array(
       (await this.fileDescriptor.read(Buffer.alloc(4), 0, 4, offset)).buffer.buffer
     ).at(0);
@@ -71,21 +71,39 @@ export class FileHandleProvider implements FileProvider {
     return val;
   }
 
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: bigint): Promise<bigint> {
+    const val = new BigInt64Array(
+      (await this.fileDescriptor.read(Buffer.alloc(8), 0, 4, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
   /**
    * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
-   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param startOffsset The offset, in byte, from the start of the file where to start reading the data.
    * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
    */
-  async slice(startOffset: number, endOffset: number): Promise<ArrayBuffer> {
-    const length = endOffset - startOffset;
-    return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffset)).buffer
+  async slice(startOffsset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    const bigLength = endOffset - startOffsset;
+    if (bigLength > Number.MAX_SAFE_INTEGER) {
+      throw new Error('too big slice');
+    }
+    const length = Number(bigLength);
+    return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffsset)).buffer
       .buffer;
   }
 
   /**
    * the length (in bytes) of the data.
    */
-  get length(): number {
+  get length(): bigint {
     return this.size;
   }
 }

package/src/slpk-extractor/helpers/fs-promises.ts

@@ -0,0 +1,66 @@
+import {read, open, stat, BigIntStats} from 'fs';
+
+/** file reading result */
+export type FileReadResult = {
+  /** amount of the bytes read */
+  bytesRead: number;
+  /** the buffer filled with data from file*/
+  buffer: Buffer;
+};
+
+/** Object handling file info */
+export class FileHandle {
+  private fileDescriptor: number;
+  private stats: BigIntStats;
+  private constructor(fileDescriptor: number, stats: BigIntStats) {
+    this.fileDescriptor = fileDescriptor;
+    this.stats = stats;
+  }
+  /**
+   * Opens a `FileHandle`.
+   *
+   * @param path path to the file
+   * @return Fulfills with a {FileHandle} object.
+   */
+
+  static open = async (path: string): Promise<FileHandle> => {
+    const [fd, stats] = await Promise.all([
+      new Promise<number>((s) => {
+        open(path, undefined, undefined, (_err, fd) => s(fd));
+      }),
+      new Promise<BigIntStats>((s) => {
+        stat(path, {bigint: true}, (_err, stats) => s(stats));
+      })
+    ]);
+    return new FileHandle(fd, stats);
+  };
+
+  /**
+   * Reads data from the file and stores that in the given buffer.
+   *
+   * If the file is not modified concurrently, the end-of-file is reached when the
+   * number of bytes read is zero.
+   * @param buffer A buffer that will be filled with the file data read.
+   * @param offset The location in the buffer at which to start filling.
+   * @param length The number of bytes to read.
+   * @param position The location where to begin reading data from the file. If `null`, data will be read from the current file position, and the position will be updated. If `position` is an
+   * integer, the current file position will remain unchanged.
+   * @return Fulfills upon success with a FileReadResult object
+   */
+  read = (
+    buffer: Buffer,
+    offset: number,
+    length: number,
+    position: number | bigint
+  ): Promise<FileReadResult> => {
+    return new Promise((s) => {
+      read(this.fileDescriptor, buffer, offset, length, position, (_err, bytesRead, buffer) =>
+        s({bytesRead, buffer})
+      );
+    });
+  };
+
+  get stat(): BigIntStats {
+    return this.stats;
+  }
+}

package/src/slpk-extractor/slpk-extractor.ts

@@ -7,9 +7,6 @@ import {path} from '@loaders.gl/loader-utils';
 import {GZipCompression} from '@loaders.gl/compression';
 import {writeFile} from '../lib/utils/file-utils';
 
-/**
- * names of files that should be changed to index
- */
 const indexNames = [
   '3dSceneLayer.json.gz',
   '3dNodeIndexDocument.json.gz',
@@ -27,9 +24,9 @@ type File = {
 /**
  * Converter from slpk to i3s
  */
-export default class SLPKExtractor {
+export default class SLPKConverter {
   /**
-   * extract slpk to i3s
+   * Extract slpk to i3s
    * @param options
    * @param options.inputUrl the url to read SLPK file
    * @param options.outputPath the output filename
@@ -43,7 +40,7 @@ export default class SLPKExtractor {
 
     const provider = await FileHandleProvider.from(inputUrl);
 
-    let localHeader = await parseZipLocalFileHeader(0, provider);
+    let localHeader = await parseZipLocalFileHeader(0n, provider);
     while (localHeader) {
       await this.writeFile(
         await this.unGzip({
@@ -68,6 +65,7 @@ export default class SLPKExtractor {
    * Defines file name and path for i3s format
    * @param fileName initial file name and path
    */
+
   private correctIndexNames(fileName: string): string | null {
     if (indexNames.includes(path.filename(path.join('/', fileName)))) {
       return path.join(path.dirname(fileName), 'index.json.gz');
@@ -85,6 +83,7 @@ export default class SLPKExtractor {
       const compression = new GZipCompression();
 
       const decompressedData = await compression.decompress(file.data);
+
       return {data: decompressedData, name: (file.name ?? '').slice(0, -3)};
     }
     return Promise.resolve(file);

package/src/i3s-server/bin/www

@@ -1,102 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * Module dependencies.
- */
-
-const app = require('../app');
-const debug = require('debug')('i3s-server:server');
-const https = require('https');
-const http = require('http');
-const fs = require('fs');
-const path = require('path');
-
-/**
- * Get port from environment and store in Express.
- */
-
-const httpPort = normalizePort(process.env.PORT || '80'); // eslint-disable-line no-process-env, no-undef
-const httpsPort = normalizePort(process.env.HTTPS_PORT || '443'); // eslint-disable-line no-process-env, no-undef
-
-/**
- * Create HTTP server.
- */
-
-const options = {
-  key: fs.readFileSync(path.join(__dirname, '../certs/key.pem')),
-  cert: fs.readFileSync(path.join(__dirname, '../certs/cert.pem'))
-};
-
-const httpServer = http.createServer(app);
-const httpsServer = https.createServer(options, app);
-
-/**
- * Listen on provided port, on all network interfaces.
- */
-
-httpServer.listen(httpPort);
-httpServer.on('error', formErrorHandler(httpPort));
-httpServer.on('listening', formListeningHandler(httpServer));
-
-httpsServer.listen(httpsPort);
-httpsServer.on('error', formErrorHandler(httpsPort));
-httpsServer.on('listening', formListeningHandler(httpsServer));
-
-/**
- * Normalize a port into a number, string, or false.
- */
-
-function normalizePort(val) {
-  const chkPort = parseInt(val, 10);
-
-  if (isNaN(chkPort)) {
-    // named pipe
-    return val;
-  }
-
-  if (chkPort >= 0) {
-    // port number
-    return chkPort;
-  }
-
-  return false;
-}
-
-/**
- * Event listener for HTTP/HTTPS server "error" event.
- */
-
-function formErrorHandler(optionalPort) {
-  return function onError(error) {
-    if (error.syscall !== 'listen') {
-      throw error;
-    }
-
-    const bind = typeof port === 'string' ? `Pipe ${optionalPort}`: `Port ${optionalPort}`;
-
-    // handle specific listen errors with friendly messages
-    switch (error.code) {
-      case 'EACCES':
-        console.error(`${bind} requires elevated privileges`);// eslint-disable-line no-console, no-undef
-        process.exit(1);// eslint-disable-line no-process-exit, no-undef
-        break;
-      case 'EADDRINUSE':
-        console.error(`${bind} is already in use`);// eslint-disable-line no-console, no-undef
-        process.exit(1);// eslint-disable-line no-process-exit, no-undef
-        break;
-      default:
-        throw error;
-    }
-  }
-}
-
-/**
- * Event listener for HTTP/HTTPS server "listening" event.
- */
-function formListeningHandler(optionalServer) {
-  return function onListening() {
-    const addr = optionalServer.address();
-    const bind = typeof addr === 'string' ? `pipe ${addr}` : `port ${addr.port}`;
-    debug(`Listening on ${bind}`);
-  }
-}