@loaders.gl/i3s 4.0.0-alpha.6 → 4.0.0-alpha.7

package/dist/lib/parsers/parse-zip/local-file-header.d.ts

@@ -0,0 +1,30 @@
+/**
+ * zip local file header info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipLocalFileHeader = {
+    /**
+     * File name length
+     */
+    fileNameLength: number;
+    /**
+     * Extra field length
+     */
+    extraFieldLength: number;
+    /**
+     * Offset of the file data
+     */
+    fileDataOffset: number;
+    /**
+     * Compressed size
+     */
+    compressedSize: number;
+};
+/**
+ * Parses local file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+export declare const parseZipLocalFileHeader: (headerOffset: number, buffer: DataView) => ZipLocalFileHeader;
+//# sourceMappingURL=local-file-header.d.ts.map

package/dist/lib/parsers/parse-zip/local-file-header.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-file-header.d.ts","sourceRoot":"","sources":["../../../../src/lib/parsers/parse-zip/local-file-header.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,iBACpB,MAAM,UACZ,QAAQ,KACf,kBAuBF,CAAC"}

package/dist/lib/parsers/parse-zip/local-file-header.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseZipLocalFileHeader = void 0;
+/**
+ * Parses local file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+const parseZipLocalFileHeader = (headerOffset, buffer) => {
+    const offsets = {
+        COMPRESSED_SIZE_OFFSET: 18,
+        FILE_NAME_LENGTH_OFFSET: 26,
+        EXTRA_FIELD_LENGTH_OFFSET: 28,
+        FILE_NAME_OFFSET: 30
+    };
+    const fileNameLength = buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET, true);
+    const extraFieldLength = buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET, true);
+    const fileDataOffset = headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + extraFieldLength;
+    const compressedSize = buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET, true);
+    return {
+        fileNameLength,
+        extraFieldLength,
+        fileDataOffset,
+        compressedSize
+    };
+};
+exports.parseZipLocalFileHeader = parseZipLocalFileHeader;

package/dist/workers/i3s-content-worker-node.d.ts

@@ -0,0 +1,2 @@
+import '@loaders.gl/polyfills';
+//# sourceMappingURL=i3s-content-worker-node.d.ts.map

package/dist/workers/i3s-content-worker-node.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"i3s-content-worker-node.d.ts","sourceRoot":"","sources":["../../src/workers/i3s-content-worker-node.ts"],"names":[],"mappings":"AACA,OAAO,uBAAuB,CAAC"}

package/dist/workers/{i3s-content-nodejs-worker.js → i3s-content-worker-node.js}

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const loader_utils_1 = require("@loaders.gl/loader-utils");
+// Polyfills increases the bundle size significantly. Use it for NodeJS worker only
 require("@loaders.gl/polyfills");
+const loader_utils_1 = require("@loaders.gl/loader-utils");
 const i3s_content_loader_1 = require("../i3s-content-loader");
 (0, loader_utils_1.createLoaderWorker)(i3s_content_loader_1.I3SContentLoader);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/i3s",
-  "version": "4.0.0-alpha.6",
+  "version": "4.0.0-alpha.7",
   "description": "i3s .",
   "license": "MIT",
   "publishConfig": {
@@ -27,25 +27,27 @@
     "README.md"
   ],
   "scripts": {
-    "pre-build": "npm run build-bundle && npm run build-worker && npm run build-loader-worker-nodejs",
+    "pre-build": "npm run build-bundle && npm run build-worker && npm run build-worker-node",
     "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js",
-    "build-worker": "esbuild src/workers/i3s-content-worker.ts --bundle --outfile=dist/i3s-content-worker.js --define:__VERSION__=\\\"$npm_package_version\\\"",
-    "build-loader-worker-nodejs": "esbuild src/workers/i3s-content-nodejs-worker.ts --outfile=dist/i3s-content-nodejs-worker.js --platform=node --target=esnext,node12 --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
+    "build-worker": "esbuild src/workers/i3s-content-worker.ts --outfile=dist/i3s-content-worker.js --target=esnext --bundle --define:__VERSION__=\\\"$npm_package_version\\\"",
+    "build-worker-node": "esbuild src/workers/i3s-content-worker-node.ts --outfile=dist/i3s-content-worker-node.js --platform=node --target=node16 --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/draco": "4.0.0-alpha.6",
-    "@loaders.gl/images": "4.0.0-alpha.6",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.6",
-    "@loaders.gl/schema": "4.0.0-alpha.6",
-    "@loaders.gl/textures": "4.0.0-alpha.6",
-    "@loaders.gl/tiles": "4.0.0-alpha.6",
+    "@loaders.gl/compression": "4.0.0-alpha.7",
+    "@loaders.gl/draco": "4.0.0-alpha.7",
+    "@loaders.gl/images": "4.0.0-alpha.7",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.7",
+    "@loaders.gl/schema": "4.0.0-alpha.7",
+    "@loaders.gl/textures": "4.0.0-alpha.7",
+    "@loaders.gl/tiles": "4.0.0-alpha.7",
     "@luma.gl/constants": "^8.5.4",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
-    "@math.gl/geospatial": "^3.5.1"
+    "@math.gl/geospatial": "^3.5.1",
+    "md5": "^2.3.0"
   },
   "peerDependencies": {
-    "@loaders.gl/core": "4.0.0-alpha.5"
+    "@loaders.gl/core": "4.0.0-alpha.6"
   },
-  "gitHead": "acc1985050dfaa0f1f0c066f8da5bce7454a046c"
+  "gitHead": "afb59c4d8e5d8ebb9c28f111cb0c96c5527d0ffd"
 }

package/src/i3s-content-loader.ts

@@ -1,4 +1,3 @@
-import {isBrowser} from '@loaders.gl/worker-utils';
 import type {LoaderWithParser, LoaderContext} from '@loaders.gl/loader-utils';
 import type {I3SLoaderOptions} from './i3s-loader';
 import {parseI3STileContent} from './lib/parsers/parse-i3s-tile-content';
@@ -13,7 +12,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'beta';
  */
 export const I3SContentLoader: LoaderWithParser = {
   name: 'I3S Content (Indexed Scene Layers)',
-  id: isBrowser ? 'i3s-content' : 'i3s-content-nodejs',
+  id: 'i3s-content',
   module: 'i3s',
   worker: true,
   version: VERSION,

package/src/i3s-slpk-loader.ts

@@ -0,0 +1,25 @@
+import {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
+import {parseSLPK} from './lib/parsers/parse-slpk/parse-slpk';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type SLPKLoaderOptions = LoaderOptions & {
+  path?: string;
+  mode?: 'http' | 'raw';
+};
+
+/**
+ * Loader for SLPK - Scene Layer Package
+ */
+export const SLPKLoader: LoaderWithParser = {
+  name: 'I3S SLPK (Scene Layer Package)',
+  id: 'slpk',
+  module: 'i3s',
+  version: VERSION,
+  mimeTypes: ['application/octet-stream'],
+  parse: parseSLPK,
+  extensions: ['slpk'],
+  options: {}
+};

package/src/index.ts

@@ -36,9 +36,9 @@ export type {
 export {COORDINATE_SYSTEM} from './lib/parsers/constants';
 
 export {I3SLoader} from './i3s-loader';
+export {SLPKLoader} from './i3s-slpk-loader';
 export {I3SContentLoader} from './i3s-content-loader';
 export {I3SAttributeLoader, loadFeatureAttributes} from './i3s-attribute-loader';
-export {I3SNodePageLoader} from './i3s-node-page-loader';
 export {I3SBuildingSceneLayerLoader} from './i3s-building-scene-layer-loader';
-
+export {I3SNodePageLoader} from './i3s-node-page-loader';
 export {ArcGisWebSceneLoader} from './arcgis-webscene-loader';

package/src/lib/parsers/parse-slpk/parse-slpk.ts

@@ -0,0 +1,61 @@
+import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader';
+import {parseZipCDFileHeader} from '../parse-zip/cd-file-header';
+import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+import {SLPKArchive} from './slpk-archieve';
+
+/**
+ * Returns one byte from the provided buffer at the provided position
+ * @param offset - position where to read
+ * @param buffer - buffer to read
+ * @returns one byte from the provided buffer at the provided position
+ */
+const getByteAt = (offset: number, buffer: DataView): number => {
+  return buffer.getUint8(buffer.byteOffset + offset);
+};
+
+export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
+  const archive = new DataView(data);
+  const cdFileHeaderSignature = [80, 75, 1, 2];
+
+  const searchWindow = [
+    getByteAt(archive.byteLength - 1, archive),
+    getByteAt(archive.byteLength - 2, archive),
+    getByteAt(archive.byteLength - 3, archive),
+    undefined
+  ];
+
+  let hashCDOffset = 0;
+
+  // looking for the last record in the central directory
+  for (let i = archive.byteLength - 4; i > -1; i--) {
+    searchWindow[3] = searchWindow[2];
+    searchWindow[2] = searchWindow[1];
+    searchWindow[1] = searchWindow[0];
+    searchWindow[0] = getByteAt(i, archive);
+    if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) {
+      hashCDOffset = i;
+      break;
+    }
+  }
+
+  const cdFileHeader = parseZipCDFileHeader(hashCDOffset, archive);
+
+  const textDecoder = new TextDecoder();
+  if (textDecoder.decode(cdFileHeader.fileName) !== '@specialIndexFileHASH128@') {
+    throw new Error('No hash file in slpk');
+  }
+
+  const localFileHeader = parseZipLocalFileHeader(cdFileHeader.localHeaderOffset, archive);
+
+  const fileDataOffset = localFileHeader.fileDataOffset;
+  const hashFile = archive.buffer.slice(
+    fileDataOffset,
+    fileDataOffset + localFileHeader.compressedSize
+  );
+
+  if (!hashFile) {
+    throw new Error('No hash file in slpk');
+  }
+
+  return await new SLPKArchive(data, hashFile).getFile(options.path ?? '');
+}

package/src/lib/parsers/parse-slpk/slpk-archieve.ts

@@ -0,0 +1,108 @@
+import {processOnWorker} from '@loaders.gl/worker-utils';
+import md5 from 'md5';
+import {CompressionWorker} from '@loaders.gl/compression';
+import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+
+/** Element of hash array */
+type HashElement = {
+  /**
+   * File name hash
+   */
+  hash: Buffer;
+  /**
+   * File offset in the archive
+   */
+  offset: number;
+};
+
+/**
+ * Class for handling information about slpk file
+ */
+export class SLPKArchive {
+  slpkArchive: DataView;
+  hashArray: {hash: Buffer; offset: number}[];
+  constructor(slpkArchiveBuffer: ArrayBuffer, hashFile: ArrayBuffer) {
+    this.slpkArchive = new DataView(slpkArchiveBuffer);
+    this.hashArray = this.parseHashFile(hashFile);
+  }
+
+  /**
+   * Reads hash file from buffer and returns it in ready-to-use form
+   * @param hashFile - bufer containing hash file
+   * @returns Array containing file info
+   */
+  private parseHashFile(hashFile: ArrayBuffer): HashElement[] {
+    const hashFileBuffer = Buffer.from(hashFile);
+    const hashArray: HashElement[] = [];
+    for (let i = 0; i < hashFileBuffer.buffer.byteLength; i = i + 24) {
+      const offsetBuffer = new DataView(
+        hashFileBuffer.buffer.slice(
+          hashFileBuffer.byteOffset + i + 16,
+          hashFileBuffer.byteOffset + i + 24
+        )
+      );
+      const offset = offsetBuffer.getUint32(offsetBuffer.byteOffset, true);
+      hashArray.push({
+        hash: Buffer.from(
+          hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
+        ),
+        offset
+      });
+    }
+    return hashArray;
+  }
+
+  /**
+   * Returns file with the given path from slpk archive
+   * @param path - path inside the slpk
+   * @param mode - currently only raw mode supported
+   * @returns buffer with ready to use file
+   */
+  async getFile(path: string, mode: 'http' | 'raw' = 'raw'): Promise<Buffer> {
+    if (mode === 'http') {
+      throw new Error('http mode is not supported');
+    }
+
+    const fileToDecompress = this.getFileBytes(`${path}.gz`);
+
+    if (fileToDecompress) {
+      const decompressedData = await processOnWorker(CompressionWorker, fileToDecompress, {
+        compression: 'gzip',
+        operation: 'decompress',
+        _workerType: 'test',
+        gzip: {}
+      });
+      return decompressedData;
+    }
+    const fileWithoutCompression = this.getFileBytes(path);
+    if (fileWithoutCompression) {
+      return Promise.resolve(Buffer.from(fileWithoutCompression));
+    }
+    throw new Error('No such file in the archieve');
+  }
+
+  /**
+   * Trying to get raw file data by adress
+   * @param path - path inside the archive
+   * @returns buffer with the raw file data
+   */
+  private getFileBytes(path: string): ArrayBuffer | undefined {
+    const nameHash = Buffer.from(md5(path), 'hex');
+    const fileInfo = this.hashArray.find((val) => Buffer.compare(val.hash, nameHash) === 0);
+    if (!fileInfo) {
+      return undefined;
+    }
+
+    const localFileHeader = parseZipLocalFileHeader(
+      this.slpkArchive.byteOffset + fileInfo?.offset,
+      this.slpkArchive
+    );
+
+    const compressedFile = this.slpkArchive.buffer.slice(
+      localFileHeader.fileDataOffset,
+      localFileHeader.fileDataOffset + localFileHeader.compressedSize
+    );
+
+    return compressedFile;
+  }
+}

package/src/lib/parsers/parse-zip/cd-file-header.ts

@@ -0,0 +1,93 @@
+/**
+ * zip central directory file header info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipCDFileHeader = {
+  /**
+   * Compressed size
+   */
+  compressedSize: number;
+  /**
+   * Uncompressed size
+   */
+  uncompressedSize: number;
+  /**
+   * File name length
+   */
+  fileNameLength: number;
+  /**
+   * File name
+   */
+  fileName: ArrayBuffer;
+  /**
+   * Extra field offset
+   */
+  extraOffset: number;
+  /**
+   * Relative offset of local file header
+   */
+  localHeaderOffset: number;
+};
+
+/**
+ * Parses central directory file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+export const parseZipCDFileHeader = (headerOffset: number, buffer: DataView): ZipCDFileHeader => {
+  const offsets = {
+    CD_COMPRESSED_SIZE_OFFSET: 20,
+    CD_UNCOMPRESSED_SIZE_OFFSET: 24,
+    CD_FILE_NAME_LENGTH_OFFSET: 28,
+    CD_EXTRA_FIELD_LENGTH_OFFSET: 30,
+    CD_LOCAL_HEADER_OFFSET_OFFSET: 42,
+    CD_FILE_NAME_OFFSET: 46
+  };
+
+  const compressedSize = buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET, true);
+
+  const uncompressedSize = buffer.getUint32(
+    headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET,
+    true
+  );
+
+  const fileNameLength = buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET, true);
+
+  const fileName = buffer.buffer.slice(
+    headerOffset + offsets.CD_FILE_NAME_OFFSET,
+    headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength
+  );
+
+  const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength;
+
+  const oldFormatOffset = buffer.getUint32(
+    headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET,
+    true
+  );
+
+  let fileDataOffset = oldFormatOffset;
+  if (fileDataOffset === 0xffffffff) {
+    let offsetInZip64Data = 4;
+    // looking for info that might be also be in zip64 extra field
+    if (compressedSize === 0xffffffff) {
+      offsetInZip64Data += 8;
+    }
+    if (uncompressedSize === 0xffffffff) {
+      offsetInZip64Data += 8;
+    }
+
+    // getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb
+    fileDataOffset = buffer.getUint32(extraOffset + offsetInZip64Data, true); // setting it to the one from zip64
+  }
+  const localHeaderOffset = fileDataOffset;
+
+  return {
+    compressedSize,
+    uncompressedSize,
+    fileNameLength,
+    fileName,
+    extraOffset,
+    localHeaderOffset
+  };
+};

package/src/lib/parsers/parse-zip/local-file-header.ts

@@ -0,0 +1,56 @@
+/**
+ * zip local file header info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipLocalFileHeader = {
+  /**
+   * File name length
+   */
+  fileNameLength: number;
+  /**
+   * Extra field length
+   */
+  extraFieldLength: number;
+  /**
+   * Offset of the file data
+   */
+  fileDataOffset: number;
+  /**
+   * Compressed size
+   */
+  compressedSize: number;
+};
+
+/**
+ * Parses local file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+export const parseZipLocalFileHeader = (
+  headerOffset: number,
+  buffer: DataView
+): ZipLocalFileHeader => {
+  const offsets = {
+    COMPRESSED_SIZE_OFFSET: 18,
+    FILE_NAME_LENGTH_OFFSET: 26,
+    EXTRA_FIELD_LENGTH_OFFSET: 28,
+    FILE_NAME_OFFSET: 30
+  };
+
+  const fileNameLength = buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET, true);
+
+  const extraFieldLength = buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET, true);
+
+  const fileDataOffset =
+    headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + extraFieldLength;
+
+  const compressedSize = buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET, true);
+
+  return {
+    fileNameLength,
+    extraFieldLength,
+    fileDataOffset,
+    compressedSize
+  };
+};

package/src/workers/{i3s-content-nodejs-worker.ts → i3s-content-worker-node.ts}

@@ -1,5 +1,6 @@
-import {createLoaderWorker} from '@loaders.gl/loader-utils';
+// Polyfills increases the bundle size significantly. Use it for NodeJS worker only
 import '@loaders.gl/polyfills';
+import {createLoaderWorker} from '@loaders.gl/loader-utils';
 import {I3SContentLoader} from '../i3s-content-loader';
 
 createLoaderWorker(I3SContentLoader);

package/dist/es5/workers/i3s-content-nodejs-worker.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"i3s-content-nodejs-worker.js","names":["_loaderUtils","require","_i3sContentLoader","createLoaderWorker","I3SContentLoader"],"sources":["../../../src/workers/i3s-content-nodejs-worker.ts"],"sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport '@loaders.gl/polyfills';\nimport {I3SContentLoader} from '../i3s-content-loader';\n\ncreateLoaderWorker(I3SContentLoader);\n"],"mappings":";;AAAA,IAAAA,YAAA,GAAAC,OAAA;AACAA,OAAA;AACA,IAAAC,iBAAA,GAAAD,OAAA;AAEA,IAAAE,+BAAkB,EAACC,kCAAgB,CAAC"}

package/dist/esm/workers/i3s-content-nodejs-worker.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"i3s-content-nodejs-worker.js","names":["createLoaderWorker","I3SContentLoader"],"sources":["../../../src/workers/i3s-content-nodejs-worker.ts"],"sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport '@loaders.gl/polyfills';\nimport {I3SContentLoader} from '../i3s-content-loader';\n\ncreateLoaderWorker(I3SContentLoader);\n"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,0BAA0B;AAC3D,OAAO,uBAAuB;AAC9B,SAAQC,gBAAgB,QAAO,uBAAuB;AAEtDD,kBAAkB,CAACC,gBAAgB,CAAC"}