@loaders.gl/i3s 4.0.0-alpha.10 → 4.0.0-alpha.11

package/src/lib/parsers/parse-slpk/slpk-archieve.ts

@@ -1,7 +1,7 @@
-import {processOnWorker} from '@loaders.gl/worker-utils';
 import md5 from 'md5';
-import {CompressionWorker} from '@loaders.gl/compression';
 import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
+import {GZipCompression} from '@loaders.gl/compression';
 
 /** Element of hash array */
 type HashElement = {
@@ -15,6 +15,7 @@ type HashElement = {
   offset: number;
 };
 
+/** Description of real paths for different file types */
 const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
   {
     test: /^$/,
@@ -114,7 +115,7 @@ export class SLPKArchive {
       if (decompressedFile) {
         return Buffer.from(decompressedFile);
       }
-      const fileWithoutCompression = this.getFileBytes(path);
+      const fileWithoutCompression = await this.getFileBytes(path);
       if (fileWithoutCompression) {
         return Buffer.from(fileWithoutCompression);
       }
@@ -129,17 +130,14 @@ export class SLPKArchive {
    * @returns buffer with the file data
    */
   private async getDataByPath(path: string): Promise<ArrayBuffer | undefined> {
-    const data = this.getFileBytes(path);
+    const data = await this.getFileBytes(path);
     if (!data) {
       return undefined;
     }
     if (/\.gz$/.test(path)) {
-      const decompressedData = await processOnWorker(CompressionWorker, data, {
-        compression: 'gzip',
-        operation: 'decompress',
-        _workerType: 'test',
-        gzip: {}
-      });
+      const compression = new GZipCompression();
+
+      const decompressedData = await compression.decompress(data);
       return decompressedData;
     }
     return Buffer.from(data);
@@ -150,17 +148,20 @@ export class SLPKArchive {
    * @param path - path inside the archive
    * @returns buffer with the raw file data
    */
-  private getFileBytes(path: string): ArrayBuffer | undefined {
+  private async getFileBytes(path: string): Promise<ArrayBuffer | undefined> {
     const nameHash = Buffer.from(md5(path), 'hex');
     const fileInfo = this.hashArray.find((val) => Buffer.compare(val.hash, nameHash) === 0);
     if (!fileInfo) {
       return undefined;
     }
 
-    const localFileHeader = parseZipLocalFileHeader(
+    const localFileHeader = await parseZipLocalFileHeader(
       this.slpkArchive.byteOffset + fileInfo?.offset,
-      this.slpkArchive
+      new DataViewFileProvider(this.slpkArchive)
     );
+    if (!localFileHeader) {
+      return undefined;
+    }
 
     const compressedFile = this.slpkArchive.buffer.slice(
       localFileHeader.fileDataOffset,

package/src/lib/parsers/parse-zip/buffer-file-provider.ts

@@ -0,0 +1,55 @@
+import {FileProvider} from './file-provider';
+
+/**
+ * Provides file data using DataView
+ */
+export class DataViewFileProvider implements FileProvider {
+  /**
+   * The DataView from which data is provided
+   */
+  private file: DataView;
+
+  constructor(file: DataView) {
+    this.file = file;
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: number): Promise<number> {
+    return Promise.resolve(this.file.getUint8(offset));
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: number): Promise<number> {
+    return Promise.resolve(this.file.getUint16(offset, true));
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint32(offset: number): Promise<number> {
+    return Promise.resolve(this.file.getUint32(offset, true));
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: number, endOffset: number): Promise<ArrayBuffer> {
+    return Promise.resolve(this.file.buffer.slice(startOffset, endOffset));
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length() {
+    return this.file.byteLength;
+  }
+}

package/src/lib/parsers/parse-zip/cd-file-header.ts

@@ -1,31 +1,21 @@
+import {FileProvider} from './file-provider';
+
 /**
  * zip central directory file header info
  * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
  */
 export type ZipCDFileHeader = {
-  /**
-   * Compressed size
-   */
+  /** Compressed size */
   compressedSize: number;
-  /**
-   * Uncompressed size
-   */
+  /** Uncompressed size */
   uncompressedSize: number;
-  /**
-   * File name length
-   */
+  /** File name length */
   fileNameLength: number;
-  /**
-   * File name
-   */
-  fileName: ArrayBuffer;
-  /**
-   * Extra field offset
-   */
+  /** File name */
+  fileName: string;
+  /** Extra field offset */
   extraOffset: number;
-  /**
-   * Relative offset of local file header
-   */
+  /** Relative offset of local file header */
   localHeaderOffset: number;
 };
 
@@ -35,7 +25,10 @@ export type ZipCDFileHeader = {
  * @param buffer - buffer containing whole array
  * @returns Info from the header
  */
-export const parseZipCDFileHeader = (headerOffset: number, buffer: DataView): ZipCDFileHeader => {
+export const parseZipCDFileHeader = async (
+  headerOffset: number,
+  buffer: FileProvider
+): Promise<ZipCDFileHeader> => {
   const offsets = {
     CD_COMPRESSED_SIZE_OFFSET: 20,
     CD_UNCOMPRESSED_SIZE_OFFSET: 24,
@@ -45,25 +38,25 @@ export const parseZipCDFileHeader = (headerOffset: number, buffer: DataView): Zi
     CD_FILE_NAME_OFFSET: 46
   };
 
-  const compressedSize = buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET, true);
+  const compressedSize = await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET);
 
-  const uncompressedSize = buffer.getUint32(
-    headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET,
-    true
+  const uncompressedSize = await buffer.getUint32(
+    headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET
   );
 
-  const fileNameLength = buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET, true);
+  const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET);
 
-  const fileName = buffer.buffer.slice(
-    headerOffset + offsets.CD_FILE_NAME_OFFSET,
-    headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength
+  const fileName = new TextDecoder().decode(
+    await buffer.slice(
+      headerOffset + offsets.CD_FILE_NAME_OFFSET,
+      headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength
+    )
   );
 
   const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength;
 
-  const oldFormatOffset = buffer.getUint32(
-    headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET,
-    true
+  const oldFormatOffset = await buffer.getUint32(
+    headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET
   );
 
   let fileDataOffset = oldFormatOffset;
@@ -78,7 +71,7 @@ export const parseZipCDFileHeader = (headerOffset: number, buffer: DataView): Zi
     }
 
     // getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb
-    fileDataOffset = buffer.getUint32(extraOffset + offsetInZip64Data, true); // setting it to the one from zip64
+    fileDataOffset = await buffer.getUint32(extraOffset + offsetInZip64Data); // setting it to the one from zip64
   }
   const localHeaderOffset = fileDataOffset;
 

package/src/lib/parsers/parse-zip/file-provider.ts

@@ -0,0 +1,34 @@
+/**
+ * Interface for providing file data
+ */
+export interface FileProvider {
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: number): Promise<number>;
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: number): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the file of the view where to read the data.
+   */
+  getUint32(offset: number): Promise<number>;
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: number, endOffset: number): Promise<ArrayBuffer>;
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  length: number;
+}

package/src/lib/parsers/parse-zip/local-file-header.ts

@@ -1,54 +1,63 @@
+import {FileProvider} from './file-provider';
+
 /**
  * zip local file header info
  * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
  */
 export type ZipLocalFileHeader = {
-  /**
-   * File name length
-   */
+  /** File name length */
   fileNameLength: number;
-  /**
-   * Extra field length
-   */
+  /** File name */
+  fileName: string;
+  /** Extra field length */
   extraFieldLength: number;
-  /**
-   * Offset of the file data
-   */
+  /** Offset of the file data */
   fileDataOffset: number;
-  /**
-   * Compressed size
-   */
+  /** Compressed size */
   compressedSize: number;
 };
 
+const offsets = {
+  COMPRESSED_SIZE_OFFSET: 18,
+  FILE_NAME_LENGTH_OFFSET: 26,
+  EXTRA_FIELD_LENGTH_OFFSET: 28,
+  FILE_NAME_OFFSET: 30
+};
+
+const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
+
 /**
  * Parses local file header of zip file
  * @param headerOffset - offset in the archive where header starts
  * @param buffer - buffer containing whole array
  * @returns Info from the header
  */
-export const parseZipLocalFileHeader = (
+export const parseZipLocalFileHeader = async (
   headerOffset: number,
-  buffer: DataView
-): ZipLocalFileHeader => {
-  const offsets = {
-    COMPRESSED_SIZE_OFFSET: 18,
-    FILE_NAME_LENGTH_OFFSET: 26,
-    EXTRA_FIELD_LENGTH_OFFSET: 28,
-    FILE_NAME_OFFSET: 30
-  };
+  buffer: FileProvider
+): Promise<ZipLocalFileHeader | undefined> => {
+  if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4)).compare(signature) !== 0) {
+    return Promise.resolve(undefined);
+  }
 
-  const fileNameLength = buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET, true);
+  const fileNameLength = await buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET);
 
-  const extraFieldLength = buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET, true);
+  const fileName = new TextDecoder().decode(
+    await buffer.slice(
+      headerOffset + offsets.FILE_NAME_OFFSET,
+      headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength
+    )
+  );
+  const extraFieldLength = await buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET);
 
   const fileDataOffset =
     headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + extraFieldLength;
 
-  const compressedSize = buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET, true);
+  const compressedSize = await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET);
 
   return {
     fileNameLength,
+    fileName,
     extraFieldLength,
     fileDataOffset,
     compressedSize