@loaders.gl/zip 4.0.0-alpha.19 → 4.0.0-alpha.21

package/src/filesystems/zip-filesystem.ts

@@ -0,0 +1,132 @@
+import {FileSystem, isBrowser} from '@loaders.gl/core';
+import {FileProvider, isFileProvider} from '../file-provider/file-provider';
+import {FileHandleFile} from '../file-provider/file-handle-file';
+import {ZipCDFileHeader, zipCDFileHeaderGenerator} from '../parse-zip/cd-file-header';
+import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+
+type CompressionHandler = (compressedFile: ArrayBuffer) => Promise<ArrayBuffer>;
+/** Handling different compression types in zip */
+const COMPRESSION_METHODS: {[key: number]: CompressionHandler} = {
+  /** No compression */
+  0: async (compressedFile) => compressedFile
+};
+
+/**
+ * FileSystem adapter for a ZIP file
+ * Holds FileProvider object that provides random access to archived files
+ */
+export class ZipFileSystem implements FileSystem {
+  /** FileProvider instance promise */
+  private fileProvider: Promise<FileProvider | null> = Promise.resolve(null);
+
+  /**
+   * Constructor
+   * @param file - instance of FileProvider or file path string
+   */
+  constructor(file: FileProvider | string) {
+    // Try to open file in NodeJS
+    if (typeof file === 'string') {
+      if (!isBrowser) {
+        this.fileProvider = FileHandleFile.from(file);
+      } else {
+        throw new Error('Cannot open file for random access in a WEB browser');
+      }
+    } else if (isFileProvider(file)) {
+      this.fileProvider = Promise.resolve(file);
+    }
+  }
+
+  /** Clean up resources */
+  async destroy() {
+    const fileProvider = await this.fileProvider;
+    if (fileProvider) {
+      fileProvider.destroy();
+    }
+  }
+
+  /**
+   * Get file names list from zip archive
+   * @returns array of file names
+   */
+  async readdir(): Promise<string[]> {
+    const fileProvider = await this.fileProvider;
+    if (!fileProvider) {
+      throw new Error('No data detected in the zip archive');
+    }
+    const fileNames: string[] = [];
+    const zipCDIterator = zipCDFileHeaderGenerator(fileProvider);
+    for await (const cdHeader of zipCDIterator) {
+      fileNames.push(cdHeader.fileName);
+    }
+    return fileNames;
+  }
+
+  /**
+   * Get file metadata
+   * @param filename - name of a file
+   * @returns central directory data
+   */
+  async stat(filename: string): Promise<ZipCDFileHeader & {size: number}> {
+    const cdFileHeader = await this.getCDFileHeader(filename);
+    return {...cdFileHeader, size: Number(cdFileHeader.uncompressedSize)};
+  }
+
+  /**
+   * Implementation of fetch against this file system
+   * @param filename - name of a file
+   * @returns - Response with file data
+   */
+  async fetch(filename: string): Promise<Response> {
+    const fileProvider = await this.fileProvider;
+    if (!fileProvider) {
+      throw new Error('No data detected in the zip archive');
+    }
+    const cdFileHeader = await this.getCDFileHeader(filename);
+    const localFileHeader = await parseZipLocalFileHeader(
+      cdFileHeader.localHeaderOffset,
+      fileProvider
+    );
+    if (!localFileHeader) {
+      throw new Error('Local file header has not been found in the zip archive`');
+    }
+
+    const compressionHandler = COMPRESSION_METHODS[localFileHeader.compressionMethod.toString()];
+    if (!compressionHandler) {
+      throw Error('Only Deflation compression is supported');
+    }
+
+    const compressedFile = await fileProvider.slice(
+      localFileHeader.fileDataOffset,
+      localFileHeader.fileDataOffset + localFileHeader.compressedSize
+    );
+
+    const uncompressedFile = await compressionHandler(compressedFile);
+
+    const response = new Response(uncompressedFile);
+    return response;
+  }
+
+  /**
+   * Get central directory file header
+   * @param filename - name of a file
+   * @returns central directory file header
+   */
+  private async getCDFileHeader(filename: string): Promise<ZipCDFileHeader> {
+    const fileProvider = await this.fileProvider;
+    if (!fileProvider) {
+      throw new Error('No data detected in the zip archive');
+    }
+    const zipCDIterator = zipCDFileHeaderGenerator(fileProvider);
+    let result: ZipCDFileHeader | null = null;
+    for await (const cdHeader of zipCDIterator) {
+      if (cdHeader.fileName === filename) {
+        result = cdHeader;
+        break;
+      }
+    }
+    if (!result) {
+      throw new Error('File has not been found in the zip archive');
+    }
+    return result;
+  }
+}

package/src/hash-file-utility.ts

@@ -0,0 +1,101 @@
+import md5 from 'md5';
+import {zipCDFileHeaderGenerator} from './parse-zip/cd-file-header';
+import {FileProvider} from './file-provider/file-provider';
+
+/** Element of hash array */
+export type HashElement = {
+  /** File name hash */
+  hash: Buffer;
+  /** File offset in the archive */
+  offset: bigint;
+};
+
+/**
+ * Comparing md5 hashes according to https://github.com/Esri/i3s-spec/blob/master/docs/2.0/slpk_hashtable.pcsl.md step 5
+ * @param hash1 hash to compare
+ * @param hash2 hash to compare
+ * @returns -1 if hash1 < hash2, 0 of hash1 === hash2, 1 if hash1 > hash2
+ */
+export const compareHashes = (hash1: Buffer, hash2: Buffer): number => {
+  const h1 = new BigUint64Array(hash1.buffer, hash1.byteOffset, 2);
+  const h2 = new BigUint64Array(hash2.buffer, hash2.byteOffset, 2);
+
+  const diff = h1[0] === h2[0] ? h1[1] - h2[1] : h1[0] - h2[0];
+
+  if (diff < 0n) {
+    return -1;
+  } else if (diff === 0n) {
+    return 0;
+  }
+  return 1;
+};
+
+/**
+ * Reads hash file from buffer and returns it in ready-to-use form
+ * @param hashFile - bufer containing hash file
+ * @returns Array containing file info
+ */
+export const parseHashFile = (hashFile: ArrayBuffer): HashElement[] => {
+  const hashFileBuffer = Buffer.from(hashFile);
+  const hashArray: HashElement[] = [];
+  for (let i = 0; i < hashFileBuffer.buffer.byteLength; i = i + 24) {
+    const offsetBuffer = new DataView(
+      hashFileBuffer.buffer.slice(
+        hashFileBuffer.byteOffset + i + 16,
+        hashFileBuffer.byteOffset + i + 24
+      )
+    );
+    const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true);
+    hashArray.push({
+      hash: Buffer.from(
+        hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
+      ),
+      offset
+    });
+  }
+  return hashArray;
+};
+
+/**
+ * Binary search in the hash info
+ * @param hashToSearch hash that we need to find
+ * @returns required hash element or undefined if not found
+ */
+export const findBin = (
+  hashToSearch: Buffer,
+  hashArray: HashElement[]
+): HashElement | undefined => {
+  let lowerBorder = 0;
+  let upperBorder = hashArray.length;
+
+  while (upperBorder - lowerBorder > 1) {
+    const middle = lowerBorder + Math.floor((upperBorder - lowerBorder) / 2);
+    const value = compareHashes(hashArray[middle].hash, hashToSearch);
+    if (value === 0) {
+      return hashArray[middle];
+    } else if (value < 0) {
+      lowerBorder = middle;
+    } else {
+      upperBorder = middle;
+    }
+  }
+  return undefined;
+};
+
+/**
+ * generates hash info from central directory
+ * @param fileProvider - provider of the archive
+ * @returns ready to use hash info
+ */
+export const generateHashInfo = async (fileProvider: FileProvider): Promise<HashElement[]> => {
+  const zipCDIterator = zipCDFileHeaderGenerator(fileProvider);
+  const hashInfo: HashElement[] = [];
+  for await (const cdHeader of zipCDIterator) {
+    hashInfo.push({
+      hash: Buffer.from(md5(cdHeader.fileName.split('\\').join('/').toLocaleLowerCase()), 'hex'),
+      offset: cdHeader.localHeaderOffset
+    });
+  }
+  hashInfo.sort((a, b) => compareHashes(a.hash, b.hash));
+  return hashInfo;
+};

package/src/index.ts

@@ -1,3 +1,22 @@
 export {ZipLoader} from './zip-loader';
 export {ZipWriter} from './zip-writer';
 export {default as TarBuilder} from './tar-builder';
+
+export type {FileProvider} from './file-provider/file-provider';
+export {FileHandleFile} from './file-provider/file-handle-file';
+export {DataViewFile} from './file-provider/data-view-file';
+
+export {
+  parseZipCDFileHeader,
+  zipCDFileHeaderGenerator,
+  signature as cdSignature
+} from './parse-zip/cd-file-header';
+export {
+  parseZipLocalFileHeader,
+  signature as localHeaderSignature
+} from './parse-zip/local-file-header';
+export {parseEoCDRecord} from './parse-zip/end-of-central-directory';
+export {searchFromTheEnd} from './parse-zip/search-from-the-end';
+
+export type {HashElement} from './hash-file-utility';
+export {compareHashes, parseHashFile, findBin, generateHashInfo} from './hash-file-utility';

package/src/parse-zip/cd-file-header.ts

@@ -0,0 +1,114 @@
+import {FileProvider} from '../file-provider/file-provider';
+import {parseEoCDRecord} from './end-of-central-directory';
+import {ZipSignature} from './search-from-the-end';
+
+/**
+ * zip central directory file header info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipCDFileHeader = {
+  /** Compressed size */
+  compressedSize: bigint;
+  /** Uncompressed size */
+  uncompressedSize: bigint;
+  /** Extra field size */
+  extraFieldLength: number;
+  /** File name length */
+  fileNameLength: number;
+  /** File name */
+  fileName: string;
+  /** Extra field offset */
+  extraOffset: bigint;
+  /** Relative offset of local file header */
+  localHeaderOffset: bigint;
+};
+
+// offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format)
+const CD_COMPRESSED_SIZE_OFFSET = 20n;
+const CD_UNCOMPRESSED_SIZE_OFFSET = 24n;
+const CD_FILE_NAME_LENGTH_OFFSET = 28n;
+const CD_EXTRA_FIELD_LENGTH_OFFSET = 30n;
+const CD_LOCAL_HEADER_OFFSET_OFFSET = 42n;
+const CD_FILE_NAME_OFFSET = 46n;
+
+export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];
+
+/**
+ * Parses central directory file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+export const parseZipCDFileHeader = async (
+  headerOffset: bigint,
+  buffer: FileProvider
+): Promise<ZipCDFileHeader | null> => {
+  if (
+    Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(
+      Buffer.from(signature)
+    ) !== 0
+  ) {
+    return null;
+  }
+
+  let compressedSize = BigInt(await buffer.getUint32(headerOffset + CD_COMPRESSED_SIZE_OFFSET));
+
+  let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + CD_UNCOMPRESSED_SIZE_OFFSET));
+
+  const extraFieldLength = await buffer.getUint16(headerOffset + CD_EXTRA_FIELD_LENGTH_OFFSET);
+
+  const fileNameLength = await buffer.getUint16(headerOffset + CD_FILE_NAME_LENGTH_OFFSET);
+
+  const fileName = new TextDecoder().decode(
+    await buffer.slice(
+      headerOffset + CD_FILE_NAME_OFFSET,
+      headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength)
+    )
+  );
+
+  const extraOffset = headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength);
+
+  const oldFormatOffset = await buffer.getUint32(headerOffset + CD_LOCAL_HEADER_OFFSET_OFFSET);
+
+  let fileDataOffset = BigInt(oldFormatOffset);
+  let offsetInZip64Data = 4n;
+  // looking for info that might be also be in zip64 extra field
+  if (uncompressedSize === BigInt(0xffffffff)) {
+    uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    offsetInZip64Data += 8n;
+  }
+  if (compressedSize === BigInt(0xffffffff)) {
+    compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    offsetInZip64Data += 8n;
+  }
+  if (fileDataOffset === BigInt(0xffffffff)) {
+    fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
+  }
+  const localHeaderOffset = fileDataOffset;
+
+  return {
+    compressedSize,
+    uncompressedSize,
+    extraFieldLength,
+    fileNameLength,
+    fileName,
+    extraOffset,
+    localHeaderOffset
+  };
+};
+
+/**
+ * Create iterator over files of zip archive
+ * @param fileProvider - file provider that provider random access to the file
+ */
+export async function* zipCDFileHeaderGenerator(fileProvider: FileProvider) {
+  const {cdStartOffset} = await parseEoCDRecord(fileProvider);
+  let cdHeader = await parseZipCDFileHeader(cdStartOffset, fileProvider);
+  while (cdHeader) {
+    yield cdHeader;
+    cdHeader = await parseZipCDFileHeader(
+      cdHeader.extraOffset + BigInt(cdHeader.extraFieldLength),
+      fileProvider
+    );
+  }
+}

package/src/parse-zip/end-of-central-directory.ts

@@ -0,0 +1,71 @@
+import {FileProvider} from '../file-provider/file-provider';
+import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
+
+/**
+ * End of central directory info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipEoCDRecord = {
+  /** Relative offset of local file header */
+  cdStartOffset: bigint;
+  /** Relative offset of local file header */
+  cdRecordsNumber: bigint;
+};
+
+const eoCDSignature: ZipSignature = [0x50, 0x4b, 0x05, 0x06];
+const zip64EoCDLocatorSignature = Buffer.from([0x50, 0x4b, 0x06, 0x07]);
+const zip64EoCDSignature = Buffer.from([0x50, 0x4b, 0x06, 0x06]);
+
+// offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format)
+const CD_RECORDS_NUMBER_OFFSET = 8n;
+const CD_START_OFFSET_OFFSET = 16n;
+const ZIP64_EOCD_START_OFFSET_OFFSET = 8n;
+const ZIP64_CD_RECORDS_NUMBER_OFFSET = 24n;
+const ZIP64_CD_START_OFFSET_OFFSET = 48n;
+
+/**
+ * Parses end of central directory record of zip file
+ * @param fileProvider - FileProvider instance
+ * @returns Info from the header
+ */
+export const parseEoCDRecord = async (fileProvider: FileProvider): Promise<ZipEoCDRecord> => {
+  const zipEoCDOffset = await searchFromTheEnd(fileProvider, eoCDSignature);
+
+  let cdRecordsNumber = BigInt(
+    await fileProvider.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET)
+  );
+  let cdStartOffset = BigInt(await fileProvider.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET));
+
+  if (cdStartOffset === BigInt(0xffffffff) || cdRecordsNumber === BigInt(0xffffffff)) {
+    const zip64EoCDLocatorOffset = zipEoCDOffset - 20n;
+
+    if (
+      Buffer.from(
+        await fileProvider.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n)
+      ).compare(zip64EoCDLocatorSignature) !== 0
+    ) {
+      throw new Error('zip64 EoCD locator not found');
+    }
+    const zip64EoCDOffset = await fileProvider.getBigUint64(
+      zip64EoCDLocatorOffset + ZIP64_EOCD_START_OFFSET_OFFSET
+    );
+
+    if (
+      Buffer.from(await fileProvider.slice(zip64EoCDOffset, zip64EoCDOffset + 4n)).compare(
+        zip64EoCDSignature
+      ) !== 0
+    ) {
+      throw new Error('zip64 EoCD not found');
+    }
+
+    cdRecordsNumber = await fileProvider.getBigUint64(
+      zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET
+    );
+    cdStartOffset = await fileProvider.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET);
+  }
+
+  return {
+    cdRecordsNumber,
+    cdStartOffset
+  };
+};

package/src/parse-zip/local-file-header.ts

@@ -0,0 +1,91 @@
+import {FileProvider} from '../file-provider/file-provider';
+
+/**
+ * zip local file header info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipLocalFileHeader = {
+  /** File name length */
+  fileNameLength: number;
+  /** File name */
+  fileName: string;
+  /** Extra field length */
+  extraFieldLength: number;
+  /** Offset of the file data */
+  fileDataOffset: bigint;
+  /** Compressed size */
+  compressedSize: bigint;
+  /** Compression method */
+  compressionMethod: number;
+};
+
+// offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format)
+const COMPRESSION_METHOD_OFFSET = 8n;
+const COMPRESSED_SIZE_OFFSET = 18n;
+const UNCOMPRESSED_SIZE_OFFSET = 22n;
+const FILE_NAME_LENGTH_OFFSET = 26n;
+const EXTRA_FIELD_LENGTH_OFFSET = 28n;
+const FILE_NAME_OFFSET = 30n;
+
+export const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
+
+/**
+ * Parses local file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+export const parseZipLocalFileHeader = async (
+  headerOffset: bigint,
+  buffer: FileProvider
+): Promise<ZipLocalFileHeader | null> => {
+  if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(signature) !== 0) {
+    return null;
+  }
+
+  const fileNameLength = await buffer.getUint16(headerOffset + FILE_NAME_LENGTH_OFFSET);
+
+  const fileName = new TextDecoder()
+    .decode(
+      await buffer.slice(
+        headerOffset + FILE_NAME_OFFSET,
+        headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength)
+      )
+    )
+    .split('\\')
+    .join('/');
+  const extraFieldLength = await buffer.getUint16(headerOffset + EXTRA_FIELD_LENGTH_OFFSET);
+
+  let fileDataOffset = headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength);
+
+  const compressionMethod = await buffer.getUint16(headerOffset + COMPRESSION_METHOD_OFFSET);
+
+  let compressedSize = BigInt(await buffer.getUint32(headerOffset + COMPRESSED_SIZE_OFFSET)); // add zip 64 logic
+
+  let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + UNCOMPRESSED_SIZE_OFFSET)); // add zip 64 logic
+
+  const extraOffset = headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength);
+
+  let offsetInZip64Data = 4n;
+  // looking for info that might be also be in zip64 extra field
+  if (uncompressedSize === BigInt(0xffffffff)) {
+    uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    offsetInZip64Data += 8n;
+  }
+  if (compressedSize === BigInt(0xffffffff)) {
+    compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    offsetInZip64Data += 8n;
+  }
+  if (fileDataOffset === BigInt(0xffffffff)) {
+    fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
+  }
+
+  return {
+    fileNameLength,
+    fileName,
+    extraFieldLength,
+    fileDataOffset,
+    compressedSize,
+    compressionMethod
+  };
+};

package/src/parse-zip/search-from-the-end.ts

@@ -0,0 +1,38 @@
+import {FileProvider} from '../file-provider/file-provider';
+
+/** Description of zip signature type */
+export type ZipSignature = [number, number, number, number];
+
+/**
+ * looking for the last occurrence of the provided
+ * @param file
+ * @param target
+ * @returns
+ */
+export const searchFromTheEnd = async (
+  file: FileProvider,
+  target: ZipSignature
+): Promise<bigint> => {
+  const searchWindow = [
+    await file.getUint8(file.length - 1n),
+    await file.getUint8(file.length - 2n),
+    await file.getUint8(file.length - 3n),
+    undefined
+  ];
+
+  let targetOffset = 0n;
+
+  // looking for the last record in the central directory
+  for (let i = file.length - 4n; i > -1; i--) {
+    searchWindow[3] = searchWindow[2];
+    searchWindow[2] = searchWindow[1];
+    searchWindow[1] = searchWindow[0];
+    searchWindow[0] = await file.getUint8(i);
+    if (searchWindow.every((val, index) => val === target[index])) {
+      targetOffset = i;
+      break;
+    }
+  }
+
+  return targetOffset;
+};