@loaders.gl/zip 4.0.0-beta.2 → 4.0.0-beta.4

package/src/hash-file-utility.ts

@@ -1,101 +1,53 @@
-import md5 from 'md5';
+import {MD5Hash} from '@loaders.gl/crypto';
 import {FileProvider} from '@loaders.gl/loader-utils';
-import {zipCDFileHeaderGenerator} from './parse-zip/cd-file-header';
-
-/** Element of hash array */
-export type HashElement = {
-  /** File name hash */
-  hash: Buffer;
-  /** File offset in the archive */
-  offset: bigint;
-};
+import {makeZipCDHeaderIterator} from './parse-zip/cd-file-header';
 
 /**
- * Comparing md5 hashes according to https://github.com/Esri/i3s-spec/blob/master/docs/2.0/slpk_hashtable.pcsl.md step 5
- * @param hash1 hash to compare
- * @param hash2 hash to compare
- * @returns -1 if hash1 < hash2, 0 of hash1 === hash2, 1 if hash1 > hash2
+ * Reads hash file from buffer and returns it in ready-to-use form
+ * @param arrayBuffer - buffer containing hash file
+ * @returns Map containing hash and offset
  */
-export const compareHashes = (hash1: Buffer, hash2: Buffer): number => {
-  const h1 = new BigUint64Array(hash1.buffer, hash1.byteOffset, 2);
-  const h2 = new BigUint64Array(hash2.buffer, hash2.byteOffset, 2);
+export function parseHashTable(arrayBuffer: ArrayBuffer): Record<string, bigint> {
+  const dataView = new DataView(arrayBuffer);
 
-  const diff = h1[0] === h2[0] ? h1[1] - h2[1] : h1[0] - h2[0];
+  const hashMap: Record<string, bigint> = {};
 
-  if (diff < 0n) {
-    return -1;
-  } else if (diff === 0n) {
-    return 0;
+  for (let i = 0; i < arrayBuffer.byteLength; i = i + 24) {
+    const offset = dataView.getBigUint64(i + 16, true);
+    const hash = bufferToHex(arrayBuffer, i, 16);
+    hashMap[hash] = offset;
   }
-  return 1;
-};
 
-/**
- * Reads hash file from buffer and returns it in ready-to-use form
- * @param hashFile - bufer containing hash file
- * @returns Array containing file info
- */
-export const parseHashFile = (hashFile: ArrayBuffer): HashElement[] => {
-  const hashFileBuffer = Buffer.from(hashFile);
-  const hashArray: HashElement[] = [];
-  for (let i = 0; i < hashFileBuffer.buffer.byteLength; i = i + 24) {
-    const offsetBuffer = new DataView(
-      hashFileBuffer.buffer.slice(
-        hashFileBuffer.byteOffset + i + 16,
-        hashFileBuffer.byteOffset + i + 24
-      )
-    );
-    const offset = offsetBuffer.getBigUint64(offsetBuffer.byteOffset, true);
-    hashArray.push({
-      hash: Buffer.from(
-        hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
-      ),
-      offset
-    });
-  }
-  return hashArray;
-};
+  return hashMap;
+}
 
-/**
- * Binary search in the hash info
- * @param hashToSearch hash that we need to find
- * @returns required hash element or undefined if not found
- */
-export const findBin = (
-  hashToSearch: Buffer,
-  hashArray: HashElement[]
-): HashElement | undefined => {
-  let lowerBorder = 0;
-  let upperBorder = hashArray.length;
-
-  while (upperBorder - lowerBorder > 1) {
-    const middle = lowerBorder + Math.floor((upperBorder - lowerBorder) / 2);
-    const value = compareHashes(hashArray[middle].hash, hashToSearch);
-    if (value === 0) {
-      return hashArray[middle];
-    } else if (value < 0) {
-      lowerBorder = middle;
-    } else {
-      upperBorder = middle;
-    }
-  }
-  return undefined;
-};
+function bufferToHex(buffer: ArrayBuffer, start: number, length: number): string {
+  // buffer is an ArrayBuffer
+  return [...new Uint8Array(buffer, start, length)]
+    .map((x) => x.toString(16).padStart(2, '0'))
+    .join('');
+}
 
 /**
- * generates hash info from central directory
+ * generates hash info from zip files "central directory"
  * @param fileProvider - provider of the archive
  * @returns ready to use hash info
  */
-export const generateHashInfo = async (fileProvider: FileProvider): Promise<HashElement[]> => {
-  const zipCDIterator = zipCDFileHeaderGenerator(fileProvider);
-  const hashInfo: HashElement[] = [];
+export async function makeHashTableFromZipHeaders(
+  fileProvider: FileProvider
+): Promise<Record<string, bigint>> {
+  const zipCDIterator = makeZipCDHeaderIterator(fileProvider);
+  const md5Hash = new MD5Hash();
+  const textEncoder = new TextEncoder();
+
+  const hashTable: Record<string, bigint> = {};
+
   for await (const cdHeader of zipCDIterator) {
-    hashInfo.push({
-      hash: Buffer.from(md5(cdHeader.fileName.split('\\').join('/').toLocaleLowerCase()), 'hex'),
-      offset: cdHeader.localHeaderOffset
-    });
+    const filename = cdHeader.fileName.split('\\').join('/').toLocaleLowerCase();
+    const arrayBuffer = textEncoder.encode(filename).buffer;
+    const md5 = await md5Hash.hash(arrayBuffer, 'hex');
+    hashTable[md5] = cdHeader.localHeaderOffset;
   }
-  hashInfo.sort((a, b) => compareHashes(a.hash, b.hash));
-  return hashInfo;
-};
+
+  return hashTable;
+}

package/src/index.ts

@@ -6,7 +6,7 @@ export {TarBuilder} from './tar-builder';
 
 export {
   parseZipCDFileHeader,
-  zipCDFileHeaderGenerator,
+  makeZipCDHeaderIterator,
   signature as cdSignature
 } from './parse-zip/cd-file-header';
 export {
@@ -16,7 +16,7 @@ export {
 export {parseEoCDRecord} from './parse-zip/end-of-central-directory';
 export {searchFromTheEnd} from './parse-zip/search-from-the-end';
 
-export type {HashElement} from './hash-file-utility';
-export {compareHashes, parseHashFile, findBin, generateHashInfo} from './hash-file-utility';
+// export type {HashElement} from './hash-file-utility';
+export {parseHashTable, makeHashTableFromZipHeaders} from './hash-file-utility';
 
 export {ZipFileSystem} from './filesystems/zip-filesystem';

package/src/parse-zip/cd-file-header.ts

@@ -1,4 +1,4 @@
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
 import {parseEoCDRecord} from './end-of-central-directory';
 import {ZipSignature} from './search-from-the-end';
 
@@ -31,7 +31,7 @@ const CD_EXTRA_FIELD_LENGTH_OFFSET = 30n;
 const CD_LOCAL_HEADER_OFFSET_OFFSET = 42n;
 const CD_FILE_NAME_OFFSET = 46n;
 
-export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];
+export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x01, 0x02]);
 
 /**
  * Parses central directory file header of zip file
@@ -41,48 +41,39 @@ export const signature: ZipSignature = [0x50, 0x4b, 0x01, 0x02];
  */
 export const parseZipCDFileHeader = async (
   headerOffset: bigint,
-  buffer: FileProvider
+  file: FileProvider
 ): Promise<ZipCDFileHeader | null> => {
-  if (
-    Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(
-      Buffer.from(signature)
-    ) !== 0
-  ) {
+  const magicBytes = await file.slice(headerOffset, headerOffset + 4n);
+  if (!compareArrayBuffers(magicBytes, signature.buffer)) {
     return null;
   }
 
-  let compressedSize = BigInt(await buffer.getUint32(headerOffset + CD_COMPRESSED_SIZE_OFFSET));
-
-  let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + CD_UNCOMPRESSED_SIZE_OFFSET));
-
-  const extraFieldLength = await buffer.getUint16(headerOffset + CD_EXTRA_FIELD_LENGTH_OFFSET);
-
-  const fileNameLength = await buffer.getUint16(headerOffset + CD_FILE_NAME_LENGTH_OFFSET);
-
-  const fileName = new TextDecoder().decode(
-    await buffer.slice(
-      headerOffset + CD_FILE_NAME_OFFSET,
-      headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength)
-    )
+  let compressedSize = BigInt(await file.getUint32(headerOffset + CD_COMPRESSED_SIZE_OFFSET));
+  let uncompressedSize = BigInt(await file.getUint32(headerOffset + CD_UNCOMPRESSED_SIZE_OFFSET));
+  const extraFieldLength = await file.getUint16(headerOffset + CD_EXTRA_FIELD_LENGTH_OFFSET);
+  const fileNameLength = await file.getUint16(headerOffset + CD_FILE_NAME_LENGTH_OFFSET);
+  const filenameBytes = await file.slice(
+    headerOffset + CD_FILE_NAME_OFFSET,
+    headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength)
   );
+  const fileName = new TextDecoder().decode(filenameBytes);
 
   const extraOffset = headerOffset + CD_FILE_NAME_OFFSET + BigInt(fileNameLength);
-
-  const oldFormatOffset = await buffer.getUint32(headerOffset + CD_LOCAL_HEADER_OFFSET_OFFSET);
+  const oldFormatOffset = await file.getUint32(headerOffset + CD_LOCAL_HEADER_OFFSET_OFFSET);
 
   let fileDataOffset = BigInt(oldFormatOffset);
   let offsetInZip64Data = 4n;
   // looking for info that might be also be in zip64 extra field
   if (uncompressedSize === BigInt(0xffffffff)) {
-    uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    uncompressedSize = await file.getBigUint64(extraOffset + offsetInZip64Data);
     offsetInZip64Data += 8n;
   }
   if (compressedSize === BigInt(0xffffffff)) {
-    compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+    compressedSize = await file.getBigUint64(extraOffset + offsetInZip64Data);
     offsetInZip64Data += 8n;
   }
   if (fileDataOffset === BigInt(0xffffffff)) {
-    fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
+    fileDataOffset = await file.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
   }
   const localHeaderOffset = fileDataOffset;
 
@@ -101,7 +92,9 @@ export const parseZipCDFileHeader = async (
  * Create iterator over files of zip archive
  * @param fileProvider - file provider that provider random access to the file
  */
-export async function* zipCDFileHeaderGenerator(fileProvider: FileProvider) {
+export async function* makeZipCDHeaderIterator(
+  fileProvider: FileProvider
+): AsyncIterable<ZipCDFileHeader> {
   const {cdStartOffset} = await parseEoCDRecord(fileProvider);
   let cdHeader = await parseZipCDFileHeader(cdStartOffset, fileProvider);
   while (cdHeader) {

package/src/parse-zip/end-of-central-directory.ts

@@ -1,4 +1,4 @@
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
 import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
 
 /**
@@ -12,9 +12,9 @@ export type ZipEoCDRecord = {
   cdRecordsNumber: bigint;
 };
 
-const eoCDSignature: ZipSignature = [0x50, 0x4b, 0x05, 0x06];
-const zip64EoCDLocatorSignature = [0x50, 0x4b, 0x06, 0x07];
-const zip64EoCDSignature = [0x50, 0x4b, 0x06, 0x06];
+const eoCDSignature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x05, 0x06]);
+const zip64EoCDLocatorSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x07]);
+const zip64EoCDSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x06]);
 
 // offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format)
 const CD_RECORDS_NUMBER_OFFSET = 8n;
@@ -25,43 +25,33 @@ const ZIP64_CD_START_OFFSET_OFFSET = 48n;
 
 /**
  * Parses end of central directory record of zip file
- * @param fileProvider - FileProvider instance
+ * @param file - FileProvider instance
  * @returns Info from the header
  */
-export const parseEoCDRecord = async (fileProvider: FileProvider): Promise<ZipEoCDRecord> => {
-  const zipEoCDOffset = await searchFromTheEnd(fileProvider, eoCDSignature);
+export const parseEoCDRecord = async (file: FileProvider): Promise<ZipEoCDRecord> => {
+  const zipEoCDOffset = await searchFromTheEnd(file, eoCDSignature);
 
-  let cdRecordsNumber = BigInt(
-    await fileProvider.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET)
-  );
-  let cdStartOffset = BigInt(await fileProvider.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET));
+  let cdRecordsNumber = BigInt(await file.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET));
+  let cdStartOffset = BigInt(await file.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET));
 
   if (cdStartOffset === BigInt(0xffffffff) || cdRecordsNumber === BigInt(0xffffffff)) {
     const zip64EoCDLocatorOffset = zipEoCDOffset - 20n;
 
-    if (
-      Buffer.from(
-        await fileProvider.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n)
-      ).compare(Buffer.from(zip64EoCDLocatorSignature)) !== 0
-    ) {
+    const magicBytes = await file.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n);
+    if (!compareArrayBuffers(magicBytes, zip64EoCDLocatorSignature)) {
       throw new Error('zip64 EoCD locator not found');
     }
-    const zip64EoCDOffset = await fileProvider.getBigUint64(
+    const zip64EoCDOffset = await file.getBigUint64(
       zip64EoCDLocatorOffset + ZIP64_EOCD_START_OFFSET_OFFSET
     );
 
-    if (
-      Buffer.from(await fileProvider.slice(zip64EoCDOffset, zip64EoCDOffset + 4n)).compare(
-        Buffer.from(zip64EoCDSignature)
-      ) !== 0
-    ) {
+    const endOfCDMagicBytes = await file.slice(zip64EoCDOffset, zip64EoCDOffset + 4n);
+    if (!compareArrayBuffers(endOfCDMagicBytes, zip64EoCDSignature.buffer)) {
       throw new Error('zip64 EoCD not found');
     }
 
-    cdRecordsNumber = await fileProvider.getBigUint64(
-      zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET
-    );
-    cdStartOffset = await fileProvider.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET);
+    cdRecordsNumber = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET);
+    cdStartOffset = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET);
   }
 
   return {

package/src/parse-zip/local-file-header.ts

@@ -1,4 +1,5 @@
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
+import {ZipSignature} from './search-from-the-end';
 
 /**
  * zip local file header info
@@ -27,7 +28,7 @@ const FILE_NAME_LENGTH_OFFSET = 26n;
 const EXTRA_FIELD_LENGTH_OFFSET = 28n;
 const FILE_NAME_OFFSET = 30n;
 
-export const signature = [0x50, 0x4b, 0x03, 0x04];
+export const signature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x03, 0x04]);
 
 /**
  * Parses local file header of zip file
@@ -39,11 +40,8 @@ export const parseZipLocalFileHeader = async (
   headerOffset: bigint,
   buffer: FileProvider
 ): Promise<ZipLocalFileHeader | null> => {
-  if (
-    Buffer.from(await buffer.slice(headerOffset, headerOffset + 4n)).compare(
-      Buffer.from(signature)
-    ) !== 0
-  ) {
+  const magicBytes = await buffer.slice(headerOffset, headerOffset + 4n);
+  if (!compareArrayBuffers(magicBytes, signature)) {
     return null;
   }
 

package/src/parse-zip/search-from-the-end.ts

@@ -1,7 +1,7 @@
 import {FileProvider} from '@loaders.gl/loader-utils';
 
 /** Description of zip signature type */
-export type ZipSignature = [number, number, number, number];
+export type ZipSignature = Uint8Array;
 
 /**
  * looking for the last occurrence of the provided

package/dist/bundle.d.ts

@@ -1,2 +0,0 @@
-declare const moduleExports: any;
-//# sourceMappingURL=bundle.d.ts.map

package/dist/bundle.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"bundle.d.ts","sourceRoot":"","sources":["../src/bundle.ts"],"names":[],"mappings":"AACA,QAAA,MAAM,aAAa,KAAqB,CAAC"}

package/dist/es5/bundle.js

@@ -1,6 +0,0 @@
-"use strict";
-
-var moduleExports = require('./index');
-globalThis.loaders = globalThis.loaders || {};
-module.exports = Object.assign(globalThis.loaders, moduleExports);
-//# sourceMappingURL=bundle.js.map

package/dist/es5/bundle.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"bundle.js","names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"sources":["../../src/bundle.ts"],"sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"mappings":";;AACA,IAAMA,aAAa,GAAGC,OAAO,CAAC,SAAS,CAAC;AACxCC,UAAU,CAACC,OAAO,GAAGD,UAAU,CAACC,OAAO,IAAI,CAAC,CAAC;AAC7CC,MAAM,CAACC,OAAO,GAAGC,MAAM,CAACC,MAAM,CAACL,UAAU,CAACC,OAAO,EAAEH,aAAa,CAAC"}