@loaders.gl/zip 4.2.0-alpha.4 → 4.2.0-alpha.5

package/dist/parse-zip/local-file-header.js

@@ -1,5 +1,9 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 import { compareArrayBuffers, concatenateArrayBuffers } from '@loaders.gl/loader-utils';
 import { createZip64Info, setFieldToNumber } from "./zip64-info-generation.js";
+// offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format)
 const COMPRESSION_METHOD_OFFSET = 8n;
 const COMPRESSED_SIZE_OFFSET = 18n;
 const UNCOMPRESSED_SIZE_OFFSET = 22n;
@@ -7,109 +11,146 @@ const FILE_NAME_LENGTH_OFFSET = 26n;
 const EXTRA_FIELD_LENGTH_OFFSET = 28n;
 const FILE_NAME_OFFSET = 30n;
 export const signature = new Uint8Array([0x50, 0x4b, 0x03, 0x04]);
+/**
+ * Parses local file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
 export const parseZipLocalFileHeader = async (headerOffset, buffer) => {
-  const magicBytes = await buffer.slice(headerOffset, headerOffset + 4n);
-  if (!compareArrayBuffers(magicBytes, signature)) {
-    return null;
-  }
-  const fileNameLength = await buffer.getUint16(headerOffset + FILE_NAME_LENGTH_OFFSET);
-  const fileName = new TextDecoder().decode(await buffer.slice(headerOffset + FILE_NAME_OFFSET, headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength))).split('\\').join('/');
-  const extraFieldLength = await buffer.getUint16(headerOffset + EXTRA_FIELD_LENGTH_OFFSET);
-  let fileDataOffset = headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength);
-  const compressionMethod = await buffer.getUint16(headerOffset + COMPRESSION_METHOD_OFFSET);
-  let compressedSize = BigInt(await buffer.getUint32(headerOffset + COMPRESSED_SIZE_OFFSET));
-  let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + UNCOMPRESSED_SIZE_OFFSET));
-  const extraOffset = headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength);
-  let offsetInZip64Data = 4n;
-  if (uncompressedSize === BigInt(0xffffffff)) {
-    uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
-    offsetInZip64Data += 8n;
-  }
-  if (compressedSize === BigInt(0xffffffff)) {
-    compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
-    offsetInZip64Data += 8n;
-  }
-  if (fileDataOffset === BigInt(0xffffffff)) {
-    fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
-  }
-  return {
-    fileNameLength,
-    fileName,
-    extraFieldLength,
-    fileDataOffset,
-    compressedSize,
-    compressionMethod
-  };
+    const magicBytes = await buffer.slice(headerOffset, headerOffset + 4n);
+    if (!compareArrayBuffers(magicBytes, signature)) {
+        return null;
+    }
+    const fileNameLength = await buffer.getUint16(headerOffset + FILE_NAME_LENGTH_OFFSET);
+    const fileName = new TextDecoder()
+        .decode(await buffer.slice(headerOffset + FILE_NAME_OFFSET, headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength)))
+        .split('\\')
+        .join('/');
+    const extraFieldLength = await buffer.getUint16(headerOffset + EXTRA_FIELD_LENGTH_OFFSET);
+    let fileDataOffset = headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength + extraFieldLength);
+    const compressionMethod = await buffer.getUint16(headerOffset + COMPRESSION_METHOD_OFFSET);
+    let compressedSize = BigInt(await buffer.getUint32(headerOffset + COMPRESSED_SIZE_OFFSET)); // add zip 64 logic
+    let uncompressedSize = BigInt(await buffer.getUint32(headerOffset + UNCOMPRESSED_SIZE_OFFSET)); // add zip 64 logic
+    const extraOffset = headerOffset + FILE_NAME_OFFSET + BigInt(fileNameLength);
+    let offsetInZip64Data = 4n;
+    // looking for info that might be also be in zip64 extra field
+    if (uncompressedSize === BigInt(0xffffffff)) {
+        uncompressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+        offsetInZip64Data += 8n;
+    }
+    if (compressedSize === BigInt(0xffffffff)) {
+        compressedSize = await buffer.getBigUint64(extraOffset + offsetInZip64Data);
+        offsetInZip64Data += 8n;
+    }
+    if (fileDataOffset === BigInt(0xffffffff)) {
+        fileDataOffset = await buffer.getBigUint64(extraOffset + offsetInZip64Data); // setting it to the one from zip64
+    }
+    return {
+        fileNameLength,
+        fileName,
+        extraFieldLength,
+        fileDataOffset,
+        compressedSize,
+        compressionMethod
+    };
 };
+/**
+ * generates local header for the file
+ * @param options info that can be placed into local header
+ * @returns buffer with header
+ */
 export function generateLocalHeader(options) {
-  const optionsToUse = {
-    ...options,
-    extraLength: 0,
-    fnlength: options.fileName.length
-  };
-  let zip64header = new ArrayBuffer(0);
-  const optionsToZip64 = {};
-  if (optionsToUse.length >= 0xffffffff) {
-    optionsToZip64.size = optionsToUse.length;
-    optionsToUse.length = 0xffffffff;
-  }
-  if (Object.keys(optionsToZip64).length) {
-    zip64header = createZip64Info(optionsToZip64);
-    optionsToUse.extraLength = zip64header.byteLength;
-  }
-  const header = new DataView(new ArrayBuffer(Number(FILE_NAME_OFFSET)));
-  for (const field of ZIP_HEADER_FIELDS) {
-    var _ref, _optionsToUse, _field$name;
-    setFieldToNumber(header, field.size, field.offset, (_ref = (_optionsToUse = optionsToUse[(_field$name = field.name) !== null && _field$name !== void 0 ? _field$name : '']) !== null && _optionsToUse !== void 0 ? _optionsToUse : field.default) !== null && _ref !== void 0 ? _ref : 0);
-  }
-  const encodedName = new TextEncoder().encode(optionsToUse.fileName);
-  const resHeader = concatenateArrayBuffers(header.buffer, encodedName, zip64header);
-  return resHeader;
+    const optionsToUse = {
+        ...options,
+        extraLength: 0,
+        fnlength: options.fileName.length
+    };
+    let zip64header = new ArrayBuffer(0);
+    const optionsToZip64 = {};
+    if (optionsToUse.length >= 0xffffffff) {
+        optionsToZip64.size = optionsToUse.length;
+        optionsToUse.length = 0xffffffff;
+    }
+    if (Object.keys(optionsToZip64).length) {
+        zip64header = createZip64Info(optionsToZip64);
+        optionsToUse.extraLength = zip64header.byteLength;
+    }
+    // base length without file name and extra info is static
+    const header = new DataView(new ArrayBuffer(Number(FILE_NAME_OFFSET)));
+    for (const field of ZIP_HEADER_FIELDS) {
+        setFieldToNumber(header, field.size, field.offset, optionsToUse[field.name ?? ''] ?? field.default ?? 0);
+    }
+    const encodedName = new TextEncoder().encode(optionsToUse.fileName);
+    const resHeader = concatenateArrayBuffers(header.buffer, encodedName, zip64header);
+    return resHeader;
 }
-const ZIP_HEADER_FIELDS = [{
-  offset: 0,
-  size: 4,
-  default: new DataView(signature.buffer).getUint32(0, true)
-}, {
-  offset: 4,
-  size: 2,
-  default: 45
-}, {
-  offset: 6,
-  size: 2,
-  default: 0
-}, {
-  offset: 8,
-  size: 2,
-  default: 0
-}, {
-  offset: 10,
-  size: 2,
-  default: 0
-}, {
-  offset: 12,
-  size: 2,
-  default: 0
-}, {
-  offset: 14,
-  size: 4,
-  name: 'crc32'
-}, {
-  offset: 18,
-  size: 4,
-  name: 'length'
-}, {
-  offset: 22,
-  size: 4,
-  name: 'length'
-}, {
-  offset: 26,
-  size: 2,
-  name: 'fnlength'
-}, {
-  offset: 28,
-  size: 2,
-  default: 0,
-  name: 'extraLength'
-}];
-//# sourceMappingURL=local-file-header.js.map
+const ZIP_HEADER_FIELDS = [
+    // Local file header signature = 0x04034b50
+    {
+        offset: 0,
+        size: 4,
+        default: new DataView(signature.buffer).getUint32(0, true)
+    },
+    // Version needed to extract (minimum)
+    {
+        offset: 4,
+        size: 2,
+        default: 45
+    },
+    // General purpose bit flag
+    {
+        offset: 6,
+        size: 2,
+        default: 0
+    },
+    // Compression method
+    {
+        offset: 8,
+        size: 2,
+        default: 0
+    },
+    // File last modification time
+    {
+        offset: 10,
+        size: 2,
+        default: 0
+    },
+    // File last modification date
+    {
+        offset: 12,
+        size: 2,
+        default: 0
+    },
+    // CRC-32 of uncompressed data
+    {
+        offset: 14,
+        size: 4,
+        name: 'crc32'
+    },
+    // Compressed size (or 0xffffffff for ZIP64)
+    {
+        offset: 18,
+        size: 4,
+        name: 'length'
+    },
+    // Uncompressed size (or 0xffffffff for ZIP64)
+    {
+        offset: 22,
+        size: 4,
+        name: 'length'
+    },
+    // File name length (n)
+    {
+        offset: 26,
+        size: 2,
+        name: 'fnlength'
+    },
+    // Extra field length (m)
+    {
+        offset: 28,
+        size: 2,
+        default: 0,
+        name: 'extraLength'
+    }
+];

package/dist/parse-zip/search-from-the-end.js

@@ -1,16 +1,30 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+/**
+ * looking for the last occurrence of the provided
+ * @param file
+ * @param target
+ * @returns
+ */
 export const searchFromTheEnd = async (file, target) => {
-  const searchWindow = [await file.getUint8(file.length - 1n), await file.getUint8(file.length - 2n), await file.getUint8(file.length - 3n), undefined];
-  let targetOffset = 0n;
-  for (let i = file.length - 4n; i > -1; i--) {
-    searchWindow[3] = searchWindow[2];
-    searchWindow[2] = searchWindow[1];
-    searchWindow[1] = searchWindow[0];
-    searchWindow[0] = await file.getUint8(i);
-    if (searchWindow.every((val, index) => val === target[index])) {
-      targetOffset = i;
-      break;
+    const searchWindow = [
+        await file.getUint8(file.length - 1n),
+        await file.getUint8(file.length - 2n),
+        await file.getUint8(file.length - 3n),
+        undefined
+    ];
+    let targetOffset = 0n;
+    // looking for the last record in the central directory
+    for (let i = file.length - 4n; i > -1; i--) {
+        searchWindow[3] = searchWindow[2];
+        searchWindow[2] = searchWindow[1];
+        searchWindow[1] = searchWindow[0];
+        searchWindow[0] = await file.getUint8(i);
+        if (searchWindow.every((val, index) => val === target[index])) {
+            targetOffset = i;
+            break;
+        }
     }
-  }
-  return targetOffset;
+    return targetOffset;
 };
-//# sourceMappingURL=search-from-the-end.js.map

package/dist/parse-zip/zip-composition.js

@@ -4,111 +4,161 @@ import { CRC32Hash } from '@loaders.gl/crypto';
 import { generateLocalHeader } from "./local-file-header.js";
 import { generateCDHeader } from "./cd-file-header.js";
 import { fetchFile } from '@loaders.gl/core';
+/**
+ * cut off CD and EoCD records from zip file
+ * @param provider zip file
+ * @returns tuple with three values: CD, EoCD record, EoCD information
+ */
 async function cutTheTailOff(provider) {
-  const oldEoCDinfo = await parseEoCDRecord(provider);
-  const oldCDStartOffset = oldEoCDinfo.cdStartOffset;
-  const oldCDLength = Number(oldEoCDinfo.offsets.zip64EoCDOffset ? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset : oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset);
-  const zipEnding = await provider.slice(oldCDStartOffset, provider.length);
-  await provider.truncate(Number(oldCDStartOffset));
-  const oldCDBody = zipEnding.slice(0, oldCDLength);
-  const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength);
-  return [oldCDBody, eocdBody, oldEoCDinfo];
+    // define where the body ends
+    const oldEoCDinfo = await parseEoCDRecord(provider);
+    const oldCDStartOffset = oldEoCDinfo.cdStartOffset;
+    // define cd length
+    const oldCDLength = Number(oldEoCDinfo.offsets.zip64EoCDOffset
+        ? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset
+        : oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset);
+    // cut off everything except of archieve body
+    const zipEnding = await provider.slice(oldCDStartOffset, provider.length);
+    await provider.truncate(Number(oldCDStartOffset));
+    // divide cd body and eocd record
+    const oldCDBody = zipEnding.slice(0, oldCDLength);
+    const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength);
+    return [oldCDBody, eocdBody, oldEoCDinfo];
 }
+/**
+ * generates CD and local headers for the file
+ * @param fileName name of the file
+ * @param fileToAdd buffer with the file
+ * @param localFileHeaderOffset offset of the file local header
+ * @returns tuple with two values: local header and file body, cd header
+ */
 async function generateFileHeaders(fileName, fileToAdd, localFileHeaderOffset) {
-  const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16);
-  const newFileLocalHeader = generateLocalHeader({
-    crc32: newFileCRC322,
-    fileName,
-    length: fileToAdd.byteLength
-  });
-  const newFileCDHeader = generateCDHeader({
-    crc32: newFileCRC322,
-    fileName,
-    offset: localFileHeaderOffset,
-    length: fileToAdd.byteLength
-  });
-  return [new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)), new Uint8Array(newFileCDHeader)];
+    // generating CRC32 of the content
+    const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16);
+    // generate local header for the file
+    const newFileLocalHeader = generateLocalHeader({
+        crc32: newFileCRC322,
+        fileName,
+        length: fileToAdd.byteLength
+    });
+    // generate hash file cd header
+    const newFileCDHeader = generateCDHeader({
+        crc32: newFileCRC322,
+        fileName,
+        offset: localFileHeaderOffset,
+        length: fileToAdd.byteLength
+    });
+    return [
+        new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)),
+        new Uint8Array(newFileCDHeader)
+    ];
 }
+/**
+ * adds one file in the end of the archieve
+ * @param zipUrl path to the file
+ * @param fileToAdd new file body
+ * @param fileName new file name
+ */
 export async function addOneFile(zipUrl, fileToAdd, fileName) {
-  const provider = new FileHandleFile(zipUrl, true);
-  const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider);
-  const newFileOffset = provider.length;
-  const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset);
-  await provider.append(localPart);
-  const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart);
-  const newCDStartOffset = provider.length;
-  await provider.append(new Uint8Array(newCDBody));
-  const eocdOffset = provider.length;
-  await provider.append(updateEoCD(eocdBody, oldEoCDinfo.offsets, newCDStartOffset, eocdOffset, oldEoCDinfo.cdRecordsNumber + 1n));
+    // init file handler
+    const provider = new FileHandleFile(zipUrl, true);
+    const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider);
+    // remember the new file local header start offset
+    const newFileOffset = provider.length;
+    const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset);
+    // write down the file local header
+    await provider.append(localPart);
+    // add the file CD header to the CD
+    const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart);
+    // remember the CD start offset
+    const newCDStartOffset = provider.length;
+    // write down new CD
+    await provider.append(new Uint8Array(newCDBody));
+    // remember where eocd starts
+    const eocdOffset = provider.length;
+    await provider.append(updateEoCD(eocdBody, oldEoCDinfo.offsets, newCDStartOffset, eocdOffset, oldEoCDinfo.cdRecordsNumber + 1n));
 }
+/**
+ * creates zip archive with no compression
+ * @note This is a node specific function that works on files
+ * @param inputPath path where files for the achive are stored
+ * @param outputPath path where zip archive will be placed
+ */
 export async function createZip(inputPath, outputPath, createAdditionalData) {
-  const fileIterator = getFileIterator(inputPath);
-  const resFile = new NodeFile(outputPath, 'w');
-  const fileList = [];
-  const cdArray = [];
-  for await (const file of fileIterator) {
-    await addFile(file, resFile, cdArray, fileList);
-  }
-  if (createAdditionalData) {
-    const additionaldata = await createAdditionalData(fileList);
-    await addFile(additionaldata, resFile, cdArray);
-  }
-  const cdOffset = (await resFile.stat()).bigsize;
-  const cd = concatenateArrayBuffers(...cdArray);
-  await resFile.append(new Uint8Array(cd));
-  const eoCDStart = (await resFile.stat()).bigsize;
-  await resFile.append(new Uint8Array(generateEoCD({
-    recordsNumber: cdArray.length,
-    cdSize: cd.byteLength,
-    cdOffset,
-    eoCDStart
-  })));
+    const fileIterator = getFileIterator(inputPath);
+    const resFile = new NodeFile(outputPath, 'w');
+    const fileList = [];
+    const cdArray = [];
+    for await (const file of fileIterator) {
+        await addFile(file, resFile, cdArray, fileList);
+    }
+    if (createAdditionalData) {
+        const additionaldata = await createAdditionalData(fileList);
+        await addFile(additionaldata, resFile, cdArray);
+    }
+    const cdOffset = (await resFile.stat()).bigsize;
+    const cd = concatenateArrayBuffers(...cdArray);
+    await resFile.append(new Uint8Array(cd));
+    const eoCDStart = (await resFile.stat()).bigsize;
+    await resFile.append(new Uint8Array(generateEoCD({ recordsNumber: cdArray.length, cdSize: cd.byteLength, cdOffset, eoCDStart })));
 }
+/**
+ * Adds file to zip parts
+ * @param file file to add
+ * @param resFile zip file body
+ * @param cdArray zip file central directory
+ * @param fileList list of file offsets
+ */
 async function addFile(file, resFile, cdArray, fileList) {
-  const size = (await resFile.stat()).bigsize;
-  fileList === null || fileList === void 0 ? void 0 : fileList.push({
-    fileName: file.path,
-    localHeaderOffset: size
-  });
-  const [localPart, cdHeaderPart] = await generateFileHeaders(file.path, file.file, size);
-  await resFile.append(localPart);
-  cdArray.push(cdHeaderPart);
+    const size = (await resFile.stat()).bigsize;
+    fileList?.push({ fileName: file.path, localHeaderOffset: size });
+    const [localPart, cdHeaderPart] = await generateFileHeaders(file.path, file.file, size);
+    await resFile.append(localPart);
+    cdArray.push(cdHeaderPart);
 }
+/**
+ * creates iterator providing buffer with file content and path to every file in the input folder
+ * @param inputPath path to the input folder
+ * @returns iterator
+ */
 export function getFileIterator(inputPath) {
-  async function* iterable() {
-    const fileList = await getAllFiles(inputPath);
-    for (const filePath of fileList) {
-      const file = await (await fetchFile(path.join(inputPath, filePath))).arrayBuffer();
-      yield {
-        path: filePath,
-        file
-      };
+    async function* iterable() {
+        const fileList = await getAllFiles(inputPath);
+        for (const filePath of fileList) {
+            const file = await (await fetchFile(path.join(inputPath, filePath))).arrayBuffer();
+            yield { path: filePath, file };
+        }
     }
-  }
-  return iterable();
+    return iterable();
 }
-export async function getAllFiles(basePath) {
-  let subfolder = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : '';
-  let fsPassed = arguments.length > 2 ? arguments[2] : undefined;
-  const fs = fsPassed ? fsPassed : new NodeFilesystem({});
-  const files = await fs.readdir(pathJoin(basePath, subfolder));
-  const arrayOfFiles = [];
-  for (const file of files) {
-    const fullPath = pathJoin(basePath, subfolder, file);
-    if ((await fs.stat(fullPath)).isDirectory) {
-      const files = await getAllFiles(basePath, pathJoin(subfolder, file));
-      arrayOfFiles.push(...files);
-    } else {
-      arrayOfFiles.push(pathJoin(subfolder, file));
+/**
+ * creates a list of relative paths to all files in the provided folder
+ * @param basePath path of the root folder
+ * @param subfolder relative path from the root folder.
+ * @returns list of paths
+ */
+export async function getAllFiles(basePath, subfolder = '', fsPassed) {
+    const fs = fsPassed ? fsPassed : new NodeFilesystem({});
+    const files = await fs.readdir(pathJoin(basePath, subfolder));
+    const arrayOfFiles = [];
+    for (const file of files) {
+        const fullPath = pathJoin(basePath, subfolder, file);
+        if ((await fs.stat(fullPath)).isDirectory) {
+            const files = await getAllFiles(basePath, pathJoin(subfolder, file));
+            arrayOfFiles.push(...files);
+        }
+        else {
+            arrayOfFiles.push(pathJoin(subfolder, file));
+        }
     }
-  }
-  return arrayOfFiles;
+    return arrayOfFiles;
 }
-function pathJoin() {
-  for (var _len = arguments.length, paths = new Array(_len), _key = 0; _key < _len; _key++) {
-    paths[_key] = arguments[_key];
-  }
-  const resPaths = paths.filter(val => val.length);
-  return path.join(...resPaths);
+/**
+ * removes empty parts from path array and joins it
+ * @param paths paths to join
+ * @returns joined path
+ */
+function pathJoin(...paths) {
+    const resPaths = paths.filter((val) => val.length);
+    return path.join(...resPaths);
 }
-//# sourceMappingURL=zip-composition.js.map