@loaders.gl/zip 4.0.4 → 4.1.0-alpha.10

package/src/lib/tar/tar.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 // This file is derived from the tar-js code base under MIT license

package/src/lib/tar/types.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 /**

package/src/lib/tar/utils.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 // This file is derived from the tar-js code base under MIT license

package/src/parse-zip/cd-file-header.ts

@@ -1,9 +1,11 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
+import {FileProvider, compareArrayBuffers, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
 import {parseEoCDRecord} from './end-of-central-directory';
 import {ZipSignature} from './search-from-the-end';
+import {createZip64Info, setFieldToNumber} from './zip64-info-generation';
 
 /**
  * zip central directory file header info
@@ -188,3 +190,184 @@ const findExpectedData = (zip64data: Zip64Data): {length: number; name: string}[
 
   return zip64dataList;
 };
+
+/** info that can be placed into cd header */
+type GenerateCDOptions = {
+  /** CRC-32 of uncompressed data */
+  crc32: number;
+  /** File name */
+  fileName: string;
+  /** File size */
+  length: number;
+  /** Relative offset of local file header */
+  offset: bigint;
+};
+
+/**
+ * generates cd header for the file
+ * @param options info that can be placed into cd header
+ * @returns buffer with header
+ */
+export function generateCDHeader(options: GenerateCDOptions): ArrayBuffer {
+  const optionsToUse = {
+    ...options,
+    fnlength: options.fileName.length,
+    extraLength: 0
+  };
+
+  let zip64header: ArrayBuffer = new ArrayBuffer(0);
+
+  const optionsToZip64: any = {};
+  if (optionsToUse.offset >= 0xffffffff) {
+    optionsToZip64.offset = optionsToUse.offset;
+    optionsToUse.offset = BigInt(0xffffffff);
+  }
+  if (optionsToUse.length >= 0xffffffff) {
+    optionsToZip64.size = optionsToUse.length;
+    optionsToUse.length = 0xffffffff;
+  }
+
+  if (Object.keys(optionsToZip64).length) {
+    zip64header = createZip64Info(optionsToZip64);
+    optionsToUse.extraLength = zip64header.byteLength;
+  }
+  const header = new DataView(new ArrayBuffer(Number(CD_FILE_NAME_OFFSET)));
+
+  for (const field of ZIP_HEADER_FIELDS) {
+    setFieldToNumber(
+      header,
+      field.size,
+      field.offset,
+      optionsToUse[field.name ?? ''] ?? field.default ?? 0
+    );
+  }
+
+  const encodedName = new TextEncoder().encode(optionsToUse.fileName);
+
+  const resHeader = concatenateArrayBuffers(header.buffer, encodedName, zip64header);
+
+  return resHeader;
+}
+
+/** Fields map */
+const ZIP_HEADER_FIELDS = [
+  // Central directory file header signature = 0x02014b50
+  {
+    offset: 0,
+    size: 4,
+    default: new DataView(signature.buffer).getUint32(0, true)
+  },
+
+  // Version made by
+  {
+    offset: 4,
+    size: 2,
+    default: 45
+  },
+
+  // Version needed to extract (minimum)
+  {
+    offset: 6,
+    size: 2,
+    default: 45
+  },
+
+  // General purpose bit flag
+  {
+    offset: 8,
+    size: 2,
+    default: 0
+  },
+
+  // Compression method
+  {
+    offset: 10,
+    size: 2,
+    default: 0
+  },
+
+  // File last modification time
+  {
+    offset: 12,
+    size: 2,
+    default: 0
+  },
+
+  // File last modification date
+  {
+    offset: 14,
+    size: 2,
+    default: 0
+  },
+
+  // CRC-32 of uncompressed data
+  {
+    offset: 16,
+    size: 4,
+    name: 'crc32'
+  },
+
+  // Compressed size (or 0xffffffff for ZIP64)
+  {
+    offset: 20,
+    size: 4,
+    name: 'length'
+  },
+
+  // Uncompressed size (or 0xffffffff for ZIP64)
+  {
+    offset: 24,
+    size: 4,
+    name: 'length'
+  },
+
+  // File name length (n)
+  {
+    offset: 28,
+    size: 2,
+    name: 'fnlength'
+  },
+
+  // Extra field length (m)
+  {
+    offset: 30,
+    size: 2,
+    default: 0,
+    name: 'extraLength'
+  },
+
+  // File comment length (k)
+  {
+    offset: 32,
+    size: 2,
+    default: 0
+  },
+
+  // Disk number where file starts (or 0xffff for ZIP64)
+  {
+    offset: 34,
+    size: 2,
+    default: 0
+  },
+
+  // Internal file attributes
+  {
+    offset: 36,
+    size: 2,
+    default: 0
+  },
+
+  // External file attributes
+  {
+    offset: 38,
+    size: 4,
+    default: 0
+  },
+
+  // Relative offset of local file header
+  {
+    offset: 42,
+    size: 4,
+    name: 'offset'
+  }
+];

package/src/parse-zip/end-of-central-directory.ts

@@ -1,8 +1,10 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
 import {ZipSignature, searchFromTheEnd} from './search-from-the-end';
+import {setFieldToNumber} from './zip64-info-generation';
 
 /**
  * End of central directory info
@@ -13,6 +15,18 @@ export type ZipEoCDRecord = {
   cdStartOffset: bigint;
   /** Relative offset of local file header */
   cdRecordsNumber: bigint;
+  offsets: ZipEoCDRecordOffsets;
+};
+
+/**
+ * End of central directory offsets
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipEoCDRecordOffsets = {
+  zipEoCDOffset: bigint;
+
+  zip64EoCDOffset?: bigint;
+  zip64EoCDLocatorOffset?: bigint;
 };
 
 const eoCDSignature: ZipSignature = new Uint8Array([0x50, 0x4b, 0x05, 0x06]);
@@ -21,9 +35,13 @@ const zip64EoCDSignature = new Uint8Array([0x50, 0x4b, 0x06, 0x06]);
 
 // offsets accroding to https://en.wikipedia.org/wiki/ZIP_(file_format)
 const CD_RECORDS_NUMBER_OFFSET = 8n;
+const CD_RECORDS_NUMBER_ON_DISC_OFFSET = 10n;
+const CD_CD_BYTE_SIZE_OFFSET = 12n;
 const CD_START_OFFSET_OFFSET = 16n;
 const ZIP64_EOCD_START_OFFSET_OFFSET = 8n;
 const ZIP64_CD_RECORDS_NUMBER_OFFSET = 24n;
+const ZIP64_CD_RECORDS_NUMBER_ON_DISC_OFFSET = 32n;
+const ZIP64_CD_CD_BYTE_SIZE_OFFSET = 40n;
 const ZIP64_CD_START_OFFSET_OFFSET = 48n;
 
 /**
@@ -37,14 +55,12 @@ export const parseEoCDRecord = async (file: FileProvider): Promise<ZipEoCDRecord
   let cdRecordsNumber = BigInt(await file.getUint16(zipEoCDOffset + CD_RECORDS_NUMBER_OFFSET));
   let cdStartOffset = BigInt(await file.getUint32(zipEoCDOffset + CD_START_OFFSET_OFFSET));
 
-  if (cdStartOffset === BigInt(0xffffffff) || cdRecordsNumber === BigInt(0xffffffff)) {
-    const zip64EoCDLocatorOffset = zipEoCDOffset - 20n;
+  let zip64EoCDLocatorOffset = zipEoCDOffset - 20n;
+  let zip64EoCDOffset = 0n;
 
-    const magicBytes = await file.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n);
-    if (!compareArrayBuffers(magicBytes, zip64EoCDLocatorSignature)) {
-      throw new Error('zip64 EoCD locator not found');
-    }
-    const zip64EoCDOffset = await file.getBigUint64(
+  const magicBytes = await file.slice(zip64EoCDLocatorOffset, zip64EoCDLocatorOffset + 4n);
+  if (compareArrayBuffers(magicBytes, zip64EoCDLocatorSignature)) {
+    zip64EoCDOffset = await file.getBigUint64(
       zip64EoCDLocatorOffset + ZIP64_EOCD_START_OFFSET_OFFSET
     );
 
@@ -55,10 +71,84 @@ export const parseEoCDRecord = async (file: FileProvider): Promise<ZipEoCDRecord
 
     cdRecordsNumber = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_RECORDS_NUMBER_OFFSET);
     cdStartOffset = await file.getBigUint64(zip64EoCDOffset + ZIP64_CD_START_OFFSET_OFFSET);
+  } else {
+    zip64EoCDLocatorOffset = 0n;
   }
 
   return {
     cdRecordsNumber,
-    cdStartOffset
+    cdStartOffset,
+    offsets: {
+      zip64EoCDOffset,
+      zip64EoCDLocatorOffset,
+      zipEoCDOffset
+    }
   };
 };
+
+/**
+ * updates EoCD record to add more files to the archieve
+ * @param eocdBody buffer containing header
+ * @param oldEoCDOffsets info read from EoCD record befor updating
+ * @param newCDStartOffset CD start offset to be updated
+ * @param eocdStartOffset EoCD start offset to be updated
+ * @returns new EoCD header
+ */
+export async function updateEoCD(
+  eocdBody: ArrayBuffer,
+  oldEoCDOffsets: ZipEoCDRecordOffsets,
+  newCDStartOffset: bigint,
+  eocdStartOffset: bigint,
+  newCDRecordsNumber: bigint
+): Promise<Uint8Array> {
+  const eocd = new DataView(eocdBody);
+
+  const classicEoCDOffset = oldEoCDOffsets.zip64EoCDOffset
+    ? oldEoCDOffsets.zipEoCDOffset - oldEoCDOffsets.zip64EoCDOffset
+    : 0n;
+
+  // updating classic EoCD record with new CD records number in general and on disc
+  if (Number(newCDRecordsNumber) <= 0xffff) {
+    setFieldToNumber(eocd, 2, classicEoCDOffset + CD_RECORDS_NUMBER_OFFSET, newCDRecordsNumber);
+    setFieldToNumber(
+      eocd,
+      2,
+      classicEoCDOffset + CD_RECORDS_NUMBER_ON_DISC_OFFSET,
+      newCDRecordsNumber
+    );
+  }
+
+  // updating zip64 EoCD record with new size of CD
+  if (eocdStartOffset - newCDStartOffset <= 0xffffffff) {
+    setFieldToNumber(
+      eocd,
+      4,
+      classicEoCDOffset + CD_CD_BYTE_SIZE_OFFSET,
+      eocdStartOffset - newCDStartOffset
+    );
+  }
+
+  // updating classic EoCD record with new CD start offset
+  if (newCDStartOffset < 0xffffffff) {
+    setFieldToNumber(eocd, 4, classicEoCDOffset + CD_START_OFFSET_OFFSET, newCDStartOffset);
+  }
+
+  // updating zip64 EoCD locator and record with new EoCD record start offset and cd records number
+  if (oldEoCDOffsets.zip64EoCDLocatorOffset && oldEoCDOffsets.zip64EoCDOffset) {
+    // updating zip64 EoCD locator with new EoCD record start offset
+    const locatorOffset = oldEoCDOffsets.zip64EoCDLocatorOffset - oldEoCDOffsets.zip64EoCDOffset;
+    setFieldToNumber(eocd, 8, locatorOffset + ZIP64_EOCD_START_OFFSET_OFFSET, eocdStartOffset);
+
+    // updating zip64 EoCD record with new cd start offset
+    setFieldToNumber(eocd, 8, ZIP64_CD_START_OFFSET_OFFSET, newCDStartOffset);
+
+    // updating zip64 EoCD record with new cd records number
+    setFieldToNumber(eocd, 8, ZIP64_CD_RECORDS_NUMBER_OFFSET, newCDRecordsNumber);
+    setFieldToNumber(eocd, 8, ZIP64_CD_RECORDS_NUMBER_ON_DISC_OFFSET, newCDRecordsNumber);
+
+    // updating zip64 EoCD record with new size of CD
+    setFieldToNumber(eocd, 8, ZIP64_CD_CD_BYTE_SIZE_OFFSET, eocdStartOffset - newCDStartOffset);
+  }
+
+  return new Uint8Array(eocd.buffer);
+}

package/src/parse-zip/local-file-header.ts

@@ -1,8 +1,10 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {FileProvider, compareArrayBuffers} from '@loaders.gl/loader-utils';
+import {FileProvider, compareArrayBuffers, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
 import {ZipSignature} from './search-from-the-end';
+import {createZip64Info, setFieldToNumber} from './zip64-info-generation';
 
 /**
  * zip local file header info
@@ -94,3 +96,127 @@ export const parseZipLocalFileHeader = async (
     compressionMethod
   };
 };
+
+/** info that can be placed into cd header */
+type GenerateLocalOptions = {
+  /** CRC-32 of uncompressed data */
+  crc32: number;
+  /** File name */
+  fileName: string;
+  /** File size */
+  length: number;
+};
+
+/**
+ * generates local header for the file
+ * @param options info that can be placed into local header
+ * @returns buffer with header
+ */
+export function generateLocalHeader(options: GenerateLocalOptions): ArrayBuffer {
+  const optionsToUse = {
+    ...options,
+    extraLength: 0,
+    fnlength: options.fileName.length
+  };
+
+  let zip64header: ArrayBuffer = new ArrayBuffer(0);
+
+  const optionsToZip64: any = {};
+  if (optionsToUse.length >= 0xffffffff) {
+    optionsToZip64.size = optionsToUse.length;
+    optionsToUse.length = 0xffffffff;
+  }
+
+  if (Object.keys(optionsToZip64).length) {
+    zip64header = createZip64Info(optionsToZip64);
+    optionsToUse.extraLength = zip64header.byteLength;
+  }
+
+  // base length without file name and extra info is static
+  const header = new DataView(new ArrayBuffer(Number(FILE_NAME_OFFSET)));
+
+  for (const field of ZIP_HEADER_FIELDS) {
+    setFieldToNumber(
+      header,
+      field.size,
+      field.offset,
+      optionsToUse[field.name ?? ''] ?? field.default ?? 0
+    );
+  }
+
+  const encodedName = new TextEncoder().encode(optionsToUse.fileName);
+
+  const resHeader = concatenateArrayBuffers(header.buffer, encodedName, zip64header);
+
+  return resHeader;
+}
+
+const ZIP_HEADER_FIELDS = [
+  // Local file header signature = 0x04034b50
+  {
+    offset: 0,
+    size: 4,
+    default: new DataView(signature.buffer).getUint32(0, true)
+  },
+  // Version needed to extract (minimum)
+  {
+    offset: 4,
+    size: 2,
+    default: 45
+  },
+  // General purpose bit flag
+  {
+    offset: 6,
+    size: 2,
+    default: 0
+  },
+  // Compression method
+  {
+    offset: 8,
+    size: 2,
+    default: 0
+  },
+  // File last modification time
+  {
+    offset: 10,
+    size: 2,
+    default: 0
+  },
+  // File last modification date
+  {
+    offset: 12,
+    size: 2,
+    default: 0
+  },
+  // CRC-32 of uncompressed data
+  {
+    offset: 14,
+    size: 4,
+    name: 'crc32'
+  },
+  // Compressed size (or 0xffffffff for ZIP64)
+  {
+    offset: 18,
+    size: 4,
+    name: 'length'
+  },
+  // Uncompressed size (or 0xffffffff for ZIP64)
+  {
+    offset: 22,
+    size: 4,
+    name: 'length'
+  },
+  // File name length (n)
+  {
+    offset: 26,
+    size: 2,
+    name: 'fnlength'
+  },
+  // Extra field length (m)
+  {
+    offset: 28,
+    size: 2,
+    default: 0,
+    name: 'extraLength'
+  }
+];

package/src/parse-zip/search-from-the-end.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {FileProvider} from '@loaders.gl/loader-utils';

package/src/parse-zip/zip-compozition.ts

@@ -0,0 +1,113 @@
+import {FileHandleFile, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
+import {ZipEoCDRecord, parseEoCDRecord, updateEoCD} from './end-of-central-directory';
+import {CRC32Hash} from '@loaders.gl/crypto';
+import {generateLocalHeader} from './local-file-header';
+import {generateCDHeader} from './cd-file-header';
+
+/**
+ * cut off CD and EoCD records from zip file
+ * @param provider zip file
+ * @returns tuple with three values: CD, EoCD record, EoCD information
+ */
+async function cutTheTailOff(
+  provider: FileHandleFile
+): Promise<[ArrayBuffer, ArrayBuffer, ZipEoCDRecord]> {
+  // define where the body ends
+  const oldEoCDinfo = await parseEoCDRecord(provider);
+  const oldCDStartOffset = oldEoCDinfo.cdStartOffset;
+
+  // define cd length
+  const oldCDLength = Number(
+    oldEoCDinfo.offsets.zip64EoCDOffset
+      ? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset
+      : oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset
+  );
+
+  // cut off everything except of archieve body
+  const zipEnding = await provider.slice(oldCDStartOffset, provider.length);
+  await provider.truncate(Number(oldCDStartOffset));
+
+  // divide cd body and eocd record
+  const oldCDBody = zipEnding.slice(0, oldCDLength);
+  const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength);
+
+  return [oldCDBody, eocdBody, oldEoCDinfo];
+}
+
+/**
+ * generates CD and local headers for the file
+ * @param fileName name of the file
+ * @param fileToAdd buffer with the file
+ * @param localFileHeaderOffset offset of the file local header
+ * @returns tuple with two values: local header and file body, cd header
+ */
+async function generateFileHeaders(
+  fileName: string,
+  fileToAdd: ArrayBuffer,
+  localFileHeaderOffset: bigint
+): Promise<[Uint8Array, Uint8Array]> {
+  // generating CRC32 of the content
+  const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16);
+
+  // generate local header for the file
+  const newFileLocalHeader = generateLocalHeader({
+    crc32: newFileCRC322,
+    fileName,
+    length: fileToAdd.byteLength
+  });
+
+  // generate hash file cd header
+  const newFileCDHeader = generateCDHeader({
+    crc32: newFileCRC322,
+    fileName,
+    offset: localFileHeaderOffset,
+    length: fileToAdd.byteLength
+  });
+  return [
+    new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)),
+    new Uint8Array(newFileCDHeader)
+  ];
+}
+
+/**
+ * adds one file in the end of the archieve
+ * @param zipUrl path to the file
+ * @param fileToAdd new file body
+ * @param fileName new file name
+ */
+export async function addOneFile(zipUrl: string, fileToAdd: ArrayBuffer, fileName: string) {
+  // init file handler
+  const provider = new FileHandleFile(zipUrl, true);
+
+  const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider);
+
+  // remember the new file local header start offset
+  const newFileOffset = provider.length;
+
+  const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset);
+
+  // write down the file local header
+  await provider.append(localPart);
+
+  // add the file CD header to the CD
+  const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart);
+
+  // remember the CD start offset
+  const newCDStartOffset = provider.length;
+
+  // write down new CD
+  await provider.append(new Uint8Array(newCDBody));
+
+  // remember where eocd starts
+  const eocdOffset = provider.length;
+
+  await provider.append(
+    await updateEoCD(
+      eocdBody,
+      oldEoCDinfo.offsets,
+      newCDStartOffset,
+      eocdOffset,
+      oldEoCDinfo.cdRecordsNumber + 1n
+    )
+  );
+}