@loaders.gl/zip 4.1.0-alpha.1 → 4.1.0-alpha.10

package/src/parse-zip/search-from-the-end.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {FileProvider} from '@loaders.gl/loader-utils';

package/src/parse-zip/zip-compozition.ts

@@ -0,0 +1,113 @@
+import {FileHandleFile, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
+import {ZipEoCDRecord, parseEoCDRecord, updateEoCD} from './end-of-central-directory';
+import {CRC32Hash} from '@loaders.gl/crypto';
+import {generateLocalHeader} from './local-file-header';
+import {generateCDHeader} from './cd-file-header';
+
+/**
+ * cut off CD and EoCD records from zip file
+ * @param provider zip file
+ * @returns tuple with three values: CD, EoCD record, EoCD information
+ */
+async function cutTheTailOff(
+  provider: FileHandleFile
+): Promise<[ArrayBuffer, ArrayBuffer, ZipEoCDRecord]> {
+  // define where the body ends
+  const oldEoCDinfo = await parseEoCDRecord(provider);
+  const oldCDStartOffset = oldEoCDinfo.cdStartOffset;
+
+  // define cd length
+  const oldCDLength = Number(
+    oldEoCDinfo.offsets.zip64EoCDOffset
+      ? oldEoCDinfo.offsets.zip64EoCDOffset - oldCDStartOffset
+      : oldEoCDinfo.offsets.zipEoCDOffset - oldCDStartOffset
+  );
+
+  // cut off everything except of archieve body
+  const zipEnding = await provider.slice(oldCDStartOffset, provider.length);
+  await provider.truncate(Number(oldCDStartOffset));
+
+  // divide cd body and eocd record
+  const oldCDBody = zipEnding.slice(0, oldCDLength);
+  const eocdBody = zipEnding.slice(oldCDLength, zipEnding.byteLength);
+
+  return [oldCDBody, eocdBody, oldEoCDinfo];
+}
+
+/**
+ * generates CD and local headers for the file
+ * @param fileName name of the file
+ * @param fileToAdd buffer with the file
+ * @param localFileHeaderOffset offset of the file local header
+ * @returns tuple with two values: local header and file body, cd header
+ */
+async function generateFileHeaders(
+  fileName: string,
+  fileToAdd: ArrayBuffer,
+  localFileHeaderOffset: bigint
+): Promise<[Uint8Array, Uint8Array]> {
+  // generating CRC32 of the content
+  const newFileCRC322 = parseInt(await new CRC32Hash().hash(fileToAdd, 'hex'), 16);
+
+  // generate local header for the file
+  const newFileLocalHeader = generateLocalHeader({
+    crc32: newFileCRC322,
+    fileName,
+    length: fileToAdd.byteLength
+  });
+
+  // generate hash file cd header
+  const newFileCDHeader = generateCDHeader({
+    crc32: newFileCRC322,
+    fileName,
+    offset: localFileHeaderOffset,
+    length: fileToAdd.byteLength
+  });
+  return [
+    new Uint8Array(concatenateArrayBuffers(newFileLocalHeader, fileToAdd)),
+    new Uint8Array(newFileCDHeader)
+  ];
+}
+
+/**
+ * adds one file in the end of the archieve
+ * @param zipUrl path to the file
+ * @param fileToAdd new file body
+ * @param fileName new file name
+ */
+export async function addOneFile(zipUrl: string, fileToAdd: ArrayBuffer, fileName: string) {
+  // init file handler
+  const provider = new FileHandleFile(zipUrl, true);
+
+  const [oldCDBody, eocdBody, oldEoCDinfo] = await cutTheTailOff(provider);
+
+  // remember the new file local header start offset
+  const newFileOffset = provider.length;
+
+  const [localPart, cdHeaderPart] = await generateFileHeaders(fileName, fileToAdd, newFileOffset);
+
+  // write down the file local header
+  await provider.append(localPart);
+
+  // add the file CD header to the CD
+  const newCDBody = concatenateArrayBuffers(oldCDBody, cdHeaderPart);
+
+  // remember the CD start offset
+  const newCDStartOffset = provider.length;
+
+  // write down new CD
+  await provider.append(new Uint8Array(newCDBody));
+
+  // remember where eocd starts
+  const eocdOffset = provider.length;
+
+  await provider.append(
+    await updateEoCD(
+      eocdBody,
+      oldEoCDinfo.offsets,
+      newCDStartOffset,
+      eocdOffset,
+      oldEoCDinfo.cdRecordsNumber + 1n
+    )
+  );
+}

package/src/parse-zip/zip64-info-generation.ts

@@ -41,15 +41,31 @@ export function createZip64Info(options: Zip64Options): ArrayBuffer {
  * @param offset offset of the writing start
  * @param value value to be written
  */
-type NumberSetter = (header: DataView, offset: number, value: number) => void;
+type NumberSetter = (header: DataView, offset: number, value: number | bigint) => void;
+
+/**
+ * Writes values into buffer according to the bytes amount
+ * @param header header where to write the data
+ * @param fieldSize size of the field in bytes
+ * @param fieldOffset offset of the field
+ * @param value value to be written
+ */
+export function setFieldToNumber(
+  header: DataView,
+  fieldSize: number,
+  fieldOffset: number | bigint,
+  value: number | bigint
+): void {
+  NUMBER_SETTERS[fieldSize](header, Number(fieldOffset), value);
+}
 
 /** functions to write values into buffer according to the bytes amount */
-export const NUMBER_SETTERS: {[key: number]: NumberSetter} = {
+const NUMBER_SETTERS: {[key: number]: NumberSetter} = {
   2: (header, offset, value) => {
-    header.setUint16(offset, value, true);
+    header.setUint16(offset, Number(value), true);
   },
   4: (header, offset, value) => {
-    header.setUint32(offset, value, true);
+    header.setUint32(offset, Number(value), true);
   },
   8: (header, offset, value) => {
     header.setBigUint64(offset, BigInt(value), true);

package/src/tar-builder.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import Tar from './lib/tar/tar';

package/src/zip-loader.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';

package/src/zip-writer.ts

@@ -1,9 +1,13 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import JSZip, {JSZipGeneratorOptions} from 'jszip';
 
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
 export type ZipWriterOptions = WriterOptions & {
   zip?: {
     onUpdate?: (metadata: {percent: number}) => void;
@@ -15,19 +19,27 @@ export type ZipWriterOptions = WriterOptions & {
 /**
  * Zip exporter
  */
-export const ZipWriter: Writer<FileReaderEventMap, never, ZipWriterOptions> = {
+export const ZipWriter: WriterWithEncoder<Record<string, ArrayBuffer>, never, ZipWriterOptions> = {
   name: 'Zip Archive',
+  id: 'zip',
+  module: 'zip',
+  version: VERSION,
   extensions: ['zip'],
   category: 'archive',
   mimeTypes: ['application/zip'],
-  // @ts-ignore
+  options: {
+    zip: {
+      onUpdate: () => {}
+    },
+    jszip: {}
+  },
   encode: encodeZipAsync
 };
 
 async function encodeZipAsync(
   fileMap: Record<string, ArrayBuffer>,
   options: ZipWriterOptions = {}
-) {
+): Promise<ArrayBuffer> {
   const jsZip = new JSZip();
   // add files to the zip
   for (const subFileName in fileMap) {
@@ -38,14 +50,16 @@ async function encodeZipAsync(
     jsZip.file(subFileName, subFileData, options?.jszip || {});
   }
 
-  // always generate the full zip as an arraybuffer
-  const jszipOptions: JSZipGeneratorOptions = {...options?.jszip, type: 'arraybuffer'};
-  const {onUpdate = () => {}} = options;
+  const zipOptions = {...ZipWriter.options.zip, ...options?.zip};
+  const jszipOptions: JSZipGeneratorOptions = {...ZipWriter.options?.jszip, ...options.jszip};
 
   try {
-    return await jsZip.generateAsync(jszipOptions, onUpdate);
+    return await jsZip.generateAsync(
+      {...jszipOptions, type: 'arraybuffer'}, // generate an arraybuffer
+      zipOptions.onUpdate
+    );
   } catch (error) {
-    options.log.error(`Unable to write zip archive: ${error}`);
+    options.log.error(`Unable to encode zip archive: ${error}`);
     throw error;
   }
 }