@loaders.gl/zip 4.1.0-alpha.1 → 4.1.0-alpha.11

package/dist/zip-writer.js

@@ -1,29 +1,44 @@
 import JSZip from 'jszip';
+const VERSION = typeof "4.1.0-alpha.11" !== 'undefined' ? "4.1.0-alpha.11" : 'latest';
 export const ZipWriter = {
   name: 'Zip Archive',
+  id: 'zip',
+  module: 'zip',
+  version: VERSION,
   extensions: ['zip'],
   category: 'archive',
   mimeTypes: ['application/zip'],
+  options: {
+    zip: {
+      onUpdate: () => {}
+    },
+    jszip: {}
+  },
   encode: encodeZipAsync
 };
 async function encodeZipAsync(fileMap) {
+  var _ZipWriter$options;
   let options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
   const jsZip = new JSZip();
   for (const subFileName in fileMap) {
     const subFileData = fileMap[subFileName];
     jsZip.file(subFileName, subFileData, (options === null || options === void 0 ? void 0 : options.jszip) || {});
   }
+  const zipOptions = {
+    ...ZipWriter.options.zip,
+    ...(options === null || options === void 0 ? void 0 : options.zip)
+  };
   const jszipOptions = {
-    ...(options === null || options === void 0 ? void 0 : options.jszip),
-    type: 'arraybuffer'
+    ...((_ZipWriter$options = ZipWriter.options) === null || _ZipWriter$options === void 0 ? void 0 : _ZipWriter$options.jszip),
+    ...options.jszip
   };
-  const {
-    onUpdate = () => {}
-  } = options;
   try {
-    return await jsZip.generateAsync(jszipOptions, onUpdate);
+    return await jsZip.generateAsync({
+      ...jszipOptions,
+      type: 'arraybuffer'
+    }, zipOptions.onUpdate);
   } catch (error) {
-    options.log.error(`Unable to write zip archive: ${error}`);
+    options.log.error(`Unable to encode zip archive: ${error}`);
     throw error;
   }
 }

package/dist/zip-writer.js.map

@@ -1 +1 @@
-{"version":3,"file":"zip-writer.js","names":["JSZip","ZipWriter","name","extensions","category","mimeTypes","encode","encodeZipAsync","fileMap","options","arguments","length","undefined","jsZip","subFileName","subFileData","file","jszip","jszipOptions","type","onUpdate","generateAsync","error","log"],"sources":["../src/zip-writer.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport type {Writer, WriterOptions} from '@loaders.gl/loader-utils';\nimport JSZip, {JSZipGeneratorOptions} from 'jszip';\n\nexport type ZipWriterOptions = WriterOptions & {\n  zip?: {\n    onUpdate?: (metadata: {percent: number}) => void;\n  };\n  /** Passthrough options to jszip */\n  jszip?: JSZipGeneratorOptions;\n};\n\n/**\n * Zip exporter\n */\nexport const ZipWriter: Writer<FileReaderEventMap, never, ZipWriterOptions> = {\n  name: 'Zip Archive',\n  extensions: ['zip'],\n  category: 'archive',\n  mimeTypes: ['application/zip'],\n  // @ts-ignore\n  encode: encodeZipAsync\n};\n\nasync function encodeZipAsync(\n  fileMap: Record<string, ArrayBuffer>,\n  options: ZipWriterOptions = {}\n) {\n  const jsZip = new JSZip();\n  // add files to the zip\n  for (const subFileName in fileMap) {\n    const subFileData = fileMap[subFileName];\n\n    // jszip supports both arraybuffer and string data (the main loaders.gl types)\n    // https://stuk.github.io/jszip/documentation/api_zipobject/async.html\n    jsZip.file(subFileName, subFileData, options?.jszip || {});\n  }\n\n  // always generate the full zip as an arraybuffer\n  const jszipOptions: JSZipGeneratorOptions = {...options?.jszip, type: 'arraybuffer'};\n  const {onUpdate = () => {}} = options;\n\n  try {\n    return await jsZip.generateAsync(jszipOptions, onUpdate);\n  } catch (error) {\n    options.log.error(`Unable to write zip archive: ${error}`);\n    throw error;\n  }\n}\n"],"mappings":"AAIA,OAAOA,KAAK,MAA+B,OAAO;AAalD,OAAO,MAAMC,SAA8D,GAAG;EAC5EC,IAAI,EAAE,aAAa;EACnBC,UAAU,EAAE,CAAC,KAAK,CAAC;EACnBC,QAAQ,EAAE,SAAS;EACnBC,SAAS,EAAE,CAAC,iBAAiB,CAAC;EAE9BC,MAAM,EAAEC;AACV,CAAC;AAED,eAAeA,cAAcA,CAC3BC,OAAoC,EAEpC;EAAA,IADAC,OAAyB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EAE9B,MAAMG,KAAK,GAAG,IAAIb,KAAK,CAAC,CAAC;EAEzB,KAAK,MAAMc,WAAW,IAAIN,OAAO,EAAE;IACjC,MAAMO,WAAW,GAAGP,OAAO,CAACM,WAAW,CAAC;IAIxCD,KAAK,CAACG,IAAI,CAACF,WAAW,EAAEC,WAAW,EAAE,CAAAN,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEQ,KAAK,KAAI,CAAC,CAAC,CAAC;EAC5D;EAGA,MAAMC,YAAmC,GAAG;IAAC,IAAGT,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEQ,KAAK;IAAEE,IAAI,EAAE;EAAa,CAAC;EACpF,MAAM;IAACC,QAAQ,GAAGA,CAAA,KAAM,CAAC;EAAC,CAAC,GAAGX,OAAO;EAErC,IAAI;IACF,OAAO,MAAMI,KAAK,CAACQ,aAAa,CAACH,YAAY,EAAEE,QAAQ,CAAC;EAC1D,CAAC,CAAC,OAAOE,KAAK,EAAE;IACdb,OAAO,CAACc,GAAG,CAACD,KAAK,CAAE,gCAA+BA,KAAM,EAAC,CAAC;IAC1D,MAAMA,KAAK;EACb;AACF"}
+{"version":3,"file":"zip-writer.js","names":["JSZip","VERSION","ZipWriter","name","id","module","version","extensions","category","mimeTypes","options","zip","onUpdate","jszip","encode","encodeZipAsync","fileMap","_ZipWriter$options","arguments","length","undefined","jsZip","subFileName","subFileData","file","zipOptions","jszipOptions","generateAsync","type","error","log"],"sources":["../src/zip-writer.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';\nimport JSZip, {JSZipGeneratorOptions} from 'jszip';\n\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ZipWriterOptions = WriterOptions & {\n  zip?: {\n    onUpdate?: (metadata: {percent: number}) => void;\n  };\n  /** Passthrough options to jszip */\n  jszip?: JSZipGeneratorOptions;\n};\n\n/**\n * Zip exporter\n */\nexport const ZipWriter: WriterWithEncoder<Record<string, ArrayBuffer>, never, ZipWriterOptions> = {\n  name: 'Zip Archive',\n  id: 'zip',\n  module: 'zip',\n  version: VERSION,\n  extensions: ['zip'],\n  category: 'archive',\n  mimeTypes: ['application/zip'],\n  options: {\n    zip: {\n      onUpdate: () => {}\n    },\n    jszip: {}\n  },\n  encode: encodeZipAsync\n};\n\nasync function encodeZipAsync(\n  fileMap: Record<string, ArrayBuffer>,\n  options: ZipWriterOptions = {}\n): Promise<ArrayBuffer> {\n  const jsZip = new JSZip();\n  // add files to the zip\n  for (const subFileName in fileMap) {\n    const subFileData = fileMap[subFileName];\n\n    // jszip supports both arraybuffer and string data (the main loaders.gl types)\n    // https://stuk.github.io/jszip/documentation/api_zipobject/async.html\n    jsZip.file(subFileName, subFileData, options?.jszip || {});\n  }\n\n  const zipOptions = {...ZipWriter.options.zip, ...options?.zip};\n  const jszipOptions: JSZipGeneratorOptions = {...ZipWriter.options?.jszip, ...options.jszip};\n\n  try {\n    return await jsZip.generateAsync(\n      {...jszipOptions, type: 'arraybuffer'}, // generate an arraybuffer\n      zipOptions.onUpdate\n    );\n  } catch (error) {\n    options.log.error(`Unable to encode zip archive: ${error}`);\n    throw error;\n  }\n}\n"],"mappings":"AAKA,OAAOA,KAAK,MAA+B,OAAO;AAGlD,MAAMC,OAAO,GAAG,uBAAkB,KAAK,WAAW,sBAAiB,QAAQ;AAa3E,OAAO,MAAMC,SAAkF,GAAG;EAChGC,IAAI,EAAE,aAAa;EACnBC,EAAE,EAAE,KAAK;EACTC,MAAM,EAAE,KAAK;EACbC,OAAO,EAAEL,OAAO;EAChBM,UAAU,EAAE,CAAC,KAAK,CAAC;EACnBC,QAAQ,EAAE,SAAS;EACnBC,SAAS,EAAE,CAAC,iBAAiB,CAAC;EAC9BC,OAAO,EAAE;IACPC,GAAG,EAAE;MACHC,QAAQ,EAAEA,CAAA,KAAM,CAAC;IACnB,CAAC;IACDC,KAAK,EAAE,CAAC;EACV,CAAC;EACDC,MAAM,EAAEC;AACV,CAAC;AAED,eAAeA,cAAcA,CAC3BC,OAAoC,EAEd;EAAA,IAAAC,kBAAA;EAAA,IADtBP,OAAyB,GAAAQ,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EAE9B,MAAMG,KAAK,GAAG,IAAIrB,KAAK,CAAC,CAAC;EAEzB,KAAK,MAAMsB,WAAW,IAAIN,OAAO,EAAE;IACjC,MAAMO,WAAW,GAAGP,OAAO,CAACM,WAAW,CAAC;IAIxCD,KAAK,CAACG,IAAI,CAACF,WAAW,EAAEC,WAAW,EAAE,CAAAb,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEG,KAAK,KAAI,CAAC,CAAC,CAAC;EAC5D;EAEA,MAAMY,UAAU,GAAG;IAAC,GAAGvB,SAAS,CAACQ,OAAO,CAACC,GAAG;IAAE,IAAGD,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,GAAG;EAAA,CAAC;EAC9D,MAAMe,YAAmC,GAAG;IAAC,KAAAT,kBAAA,GAAGf,SAAS,CAACQ,OAAO,cAAAO,kBAAA,uBAAjBA,kBAAA,CAAmBJ,KAAK;IAAE,GAAGH,OAAO,CAACG;EAAK,CAAC;EAE3F,IAAI;IACF,OAAO,MAAMQ,KAAK,CAACM,aAAa,CAC9B;MAAC,GAAGD,YAAY;MAAEE,IAAI,EAAE;IAAa,CAAC,EACtCH,UAAU,CAACb,QACb,CAAC;EACH,CAAC,CAAC,OAAOiB,KAAK,EAAE;IACdnB,OAAO,CAACoB,GAAG,CAACD,KAAK,CAAE,iCAAgCA,KAAM,EAAC,CAAC;IAC3D,MAAMA,KAAK;EACb;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/zip",
-  "version": "4.1.0-alpha.1",
+  "version": "4.1.0-alpha.11",
   "description": "Zip Archive Loader",
   "license": "MIT",
   "type": "module",
@@ -22,9 +22,9 @@
   "module": "dist/index.js",
   "exports": {
     ".": {
+      "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
-      "require": "./dist/index.cjs",
-      "types": "./dist/index.d.ts"
+      "require": "./dist/index.cjs"
     }
   },
   "sideEffects": false,
@@ -38,11 +38,11 @@
     "build-bundle": "ocular-bundle ./src/index.ts"
   },
   "dependencies": {
-    "@loaders.gl/compression": "4.1.0-alpha.1",
-    "@loaders.gl/crypto": "4.1.0-alpha.1",
-    "@loaders.gl/loader-utils": "4.1.0-alpha.1",
+    "@loaders.gl/compression": "4.1.0-alpha.11",
+    "@loaders.gl/crypto": "4.1.0-alpha.11",
+    "@loaders.gl/loader-utils": "4.1.0-alpha.11",
     "jszip": "^3.1.5",
     "md5": "^2.3.0"
   },
-  "gitHead": "6a4d3da93d45115ad99861474a43c3f4a0b280a7"
+  "gitHead": "5d3e23bf93762b48c8c1d6d926ede7a97fe43ab0"
 }

package/src/filesystems/zip-filesystem.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {FileSystem, isBrowser} from '@loaders.gl/loader-utils';

package/src/hash-file-utility.ts

@@ -1,9 +1,14 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {MD5Hash} from '@loaders.gl/crypto';
-import {FileProvider} from '@loaders.gl/loader-utils';
-import {makeZipCDHeaderIterator} from './parse-zip/cd-file-header';
+import {
+  FileProvider,
+  concatenateArrayBuffers,
+  concatenateArrayBuffersFromArray
+} from '@loaders.gl/loader-utils';
+import {ZipCDFileHeader, makeZipCDHeaderIterator} from './parse-zip/cd-file-header';
 
 /**
  * Reads hash file from buffer and returns it in ready-to-use form
@@ -40,6 +45,17 @@ export async function makeHashTableFromZipHeaders(
   fileProvider: FileProvider
 ): Promise<Record<string, bigint>> {
   const zipCDIterator = makeZipCDHeaderIterator(fileProvider);
+  return getHashTable(zipCDIterator);
+}
+
+/**
+ * creates hash table from file offset iterator
+ * @param zipCDIterator iterator to use
+ * @returns hash table
+ */
+export async function getHashTable(
+  zipCDIterator: AsyncIterable<ZipCDFileHeader>
+): Promise<Record<string, bigint>> {
   const md5Hash = new MD5Hash();
   const textEncoder = new TextEncoder();
 
@@ -54,3 +70,68 @@ export async function makeHashTableFromZipHeaders(
 
   return hashTable;
 }
+
+/** item of the file offset list */
+type FileListItem = {
+  fileName: string;
+  localHeaderOffset: bigint;
+};
+
+/**
+ * creates hash file that later can be added to the SLPK archive
+ * @param zipCDIterator iterator to use
+ * @returns ArrayBuffer containing hash file
+ */
+export async function composeHashFile(
+  zipCDIterator: AsyncIterable<FileListItem> | Iterable<FileListItem>
+): Promise<ArrayBuffer> {
+  const md5Hash = new MD5Hash();
+  const textEncoder = new TextEncoder();
+
+  const hashArray: ArrayBuffer[] = [];
+
+  for await (const cdHeader of zipCDIterator) {
+    const filename = cdHeader.fileName.split('\\').join('/').toLocaleLowerCase();
+    const arrayBuffer = textEncoder.encode(filename).buffer;
+    const md5 = await md5Hash.hash(arrayBuffer, 'hex');
+    hashArray.push(
+      concatenateArrayBuffers(hexStringToBuffer(md5), bigintToBuffer(cdHeader.localHeaderOffset))
+    );
+  }
+
+  const bufferArray = hashArray.sort(compareHashes);
+
+  return concatenateArrayBuffersFromArray(bufferArray);
+}
+
+/**
+ * Function to compare md5 hashes according to https://github.com/Esri/i3s-spec/blob/master/docs/2.0/slpk_hashtable.pcsl.md
+ * @param arrA first hash to compare
+ * @param arrB second hash to compare
+ * @returns 0 if equal, negative number if a<b, pozitive if a>b
+ */
+function compareHashes(arrA: ArrayBuffer, arrB: ArrayBuffer): number {
+  const a = new BigUint64Array(arrA);
+  const b = new BigUint64Array(arrB);
+
+  return Number(a[0] === b[0] ? a[1] - b[1] : a[0] - b[0]);
+}
+
+/**
+ * converts hex string to buffer
+ * @param str hex string to convert
+ * @returns conversion result
+ */
+function hexStringToBuffer(str: string): ArrayBuffer {
+  const byteArray = str.match(/../g)?.map((h) => parseInt(h, 16));
+  return new Uint8Array(byteArray ?? new Array(16)).buffer;
+}
+
+/**
+ * converts bigint to buffer
+ * @param n bigint to convert
+ * @returns convertion result
+ */
+function bigintToBuffer(n: bigint): ArrayBuffer {
+  return new BigUint64Array([n]).buffer;
+}

package/src/index.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 export {ZipLoader} from './zip-loader';
@@ -13,12 +14,14 @@ export {
 } from './parse-zip/cd-file-header';
 export {
   parseZipLocalFileHeader,
-  signature as localHeaderSignature
+  signature as localHeaderSignature,
+  generateLocalHeader
 } from './parse-zip/local-file-header';
 export {parseEoCDRecord} from './parse-zip/end-of-central-directory';
 export {searchFromTheEnd} from './parse-zip/search-from-the-end';
+export {addOneFile, createZip} from './parse-zip/zip-composition';
 
 // export type {HashElement} from './hash-file-utility';
-export {parseHashTable, makeHashTableFromZipHeaders} from './hash-file-utility';
+export {parseHashTable, makeHashTableFromZipHeaders, composeHashFile} from './hash-file-utility';
 
 export {ZipFileSystem} from './filesystems/zip-filesystem';

package/src/lib/tar/header.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 // This file is derived from the tar-js code base under MIT license

package/src/lib/tar/tar.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 // This file is derived from the tar-js code base under MIT license

package/src/lib/tar/types.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 /**

package/src/lib/tar/utils.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 // This file is derived from the tar-js code base under MIT license

package/src/parse-zip/cd-file-header.ts

@@ -1,10 +1,11 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {FileProvider, compareArrayBuffers, concatenateArrayBuffers} from '@loaders.gl/loader-utils';
 import {parseEoCDRecord} from './end-of-central-directory';
 import {ZipSignature} from './search-from-the-end';
-import {createZip64Info, NUMBER_SETTERS} from './zip64-info-generation';
+import {createZip64Info, setFieldToNumber} from './zip64-info-generation';
 
 /**
  * zip central directory file header info
@@ -199,7 +200,7 @@ type GenerateCDOptions = {
   /** File size */
   length: number;
   /** Relative offset of local file header */
-  offset: number;
+  offset: bigint;
 };
 
 /**
@@ -219,7 +220,7 @@ export function generateCDHeader(options: GenerateCDOptions): ArrayBuffer {
   const optionsToZip64: any = {};
   if (optionsToUse.offset >= 0xffffffff) {
     optionsToZip64.offset = optionsToUse.offset;
-    optionsToUse.offset = 0xffffffff;
+    optionsToUse.offset = BigInt(0xffffffff);
   }
   if (optionsToUse.length >= 0xffffffff) {
     optionsToZip64.size = optionsToUse.length;
@@ -230,11 +231,12 @@ export function generateCDHeader(options: GenerateCDOptions): ArrayBuffer {
     zip64header = createZip64Info(optionsToZip64);
     optionsToUse.extraLength = zip64header.byteLength;
   }
-  const header = new DataView(new ArrayBuffer(46));
+  const header = new DataView(new ArrayBuffer(Number(CD_FILE_NAME_OFFSET)));
 
   for (const field of ZIP_HEADER_FIELDS) {
-    NUMBER_SETTERS[field.size](
+    setFieldToNumber(
       header,
+      field.size,
       field.offset,
       optionsToUse[field.name ?? ''] ?? field.default ?? 0
     );