@zip.js/zip.js 2.19.0 → 2.21.0

package/lib/core/zip-writer.js

@@ -114,7 +114,6 @@ import {
 	BITFLAG_ENCRYPTED,
 	BITFLAG_DATA_DESCRIPTOR,
 	BITFLAG_LANG_ENCODING_FLAG,
-	BITFLAG_LEVEL,
 	BITFLAG_LEVEL_FAST_MASK,
 	BITFLAG_LEVEL_SUPER_FAST_MASK,
 	BITFLAG_LEVEL_MAX_MASK,
@@ -140,6 +139,9 @@ import {
 	VERSION_AES,
 	DIRECTORY_SIGNATURE,
 	HEADER_SIZE,
+	ERR_LOCAL_FILE_HEADER_NOT_FOUND,
+	ERR_OVERLAPPING_ENTRY,
+	ERR_EXTRAFIELD_ZIP64_NOT_FOUND,
 	HEADER_OFFSET_VERSION,
 	HEADER_OFFSET_SIGNATURE,
 	HEADER_OFFSET_COMPRESSED_SIZE,
@@ -342,130 +344,8 @@ class ZipWriter {
 		return watchPromiseError(this, prependZipEntries(this, reader));
 	}
 
-	appendZip(reader) {
-		return watchPromiseError(this, this.appendZipEntries(reader));
-	}
-
-	async appendZipEntries(reader) {
-		const zipWriter = this;
-		const { pendingAddFileCalls, filenames, fileEntries } = zipWriter;
-		while (pendingAddFileCalls.size) {
-			await Promise.allSettled(Array.from(pendingAddFileCalls));
-		}
-		let resolveAppendZip;
-		const promiseAppendZip = new Promise(resolve => resolveAppendZip = resolve);
-		pendingAddFileCalls.add(promiseAppendZip);
-		const appendedFilenames = [];
-		let releaseLockWriter;
-		try {
-			reader = new GenericReader(reader);
-			await initStream(reader);
-			if (reader.size === UNDEFINED_VALUE || !reader.readUint8Array) {
-				reader = new BlobReader(await streamToBlob(reader.readable));
-				await initStream(reader);
-			}
-			const { ZipReader } = await import("./zip-reader.js");
-			const zipReader = new ZipReader(reader);
-			const entries = await zipReader.getEntries();
-			await zipReader.close();
-			await initStream(zipWriter.writer);
-			const { directoryOffset } = zipReader;
-			entries.forEach(({ filename }) => {
-				if (filenames.has(filename)) {
-					throw new Error(ERR_DUPLICATED_NAME);
-				}
-				filenames.add(filename);
-				appendedFilenames.push(filename);
-			});
-			zipWriter.writerLocked = true;
-			const { lockWriter } = zipWriter;
-			zipWriter.lockWriter = new Promise(resolve => releaseLockWriter = () => {
-				zipWriter.writerLocked = false;
-				resolve();
-			});
-			await lockWriter;
-			if (zipWriter.addSplitZipSignature) {
-				delete zipWriter.addSplitZipSignature;
-				if (!await startsWithSplitZipSignature(reader)) {
-					await writeData(zipWriter.writer, getSplitZipSignatureArray());
-					zipWriter.offset += SPLIT_ZIP_FILE_SIGNATURE_LENGTH;
-				}
-			}
-			const entryPositions = await copyZipData(zipWriter, reader, entries, directoryOffset);
-			entries.forEach(entry => {
-				const {
-					version,
-					rawLastModDate,
-					rawFilename,
-					bitFlag,
-					encrypted,
-					uncompressedSize,
-					compressedSize,
-					extraFieldZip64
-				} = entry;
-				let {
-					compressionMethod,
-					rawExtraField,
-				} = entry;
-				const { level, languageEncodingFlag, dataDescriptor } = bitFlag;
-				rawExtraField = removeExtraFieldZip64(rawExtraField || EMPTY_UINT8_ARRAY);
-				if (entry.extraFieldAES) {
-					compressionMethod = COMPRESSION_METHOD_AES;
-				}
-				const extraFieldLength = getLength(rawExtraField);
-				const zip64UncompressedSize = Boolean(extraFieldZip64) && extraFieldZip64.uncompressedSize !== UNDEFINED_VALUE;
-				const zip64CompressedSize = Boolean(extraFieldZip64) && extraFieldZip64.compressedSize !== UNDEFINED_VALUE;
-				const bitFlagValue = (getBitFlag(level, languageEncodingFlag, dataDescriptor, encrypted, compressionMethod) & ~BITFLAG_LEVEL) | (level << 1);
-				const {
-					headerArray,
-					headerView
-				} = getHeaderArrayData({
-					version,
-					bitFlag: bitFlagValue,
-					compressionMethod,
-					uncompressedSize,
-					compressedSize,
-					rawLastModDate,
-					rawFilename,
-					zip64CompressedSize,
-					zip64UncompressedSize,
-					extraFieldLength
-				});
-				const { crc32 } = entry;
-				if (crc32 !== UNDEFINED_VALUE) {
-					setUint32(headerView, HEADER_OFFSET_SIGNATURE, crc32);
-				}
-				const { offset, diskNumberStart } = entryPositions.get(entry);
-				Object.assign(entry, {
-					zip64Enabled: true,
-					zip64UncompressedSize,
-					zip64CompressedSize,
-					offset,
-					diskNumberStart,
-					zip64DiskNumberStart: false,
-					rawExtraFieldZip64: EMPTY_UINT8_ARRAY,
-					rawExtraFieldAES: EMPTY_UINT8_ARRAY,
-					rawExtraFieldExtendedTimestamp: EMPTY_UINT8_ARRAY,
-					rawExtraFieldNTFS: EMPTY_UINT8_ARRAY,
-					rawExtraFieldUnix: EMPTY_UINT8_ARRAY,
-					rawCentralExtraFieldUnix: EMPTY_UINT8_ARRAY,
-					rawExtraField,
-					rawCentralExtraField: EMPTY_UINT8_ARRAY,
-					headerArray,
-					headerView
-				});
-				fileEntries.set(entry.filename, entry);
-			});
-		} catch (error) {
-			appendedFilenames.forEach(filename => filenames.delete(filename));
-			throw error;
-		} finally {
-			resolveAppendZip();
-			pendingAddFileCalls.delete(promiseAppendZip);
-			if (releaseLockWriter) {
-				releaseLockWriter();
-			}
-		}
+	appendZip(reader, options = {}) {
+		return watchPromiseError(this, appendZipEntries(this, reader, options));
 	}
 
 	add(name = "", reader, options = {}) {
@@ -627,11 +507,140 @@ function watchPromiseError(zipWriter, promise) {
 	return watchedPromise;
 }
 
+async function appendZipEntries(zipWriter, reader, options = {}) {
+	const { pendingAddFileCalls, filenames, fileEntries } = zipWriter;
+	const filter = checkFunctionOption(options.filter);
+	while (pendingAddFileCalls.size) {
+		await Promise.allSettled(Array.from(pendingAddFileCalls));
+	}
+	let resolveAppendZip;
+	const promiseAppendZip = new Promise(resolve => resolveAppendZip = resolve);
+	pendingAddFileCalls.add(promiseAppendZip);
+	const appendedFilenames = [];
+	let releaseLockWriter;
+	try {
+		reader = new GenericReader(reader);
+		await initStream(reader);
+		if (reader.size === UNDEFINED_VALUE || !reader.readUint8Array) {
+			reader = new BlobReader(await streamToBlob(reader.readable));
+			await initStream(reader);
+		}
+		const { ZipReader, getEntryDataDescriptorLength } = await import("./zip-reader.js");
+		const zipReader = new ZipReader(reader);
+		const entries = await zipReader.getEntries();
+		const keptEntries = [];
+		for (const entry of entries) {
+			if (!filter || await filter(entry)) {
+				keptEntries.push(entry);
+			}
+		}
+		await zipReader.close();
+		await initStream(zipWriter.writer);
+		const { directoryOffset, entriesMissingZip64ExtraField } = zipReader;
+		keptEntries.forEach(entry => {
+			const { filename } = entry;
+			if (entriesMissingZip64ExtraField.has(entry)) {
+				throw new Error(ERR_EXTRAFIELD_ZIP64_NOT_FOUND);
+			}
+			if (filenames.has(filename)) {
+				throw new Error(ERR_DUPLICATED_NAME);
+			}
+			filenames.add(filename);
+			appendedFilenames.push(filename);
+		});
+		zipWriter.writerLocked = true;
+		const { lockWriter } = zipWriter;
+		zipWriter.lockWriter = new Promise(resolve => releaseLockWriter = () => {
+			zipWriter.writerLocked = false;
+			resolve();
+		});
+		await lockWriter;
+		if (zipWriter.addSplitZipSignature) {
+			delete zipWriter.addSplitZipSignature;
+			if (filter || !await startsWithSplitZipSignature(reader)) {
+				await writeData(zipWriter.writer, getSplitZipSignatureArray());
+				zipWriter.offset += SPLIT_ZIP_FILE_SIGNATURE_LENGTH;
+			}
+		}
+		const entryPositions = await copyZipData(zipWriter, reader, entries, keptEntries, directoryOffset, Boolean(filter), getEntryDataDescriptorLength);
+		keptEntries.forEach(entry => {
+			const {
+				version,
+				rawLastModDate,
+				rawFilename,
+				rawBitFlag,
+				uncompressedSize,
+				compressedSize,
+				extraFieldZip64
+			} = entry;
+			let {
+				compressionMethod,
+				rawExtraField,
+			} = entry;
+			rawExtraField = removeExtraFieldZip64(rawExtraField || EMPTY_UINT8_ARRAY);
+			if (entry.extraFieldAES) {
+				compressionMethod = COMPRESSION_METHOD_AES;
+			}
+			const extraFieldLength = getLength(rawExtraField);
+			const zip64UncompressedSize = Boolean(extraFieldZip64) && extraFieldZip64.uncompressedSize !== UNDEFINED_VALUE;
+			const zip64CompressedSize = Boolean(extraFieldZip64) && extraFieldZip64.compressedSize !== UNDEFINED_VALUE;
+			const {
+				headerArray,
+				headerView
+			} = getHeaderArrayData({
+				version,
+				bitFlag: rawBitFlag,
+				compressionMethod,
+				uncompressedSize,
+				compressedSize,
+				rawLastModDate,
+				rawFilename,
+				zip64CompressedSize,
+				zip64UncompressedSize,
+				extraFieldLength
+			});
+			const { crc32 } = entry;
+			if (crc32 !== UNDEFINED_VALUE) {
+				setUint32(headerView, HEADER_OFFSET_SIGNATURE, crc32);
+			}
+			const { offset, diskNumberStart } = entryPositions.get(entry);
+			Object.assign(entry, {
+				zip64Enabled: true,
+				zip64UncompressedSize,
+				zip64CompressedSize,
+				offset,
+				diskNumberStart,
+				zip64DiskNumberStart: false,
+				rawExtraFieldZip64: EMPTY_UINT8_ARRAY,
+				rawExtraFieldAES: EMPTY_UINT8_ARRAY,
+				rawExtraFieldExtendedTimestamp: EMPTY_UINT8_ARRAY,
+				rawExtraFieldNTFS: EMPTY_UINT8_ARRAY,
+				rawExtraFieldUnix: EMPTY_UINT8_ARRAY,
+				rawCentralExtraFieldUnix: EMPTY_UINT8_ARRAY,
+				rawExtraField,
+				rawCentralExtraField: EMPTY_UINT8_ARRAY,
+				headerArray,
+				headerView
+			});
+			fileEntries.set(entry.filename, entry);
+		});
+	} catch (error) {
+		appendedFilenames.forEach(filename => filenames.delete(filename));
+		throw error;
+	} finally {
+		resolveAppendZip();
+		pendingAddFileCalls.delete(promiseAppendZip);
+		if (releaseLockWriter) {
+			releaseLockWriter();
+		}
+	}
+}
+
 async function prependZipEntries(zipWriter, reader) {
 	if (zipWriter.filenames.size) {
 		throw new Error(ERR_ZIP_NOT_EMPTY);
 	}
-	await zipWriter.appendZipEntries(reader);
+	await appendZipEntries(zipWriter, reader);
 }
 
 async function addFileEntry(zipWriter, name, reader, options) {
@@ -2449,26 +2458,63 @@ function removeExtraFieldZip64(rawExtraField) {
 	return rawExtraField;
 }
 
-async function copyZipData(zipWriter, reader, entries, directoryOffset) {
+async function copyZipData(zipWriter, reader, entries, keptEntries, directoryOffset, compact, getEntryDataDescriptorLength) {
 	const { writer } = zipWriter;
 	const entryPositions = new Map();
-	if (writer.closeDisk) {
+	if (writer.closeDisk || compact) {
 		const sortedEntries = Array.from(entries).sort((firstEntry, secondEntry) =>
 			getSourceOffset(reader, firstEntry) - getSourceOffset(reader, secondEntry));
-		let copiedLength = 0;
-		for (const entry of sortedEntries) {
-			const sourceOffset = getSourceOffset(reader, entry);
-			await copyData(zipWriter, reader, copiedLength, sourceOffset - copiedLength);
-			if (exceedsAvailableSize(writer, await getLocalHeaderLength(reader, sourceOffset))) {
-				await writer.closeDisk();
+		const regionEnds = getRegionEnds(reader, sortedEntries, directoryOffset);
+		const keptEntrySet = new Set(keptEntries);
+		const localHeaderLengths = new Map();
+		for (let indexEntry = 0; indexEntry < sortedEntries.length; indexEntry++) {
+			const entry = sortedEntries[indexEntry];
+			if (keptEntrySet.has(entry)) {
+				const sourceOffset = getSourceOffset(reader, entry);
+				const localHeaderLength = await getLocalHeaderLength(reader, sourceOffset);
+				if (localHeaderLength === UNDEFINED_VALUE) {
+					throw new Error(ERR_LOCAL_FILE_HEADER_NOT_FOUND);
+				}
+				let entryEnd = sourceOffset + localHeaderLength + entry.compressedSize;
+				if (entry.bitFlag.dataDescriptor) {
+					const dataDescriptorLength = await getEntryDataDescriptorLength(reader, entryEnd, entry);
+					entryEnd = dataDescriptorLength === UNDEFINED_VALUE ? Math.max(entryEnd, regionEnds[indexEntry]) : entryEnd + dataDescriptorLength;
+				}
+				if (entryEnd > regionEnds[indexEntry]) {
+					throw new Error(ERR_OVERLAPPING_ENTRY);
+				}
+				regionEnds[indexEntry] = entryEnd;
+				localHeaderLengths.set(entry, localHeaderLength);
+			}
+		}
+		let copiedOffset = 0;
+		let lastPosition;
+		for (let indexEntry = 0; indexEntry < sortedEntries.length; indexEntry++) {
+			const entry = sortedEntries[indexEntry];
+			if (keptEntrySet.has(entry)) {
+				const sourceOffset = getSourceOffset(reader, entry);
+				if (sourceOffset < copiedOffset) {
+					entryPositions.set(entry, lastPosition);
+				} else {
+					if (!compact) {
+						await copyData(zipWriter, reader, copiedOffset, sourceOffset - copiedOffset);
+					}
+					if (writer.closeDisk && exceedsAvailableSize(writer, localHeaderLengths.get(entry))) {
+						await writer.closeDisk();
+					}
+					lastPosition = {
+						offset: getSegmentOffset(zipWriter, writer),
+						diskNumberStart: getDiskNumber(writer)
+					};
+					entryPositions.set(entry, lastPosition);
+					await copyData(zipWriter, reader, sourceOffset, regionEnds[indexEntry] - sourceOffset);
+					copiedOffset = regionEnds[indexEntry];
+				}
 			}
-			entryPositions.set(entry, {
-				offset: getSegmentOffset(zipWriter, writer),
-				diskNumberStart: getDiskNumber(writer)
-			});
-			copiedLength = sourceOffset;
 		}
-		await copyData(zipWriter, reader, copiedLength, directoryOffset - copiedLength);
+		if (!compact) {
+			await copyData(zipWriter, reader, copiedOffset, directoryOffset - copiedOffset);
+		}
 	} else {
 		const baseOffset = zipWriter.offset;
 		await copyData(zipWriter, reader, 0, directoryOffset);
@@ -2480,6 +2526,20 @@ async function copyZipData(zipWriter, reader, entries, directoryOffset) {
 	return entryPositions;
 }
 
+function getRegionEnds(reader, sortedEntries, directoryOffset) {
+	const regionEnds = new Array(sortedEntries.length);
+	let regionEnd = directoryOffset;
+	for (let indexEntry = sortedEntries.length - 1; indexEntry >= 0; indexEntry--) {
+		const sourceOffset = getSourceOffset(reader, sortedEntries[indexEntry]);
+		const nextEntry = sortedEntries[indexEntry + 1];
+		if (nextEntry && getSourceOffset(reader, nextEntry) > sourceOffset) {
+			regionEnd = Math.min(getSourceOffset(reader, nextEntry), directoryOffset);
+		}
+		regionEnds[indexEntry] = regionEnd;
+	}
+	return regionEnds;
+}
+
 async function copyData(zipWriter, reader, offset, size) {
 	if (size > 0) {
 		const { writer } = zipWriter;
@@ -2487,11 +2547,13 @@ async function copyData(zipWriter, reader, offset, size) {
 		try {
 			await flushBufferedData(createReadable(reader, { offset, size }), writer, UNDEFINED_VALUE, chunkLength => copiedLength += chunkLength);
 		} catch (error) {
-			zipWriter.hasCorruptedEntries = true;
-			try {
-				error.corruptedEntry = true;
-			} catch {
-				// ignored
+			if (copiedLength) {
+				zipWriter.hasCorruptedEntries = true;
+				try {
+					error.corruptedEntry = true;
+				} catch {
+					// ignored
+				}
 			}
 			throw error;
 		} finally {
@@ -2502,11 +2564,16 @@ async function copyData(zipWriter, reader, offset, size) {
 }
 
 async function getLocalHeaderLength(reader, offset) {
-	const headerArray = await readUint8Array(reader, offset, HEADER_SIZE);
+	const headerArray = offset >= 0 && offset + HEADER_SIZE <= reader.size ?
+		await readUint8Array(reader, offset, HEADER_SIZE) :
+		EMPTY_UINT8_ARRAY;
 	if (getLength(headerArray) < HEADER_SIZE) {
-		return HEADER_SIZE;
+		return UNDEFINED_VALUE;
 	}
 	const headerView = getDataView(headerArray);
+	if (getUint32(headerView, 0) != LOCAL_FILE_HEADER_SIGNATURE) {
+		return UNDEFINED_VALUE;
+	}
 	return HEADER_SIZE +
 		getUint16(headerView, HEADER_OFFSET_FILENAME_LENGTH + LOCAL_HEADER_COMMON_OFFSET) +
 		getUint16(headerView, HEADER_OFFSET_EXTRAFIELD_LENGTH + LOCAL_HEADER_COMMON_OFFSET);

package/lib/zip-core-reader.js

@@ -68,6 +68,8 @@ export {
 	WARNING_PREPENDED_DATA,
 	WARNING_PREPENDED_CENTRAL_DIRECTORY,
 	WARNING_TRAILING_CENTRAL_DIRECTORY_DATA,
+	WARNING_MISMATCHED_CENTRAL_DIRECTORY_OFFSET,
+	WARNING_MISSING_ZIP64_EXTRA_FIELD,
 	WARNING_DUPLICATE_FILENAME,
 	WARNING_MISMATCHED_ZIP64_END_OF_CENTRAL_DIRECTORY,
 	WARNING_MULTIPLE_END_OF_CENTRAL_DIRECTORY,

package/package.json

@@ -3,7 +3,7 @@
   "description": "A JavaScript library to zip and unzip files in the browser, Deno and Node.js",
   "author": "Gildas Lormeau",
   "license": "BSD-3-Clause",
-  "version": "2.19.0",
+  "version": "2.21.0",
   "type": "module",
   "keywords": [
     "zip",