@futpib/parser 1.0.0 → 1.0.1

package/src/zipParser.ts

@@ -1,16 +1,23 @@
-import invariant from "invariant";
-import zlib from "node:zlib";
-import { createArrayParser } from "./arrayParser.js";
-import { createExactSequenceParser } from "./exactSequenceParser.js";
-import { Parser, setParserName } from "./parser.js";
-import { createTupleParser } from "./tupleParser.js";
-import { promiseCompose } from "./promiseCompose.js";
-import { createFixedLengthSequenceParser } from "./fixedLengthSequenceParser.js";
-import { createOptionalParser } from "./optionalParser.js";
-import { parserCreatorCompose } from "./parserCreatorCompose.js";
-import { Zip, ZipCompression, ZipDirectoryEntry, ZipEntry, ZipFileEntry, ZipPermissions } from "./zip.js";
-import { Readable } from "node:stream";
-import { pipeline } from "node:stream/promises";
+import zlib from 'node:zlib';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import invariant from 'invariant';
+import { createArrayParser } from './arrayParser.js';
+import { createExactSequenceParser } from './exactSequenceParser.js';
+import { type Parser, setParserName } from './parser.js';
+import { createTupleParser } from './tupleParser.js';
+import { promiseCompose } from './promiseCompose.js';
+import { createFixedLengthSequenceParser } from './fixedLengthSequenceParser.js';
+import { createOptionalParser } from './optionalParser.js';
+import { parserCreatorCompose } from './parserCreatorCompose.js';
+import {
+	type Zip,
+	type ZipCompression,
+	type ZipDirectoryEntry,
+	type ZipEntry,
+	type ZipFileEntry,
+} from './zip.js';
+import { uint8ArrayAsyncIterableToUint8Array } from './uint8Array.js';
 
 // https://pkwaredownloads.blob.core.windows.net/pem/APPNOTE.txt
 
@@ -34,12 +41,12 @@ const dosDateTimeParser: Parser<Date, Uint8Array> = promiseCompose(
 		uint16LEParser,
 	]),
 	([ time, date ]) => new Date(Date.UTC(
-		1980 + ((date >> 9) & 0x7f),
-		((date >> 5) & 0xf) - 1,
-		date & 0x1f,
-		(time >> 11) & 0x1f,
-		(time >> 5) & 0x3f,
-		(time & 0x1f) * 2,
+		1980 + ((date >> 9) & 0x7F),
+		((date >> 5) & 0xF) - 1,
+		date & 0x1F,
+		(time >> 11) & 0x1F,
+		(time >> 5) & 0x3F,
+		(time & 0x1F) * 2,
 	)),
 );
 
@@ -62,12 +69,12 @@ type ZipLocalFileHeader = {
 	versionNeededToExtract: number;
 	generalPurposeBitFlag: number;
 	compressionMethod: ZipCompression;
-	lastModFile: Date;
+	lastModifiedFile: Date;
 	crc32: number;
 	compressedSize: number;
 	uncompressedSize: number;
 
-	fileName: string;
+	filePath: string;
 	extraField: Uint8Array;
 };
 
@@ -86,10 +93,10 @@ const zipLocalFileHeaderParser_ = createTupleParser([
 			uint16LEParser,
 		]),
 		([
-			fileNameLength,
+			filePathLength,
 			extraFieldLength,
 		]) => createTupleParser([
-			createFixedLengthSequenceParser(fileNameLength),
+			createFixedLengthSequenceParser(filePathLength),
 			createFixedLengthSequenceParser(extraFieldLength),
 		]),
 	)(),
@@ -104,20 +111,20 @@ const zipLocalFileHeaderParser: Parser<ZipLocalFileHeader, Uint8Array> = promise
 		versionNeededToExtract,
 		generalPurposeBitFlag,
 		compressionMethod,
-		lastModFile,
+		lastModifiedFile,
 		crc32,
 		compressedSize,
 		uncompressedSize,
-		[ fileName, extraField ],
+		[ filePath, extraField ],
 	]) => ({
 		versionNeededToExtract,
 		generalPurposeBitFlag,
 		compressionMethod,
-		lastModFile,
+		lastModifiedFile,
 		crc32,
 		compressedSize,
 		uncompressedSize,
-		fileName: Buffer.from(fileName).toString('utf8'),
+		filePath: Buffer.from(filePath).toString('utf8'),
 		extraField,
 	}),
 );
@@ -135,7 +142,7 @@ const zipDataDescriptorParser: Parser<unknown, Uint8Array> = createTupleParser([
 	uint32LEParser,
 ]);
 
-type ZipLocalFile = {
+export type ZipLocalFile = {
 	zipLocalFileHeader: ZipLocalFileHeader;
 	zipEncryptionHeader: unknown;
 	compressedData: Uint8Array;
@@ -145,7 +152,7 @@ type ZipLocalFile = {
 export const zipLocalFileParser: Parser<ZipLocalFile, Uint8Array> = promiseCompose(
 	parserCreatorCompose(
 		() => zipLocalFileHeaderParser,
-		(zipLocalFileHeader) => createTupleParser([
+		zipLocalFileHeader => createTupleParser([
 			async () => zipLocalFileHeader,
 			createOptionalParser(zipEncryptionHeaderParser),
 			createFixedLengthSequenceParser(zipLocalFileHeader.compressedSize),
@@ -191,83 +198,117 @@ const zipVersionMadeByParser: Parser<ZipVersionMadeBy, Uint8Array> = promiseComp
 	}),
 );
 
-type ZipCentralDirectoryHeader = {
+type ZipExternalFileAttributes = {
+	directory: boolean;
+}
+
+const dosExternalFileAttributesParser: Parser<ZipExternalFileAttributes, Uint8Array> = promiseCompose(
+	uint32LEParser,
+	externalFileAttributes => ({
+		directory: (externalFileAttributes & 0b0001_0000) !== 0,
+	}),
+);
+
+const unixExternalFileAttributesParser: Parser<ZipExternalFileAttributes, Uint8Array> = promiseCompose(
+	uint32LEParser,
+	externalFileAttributes => ({
+		directory: (externalFileAttributes & (0b0100_0000_0000_0000 << 16)) !== 0,
+	}),
+);
+
+const createExternalFileAttributesParser = (hostSystem: number) => promiseCompose(
+	hostSystem === 0 ? dosExternalFileAttributesParser : unixExternalFileAttributesParser,
+	externalFileAttributes => externalFileAttributes,
+);
+
+export type ZipCentralDirectoryHeader = {
 	versionMadeBy: ZipVersionMadeBy;
 	versionNeededToExtract: number;
 	generalPurposeBitFlag: number;
 	compressionMethod: ZipCompression;
-	lastModFile: Date;
+	lastModifiedFile: Date;
 	crc32: number;
 	compressedSize: number;
 	uncompressedSize: number;
 	diskNumberStart: number;
 	internalFileAttributes: number;
-	externalFileAttributes: number;
+	externalFileAttributes: ZipExternalFileAttributes;
 	relativeOffsetOfLocalHeader: number;
 
-	fileName: string;
+	filePath: string;
 	extraField: Uint8Array;
 	fileComment: string;
 };
 
-const zipCentralDirectoryHeaderParser_ = createTupleParser([
-	createExactSequenceParser<Uint8Array>(Buffer.from('504b0102', 'hex')),
-	zipVersionMadeByParser,
-	uint16LEParser,
-	uint16LEParser,
-	zipCompressionMethodParser,
-	dosDateTimeParser,
-	uint32LEParser,
-	uint32LEParser,
-	uint32LEParser,
-	parserCreatorCompose(
-		() => createTupleParser([
-			uint16LEParser,
-			uint16LEParser,
-			uint16LEParser,
-			uint16LEParser,
-			uint16LEParser,
-			uint32LEParser,
-			uint32LEParser,
-		]),
-		([
-			fileNameLength,
-			extraFieldLength,
-			fileCommentLength,
-
-			diskNumberStart,
-			internalFileAttributes,
-			externalFileAttributes,
-			relativeOffsetOfLocalHeader,
-		]) => createTupleParser([
-			createFixedLengthSequenceParser(fileNameLength),
-			createFixedLengthSequenceParser(extraFieldLength),
-			createFixedLengthSequenceParser(fileCommentLength),
-
-			async () => diskNumberStart,
-			async () => internalFileAttributes,
-			async () => externalFileAttributes,
-			async () => relativeOffsetOfLocalHeader,
-		]),
-	)(),
-]);
+const zipCentralDirectoryHeaderParser_ = parserCreatorCompose(
+	() => createTupleParser([
+		createExactSequenceParser<Uint8Array>(Buffer.from('504b0102', 'hex')),
+		zipVersionMadeByParser,
+	]),
+	([
+		_centralDirectoryHeaderSignature,
+		versionMadeBy,
+	]) => createTupleParser([
+		async () => versionMadeBy,
+		uint16LEParser,
+		uint16LEParser,
+		zipCompressionMethodParser,
+		dosDateTimeParser,
+		uint32LEParser,
+		uint32LEParser,
+		uint32LEParser,
+		parserCreatorCompose(
+			() => createTupleParser([
+				uint16LEParser,
+				uint16LEParser,
+				uint16LEParser,
+
+				uint16LEParser,
+				uint16LEParser,
+
+				createExternalFileAttributesParser(versionMadeBy.hostSystem),
+				uint32LEParser,
+			]),
+			([
+				filePathLength,
+				extraFieldLength,
+				fileCommentLength,
+
+				diskNumberStart,
+				internalFileAttributes,
+
+				externalFileAttributes,
+				relativeOffsetOfLocalHeader,
+			]) => createTupleParser([
+				createFixedLengthSequenceParser(filePathLength),
+				createFixedLengthSequenceParser(extraFieldLength),
+				createFixedLengthSequenceParser(fileCommentLength),
+
+				async () => diskNumberStart,
+				async () => internalFileAttributes,
+
+				async () => externalFileAttributes,
+				async () => relativeOffsetOfLocalHeader,
+			]),
+		)(),
+	]),
+)();
 
 setParserName(zipCentralDirectoryHeaderParser_, 'centralDirectoryHeaderParser_');
 
 export const zipCentralDirectoryHeaderParser: Parser<ZipCentralDirectoryHeader, Uint8Array> = promiseCompose(
 	zipCentralDirectoryHeaderParser_,
 	([
-		_centralDirectoryHeaderSignature,
 		versionMadeBy,
 		versionNeededToExtract,
 		generalPurposeBitFlag,
 		compressionMethod,
-		lastModFile,
+		lastModifiedFile,
 		crc32,
 		compressedSize,
 		uncompressedSize,
 		[
-			fileName,
+			filePath,
 			extraField,
 			fileComment,
 
@@ -281,7 +322,7 @@ export const zipCentralDirectoryHeaderParser: Parser<ZipCentralDirectoryHeader,
 		versionNeededToExtract,
 		generalPurposeBitFlag,
 		compressionMethod,
-		lastModFile,
+		lastModifiedFile,
 		crc32,
 		compressedSize,
 		uncompressedSize,
@@ -289,7 +330,7 @@ export const zipCentralDirectoryHeaderParser: Parser<ZipCentralDirectoryHeader,
 		internalFileAttributes,
 		externalFileAttributes,
 		relativeOffsetOfLocalHeader,
-		fileName: Buffer.from(fileName).toString('utf8'),
+		filePath: Buffer.from(filePath).toString('utf8'),
 		extraField,
 		fileComment: Buffer.from(fileComment).toString('utf8'),
 	}),
@@ -311,14 +352,14 @@ const zipFileCommentParser: Parser<string, Uint8Array> = promiseCompose(
 	uint8Array => Buffer.from(uint8Array).toString('utf8'),
 );
 
-type ZipEndOfCentralDirectoryRecord = {
-	numberOfThisDisk: number,
-	numberOfTheDiskWithTheStartOfTheCentralDirectory: number,
-	totalNumberOfEntriesInTheCentralDirectoryOnThisDisk: number,
-	totalNumberOfEntriesInTheCentralDirectory: number,
-	sizeOfTheCentralDirectory: number,
-	offsetOfStartOfCentralDirectoryWithRespectToTheStartingDiskNumber: number,
-	zipFileComment: string,
+export type ZipEndOfCentralDirectoryRecord = {
+	numberOfThisDisk: number;
+	numberOfTheDiskWithTheStartOfTheCentralDirectory: number;
+	totalNumberOfEntriesInTheCentralDirectoryOnThisDisk: number;
+	totalNumberOfEntriesInTheCentralDirectory: number;
+	sizeOfTheCentralDirectory: number;
+	offsetOfStartOfCentralDirectoryWithRespectToTheStartingDiskNumber: number;
+	zipFileComment: string;
 };
 
 const zipEndOfCentralDirectoryRecordParser_ = createTupleParser([
@@ -372,10 +413,10 @@ export async function zipEntriesFromZipSegments({
 	zipLocalFiles,
 	zipCentralDirectoryHeaders,
 }: {
-	zipLocalFiles: ZipLocalFile[],
-	zipCentralDirectoryHeaders: ZipCentralDirectoryHeader[],
+	zipLocalFiles: ZipLocalFile[];
+	zipCentralDirectoryHeaders: ZipCentralDirectoryHeader[];
 }) {
-	const decompressPromises: Promise<void>[] = [];
+	const decompressPromises: Array<Promise<void>> = [];
 
 	const entries: ZipEntry[] = await Promise.all(zipLocalFiles.map(async (zipLocalFile, index) => {
 		const {
@@ -389,41 +430,24 @@ export async function zipEntriesFromZipSegments({
 
 		invariant(centralDirectoryHeader, 'Central directory header not found for local file %s', index);
 
-		let isDosDirectory = false;
-		let permissions: ZipPermissions = {
-			type: 'unix',
-			unixPermissions: 0,
-		};
-
-		if (centralDirectoryHeader.versionMadeBy.hostSystem === 0) {
-			isDosDirectory = (centralDirectoryHeader.externalFileAttributes & 0b00010000) !== 0;
-			permissions = {
-				type: 'dos',
-				dosPermissions: centralDirectoryHeader.externalFileAttributes & 0b00111111,
-			};
-		}
-
-		if (centralDirectoryHeader.versionMadeBy.hostSystem === 3) {
-			permissions = {
-				type: 'unix',
-				unixPermissions: (centralDirectoryHeader.externalFileAttributes >> 16) & 0b111111111,
-			};
-		}
-
 		const commonFields: Omit<ZipFileEntry | ZipDirectoryEntry, 'type'> = {
-			path: zipLocalFileHeader.fileName,
+			path: zipLocalFileHeader.filePath,
 			comment: centralDirectoryHeader.fileComment,
-			date: zipLocalFileHeader.lastModFile,
-			permissions,
+			date: zipLocalFileHeader.lastModifiedFile,
+			hostSystem: centralDirectoryHeader.versionMadeBy.hostSystem === 0 ? 'dos' : 'unix',
+			attributes: centralDirectoryHeader.externalFileAttributes,
 		};
 
-		const isDirectory = isDosDirectory || commonFields.path.endsWith('/');
+		const isDirectory = (
+			centralDirectoryHeader.externalFileAttributes.directory
+				|| commonFields.path.endsWith('/')
+		);
 
 		if (isDirectory) {
 			return {
 				...commonFields,
 				type: 'directory',
-				path: commonFields.path.slice(0, -1),
+				path: commonFields.path.replace(/\/$/, ''),
 			};
 		}
 
@@ -434,18 +458,12 @@ export async function zipEntriesFromZipSegments({
 			compression: zipLocalFileHeader.compressionMethod,
 		};
 
-		if (fileEntry.content.length && fileEntry.compression === 'deflate') {
-			const deflate = zlib.createInflateRaw();
+		if (fileEntry.content.length > 0 && fileEntry.compression === 'deflate') {
+			const inflate = zlib.createInflateRaw();
 			const input = Readable.from(Buffer.from(compressedData));
 			const [ _, buffer ] = await Promise.all([
-				pipeline(input, deflate),
-				(async () => {
-					const chunks: Buffer[] = [];
-					for await (const chunk of deflate) {
-						chunks.push(chunk);
-					}
-					return Buffer.concat(chunks);
-				})(),
+				pipeline(input, inflate),
+				uint8ArrayAsyncIterableToUint8Array(inflate),
 			]);
 
 			fileEntry.content = Uint8Array.from(buffer);
@@ -465,10 +483,10 @@ export async function zipFromZipSegments({
 
 	zipEndOfCentralDirectoryRecord,
 }: {
-	zipLocalFiles: ZipLocalFile[],
-	zipCentralDirectoryHeaders: ZipCentralDirectoryHeader[],
+	zipLocalFiles: ZipLocalFile[];
+	zipCentralDirectoryHeaders: ZipCentralDirectoryHeader[];
 
-	zipEndOfCentralDirectoryRecord: ZipEndOfCentralDirectoryRecord,
+	zipEndOfCentralDirectoryRecord: ZipEndOfCentralDirectoryRecord;
 }) {
 	const entries = await zipEntriesFromZipSegments({
 		zipLocalFiles,

package/src/zipUnparser.test.ts

@@ -0,0 +1,119 @@
+import { testProp } from '@fast-check/ava';
+import { temporaryFile } from 'tempy';
+import { execa } from 'execa';
+import fsPromises from 'node:fs/promises';
+import { runUnparser } from './unparser.js';
+import { zipUnparser } from './zipUnparser.js';
+import { uint8ArrayUnparserOutputCompanion } from './unparserOutputCompanion.js';
+import { runParser } from './parser.js';
+import { zipParser } from './zipParser.js';
+import { uint8ArrayParserInputCompanion } from './parserInputCompanion.js';
+import { arbitraryZip } from './arbitraryZip.js';
+
+async function hasExecutable(executable: string) {
+	const hasExecutable = execa(executable).then(() => true, () => false);
+
+	if (!hasExecutable) {
+		console.warn(`Executable %o not found`, executable);
+	}
+
+	return hasExecutable;
+}
+
+const hasZipinfoPromise = hasExecutable('zipinfo');
+const has7zPromise = hasExecutable('7z');
+
+async function zipinfo(zipFilePath: string) {
+	const { stdout } = await execa('zipinfo', [
+		'-v',
+		zipFilePath,
+	], {
+		reject: false,
+	});
+
+	return {
+		isEmptyZipfile: stdout.endsWith('Empty zipfile.'),
+	};
+}
+
+async function _7zList(zipFilePath: string) {
+	const { stdout } = await execa('7z', [
+		'l',
+		zipFilePath,
+	], {
+		reject: false,
+	});
+
+	if (/Errors: \d+$/.test(stdout)) {
+		throw new Error(`7z reports errors: ${stdout}`);
+	}
+
+	return {
+		isEmptyZipfile: false, // TODO
+	};
+}
+
+testProp(
+	'zip',
+	[
+		arbitraryZip,
+	],
+	async (t, zip) => {
+		const actualStream = runUnparser(zipUnparser, zip, uint8ArrayUnparserOutputCompanion);
+		const actual = await runParser(zipParser, actualStream, uint8ArrayParserInputCompanion);
+
+		const isDeepEqual = t.deepEqual(actual, zip);
+
+		if (!isDeepEqual) {
+			return;
+		}
+
+		const hasZipinfo = await hasZipinfoPromise;
+		const has7z = await has7zPromise;
+
+		if (!hasZipinfo && !has7z) {
+			return;
+		}
+
+		const temporaryFilePath = temporaryFile({
+			extension: 'zip',
+		});
+
+		const zipStream = runUnparser(zipUnparser, zip, uint8ArrayUnparserOutputCompanion);
+
+		await fsPromises.writeFile(temporaryFilePath, zipStream);
+
+		if (has7z) {
+			const {
+				isEmptyZipfile,
+			} = await _7zList(temporaryFilePath);
+
+			if (isEmptyZipfile) {
+				t.deepEqual(
+					actual.entries,
+					[],
+					'7z reports the zipfile as empty, but it has entries',
+				);
+			}
+		}
+
+		if (hasZipinfo) {
+			const {
+				isEmptyZipfile,
+			} = await zipinfo(temporaryFilePath);
+
+			if (isEmptyZipfile) {
+				t.deepEqual(
+					actual.entries.filter(entry => entry.type === 'file'),
+					[],
+					'zipinfo reports the zipfile as empty, but it has file entries',
+				);
+			}
+		}
+
+		await fsPromises.unlink(temporaryFilePath);
+	},
+	{
+		verbose: true,
+	},
+);