@futpib/parser 1.0.1 → 1.0.2

package/build/zipParser.js

@@ -11,6 +11,8 @@ import { createFixedLengthSequenceParser } from './fixedLengthSequenceParser.js'
 import { createOptionalParser } from './optionalParser.js';
 import { parserCreatorCompose } from './parserCreatorCompose.js';
 import { uint8ArrayAsyncIterableToUint8Array } from './uint8Array.js';
+import { createNegativeLookaheadParser } from './negativeLookaheadParser.js';
+import { createSequenceTerminatedSequenceParser } from './sequenceTerminatedSequenceParser.js';
 // https://pkwaredownloads.blob.core.windows.net/pem/APPNOTE.txt
 const uint16LEParser = promiseCompose(createFixedLengthSequenceParser(2), array => Buffer.from(array).readUInt16LE());
 setParserName(uint16LEParser, 'uint16LEParser');
@@ -29,8 +31,9 @@ const zipCompressionMethodParser = promiseCompose(uint16LEParser, compressionMet
     }
     invariant(false, 'Unsupported compression method %s', compressionMethod);
 });
+const zipLocalFileHeaderSignatureParser = createExactSequenceParser(Buffer.from('504b0304', 'hex'));
 const zipLocalFileHeaderParser_ = createTupleParser([
-    createExactSequenceParser(Buffer.from('504b0304', 'hex')),
+    zipLocalFileHeaderSignatureParser,
     uint16LEParser,
     uint16LEParser,
     zipCompressionMethodParser,
@@ -61,20 +64,40 @@ const zipLocalFileHeaderParser = promiseCompose(zipLocalFileHeaderParser_, ([_zi
 const zipEncryptionHeaderParser = async (parserContext) => {
     parserContext.invariant(false, 'Not implemented');
 };
-const zipDataDescriptorParser = createTupleParser([
+const zipDataDescriptorSignature = Buffer.from('504b0708', 'hex');
+const zipDataDescriptorSignatureParser = createExactSequenceParser(zipDataDescriptorSignature);
+const zipDataDescriptorParser = promiseCompose(createTupleParser([
+    createNegativeLookaheadParser(zipLocalFileHeaderSignatureParser),
     // FIXME: optional in spec
-    // createOptionalParser(createExactSequenceParser<Uint8Array>(Buffer.from('504b0708', 'hex'))),
-    createExactSequenceParser(Buffer.from('504b0708', 'hex')),
+    //createOptionalParser(zipDataDescriptorSignatureParser),
+    zipDataDescriptorSignatureParser,
     uint32LEParser,
     uint32LEParser,
     uint32LEParser,
-]);
-export const zipLocalFileParser = promiseCompose(parserCreatorCompose(() => zipLocalFileHeaderParser, zipLocalFileHeader => createTupleParser([
-    async () => zipLocalFileHeader,
-    createOptionalParser(zipEncryptionHeaderParser),
-    createFixedLengthSequenceParser(zipLocalFileHeader.compressedSize),
-    createOptionalParser(zipDataDescriptorParser),
-]))(), ([zipLocalFileHeader, zipEncryptionHeader, compressedData, zipDataDescriptor,]) => ({
+]), ([_notZipLocalFileHeaderSignature, _zipDataDescriptorSignature, crc32, compressedSize, uncompressedSize,]) => ({
+    crc32,
+    compressedSize,
+    uncompressedSize,
+}));
+setParserName(zipDataDescriptorParser, 'zipDataDescriptorParser');
+export const zipLocalFileParser = promiseCompose(parserCreatorCompose(() => zipLocalFileHeaderParser, zipLocalFileHeader => {
+    const sizeInDataDescriptor = Boolean(zipLocalFileHeader.generalPurposeBitFlag & 0b0000_0000_0000_1000
+        && zipLocalFileHeader.crc32 === 0
+        && zipLocalFileHeader.compressedSize === 0
+        && zipLocalFileHeader.uncompressedSize === 0);
+    return createTupleParser([
+        async () => zipLocalFileHeader,
+        createOptionalParser(zipEncryptionHeaderParser),
+        (sizeInDataDescriptor
+            ? createSequenceTerminatedSequenceParser(zipDataDescriptorSignature, {
+                consumeTerminator: false,
+            })
+            : createFixedLengthSequenceParser(zipLocalFileHeader.compressedSize)),
+        (sizeInDataDescriptor
+            ? ((parser) => parser)
+            : createOptionalParser)(zipDataDescriptorParser),
+    ]);
+})(), ([zipLocalFileHeader, zipEncryptionHeader, compressedData, zipDataDescriptor,]) => ({
     zipLocalFileHeader,
     zipEncryptionHeader,
     compressedData,
@@ -163,7 +186,7 @@ const zipEndOfCentralDirectoryRecordParser_ = createTupleParser([
     uint32LEParser,
     zipFileCommentParser,
 ]);
-setParserName(zipEndOfCentralDirectoryRecordParser_, 'endOfCentralDirectoryRecordParser_');
+setParserName(zipEndOfCentralDirectoryRecordParser_, 'zipEndOfCentralDirectoryRecordParser_');
 export const zipEndOfCentralDirectoryRecordParser = promiseCompose(zipEndOfCentralDirectoryRecordParser_, ([_endOfCentralDirectoryRecordSignature, numberOfThisDisk, numberOfTheDiskWithTheStartOfTheCentralDirectory, totalNumberOfEntriesInTheCentralDirectoryOnThisDisk, totalNumberOfEntriesInTheCentralDirectory, sizeOfTheCentralDirectory, offsetOfStartOfCentralDirectoryWithRespectToTheStartingDiskNumber, zipFileComment,]) => ({
     numberOfThisDisk,
     numberOfTheDiskWithTheStartOfTheCentralDirectory,
@@ -173,6 +196,7 @@ export const zipEndOfCentralDirectoryRecordParser = promiseCompose(zipEndOfCentr
     offsetOfStartOfCentralDirectoryWithRespectToTheStartingDiskNumber,
     zipFileComment,
 }));
+setParserName(zipEndOfCentralDirectoryRecordParser, 'zipEndOfCentralDirectoryRecordParser');
 const zipParser_ = createTupleParser([
     createArrayParser(zipLocalFileParser),
     createOptionalParser(zipArchiveDecryptionHeaderParser),

package/build/zipUnparser.d.ts

@@ -2,4 +2,7 @@ import { Unparser } from "./unparser.js";
 import { Zip } from "./zip.js";
 import { ZipEndOfCentralDirectoryRecord } from './zipParser.js';
 export declare const zipEndOfCentralDirectoryRecordUnparser: Unparser<ZipEndOfCentralDirectoryRecord, Uint8Array>;
-export declare const zipUnparser: Unparser<Zip, Uint8Array>;
+export declare const createZipUnparser: ({ dataDescriptor, }?: {
+    dataDescriptor?: boolean;
+}) => Unparser<Zip, Uint8Array>;
+export declare const zipUnparser: Unparser<Zip, Uint8Array<ArrayBufferLike>>;

package/build/zipUnparser.js

@@ -45,7 +45,28 @@ export const zipEndOfCentralDirectoryRecordUnparser = async function* (zipEndOfC
     yield* uint32LEUnparser(zipEndOfCentralDirectoryRecord.offsetOfStartOfCentralDirectoryWithRespectToTheStartingDiskNumber, unparserContext);
     yield* uint16LEPrefixedStringUnparser(zipEndOfCentralDirectoryRecord.zipFileComment, unparserContext);
 };
-export const zipUnparser = async function* (zip, unparserContext) {
+const zipDataDescriptorUnparser = async function* ({ crc32, compressedSize, uncompressedSize, }, unparserContext) {
+    yield Buffer.from('504b0708', 'hex');
+    yield* uint32LEUnparser(crc32, unparserContext);
+    yield* uint32LEUnparser(compressedSize, unparserContext);
+    yield* uint32LEUnparser(uncompressedSize, unparserContext);
+};
+const zipLocalFileHeaderUnparser = async function* ({ versionNeededToExtract, generalPurposeBitFlag, compressionMethod, lastModifiedFile, crc32, compressedSize, uncompressedSize, filePath, extraField, }, unparserContext) {
+    yield Buffer.from('504b0304', 'hex');
+    yield* uint16LEUnparser(versionNeededToExtract, unparserContext);
+    yield* uint16LEUnparser(generalPurposeBitFlag, unparserContext);
+    yield* zipCompressionMethodUnparser(compressionMethod, unparserContext);
+    yield* dosDateTimeUnparser(lastModifiedFile, unparserContext);
+    yield* uint32LEUnparser(crc32, unparserContext);
+    yield* uint32LEUnparser(compressedSize, unparserContext);
+    yield* uint32LEUnparser(uncompressedSize, unparserContext);
+    const filePathBuffer = Buffer.from(filePath, 'utf8');
+    yield* uint16LEUnparser(filePathBuffer.length, unparserContext);
+    yield* uint16LEUnparser(extraField.length, unparserContext);
+    yield filePathBuffer;
+    yield extraField;
+};
+export const createZipUnparser = ({ dataDescriptor = false, } = {}) => async function* (zip, unparserContext) {
     const compressedContentByZipFileEntry = new Map(zip.entries.flatMap(zipEntry => {
         if (zipEntry.type !== 'file') {
             return [];
@@ -66,37 +87,44 @@ export const zipUnparser = async function* (zip, unparserContext) {
     const localHeaderPositionByZipEntry = new Map();
     for (const zipEntry of zip.entries) {
         localHeaderPositionByZipEntry.set(zipEntry, unparserContext.position);
-        yield Buffer.from('504b0304', 'hex');
-        yield* uint16LEUnparser(10, unparserContext); // version needed to extract
-        yield* uint16LEUnparser(0, unparserContext); // general purpose bit flag
-        if (zipEntry.type === 'file') {
-            yield* zipCompressionMethodUnparser(zipEntry.compression, unparserContext);
-        }
-        else {
-            yield* uint16LEUnparser(0, unparserContext);
-        }
-        yield* dosDateTimeUnparser(zipEntry.date, unparserContext);
+        const filePath = filePathByZipEntry.get(zipEntry);
+        const zipLocalFileHeader = {
+            versionNeededToExtract: 20,
+            generalPurposeBitFlag: 0,
+            compressionMethod: 'store',
+            lastModifiedFile: zipEntry.date,
+            crc32: 0,
+            compressedSize: 0,
+            uncompressedSize: 0,
+            filePath,
+            extraField: Buffer.alloc(0),
+        };
+        let shouldWriteDataDescriptor = false;
         if (zipEntry.type === 'file') {
             const compressedContent = await compressedContentByZipFileEntry.get(zipEntry);
-            yield* uint32LEUnparser(0, unparserContext); // crc32 // TODO
-            yield* uint32LEUnparser(compressedContent.length, unparserContext);
-            yield* uint32LEUnparser(zipEntry.content.length, unparserContext);
+            zipLocalFileHeader.compressionMethod = zipEntry.compression;
+            zipLocalFileHeader.crc32 = 0; // TODO
+            zipLocalFileHeader.compressedSize = compressedContent.length;
+            zipLocalFileHeader.uncompressedSize = zipEntry.content.length;
+            if (dataDescriptor) {
+                shouldWriteDataDescriptor = true;
+                zipLocalFileHeader.generalPurposeBitFlag |= 0b0000_0000_0000_1000;
+                zipLocalFileHeader.crc32 = 0;
+                zipLocalFileHeader.compressedSize = 0;
+                zipLocalFileHeader.uncompressedSize = 0;
+            }
         }
-        else {
-            yield* uint32LEUnparser(0, unparserContext);
-            yield* uint32LEUnparser(0, unparserContext);
-            yield* uint32LEUnparser(0, unparserContext);
-        }
-        const filePath = filePathByZipEntry.get(zipEntry);
-        const filePathBuffer = Buffer.from(filePath, 'utf8');
-        const extraFieldBuffer = Buffer.alloc(0);
-        yield* uint16LEUnparser(filePathBuffer.length, unparserContext);
-        yield* uint16LEUnparser(extraFieldBuffer.length, unparserContext);
-        yield filePathBuffer;
-        yield extraFieldBuffer;
+        yield* zipLocalFileHeaderUnparser(zipLocalFileHeader, unparserContext);
         if (zipEntry.type === 'file') {
             const compressedContent = await compressedContentByZipFileEntry.get(zipEntry);
             yield compressedContent;
+            if (shouldWriteDataDescriptor) {
+                yield* zipDataDescriptorUnparser({
+                    crc32: zipLocalFileHeader.crc32,
+                    compressedSize: zipLocalFileHeader.compressedSize,
+                    uncompressedSize: zipLocalFileHeader.uncompressedSize,
+                }, unparserContext);
+            }
         }
     }
     const startOfCentralDirectoryPosition = unparserContext.position;
@@ -169,3 +197,4 @@ export const zipUnparser = async function* (zip, unparserContext) {
         zipFileComment: zip.comment,
     }, unparserContext);
 };
+export const zipUnparser = createZipUnparser();

package/build/zipUnparser.test.js

@@ -1,21 +1,15 @@
-import { testProp } from '@fast-check/ava';
+import { fc, testProp } from '@fast-check/ava';
 import { temporaryFile } from 'tempy';
 import { execa } from 'execa';
 import fsPromises from 'node:fs/promises';
 import { runUnparser } from './unparser.js';
-import { zipUnparser } from './zipUnparser.js';
+import { createZipUnparser } from './zipUnparser.js';
 import { uint8ArrayUnparserOutputCompanion } from './unparserOutputCompanion.js';
 import { runParser } from './parser.js';
 import { zipParser } from './zipParser.js';
 import { uint8ArrayParserInputCompanion } from './parserInputCompanion.js';
 import { arbitraryZip } from './arbitraryZip.js';
-async function hasExecutable(executable) {
-    const hasExecutable = execa(executable).then(() => true, () => false);
-    if (!hasExecutable) {
-        console.warn(`Executable %o not found`, executable);
-    }
-    return hasExecutable;
-}
+import { hasExecutable } from './hasExecutable.js';
 const hasZipinfoPromise = hasExecutable('zipinfo');
 const has7zPromise = hasExecutable('7z');
 async function zipinfo(zipFilePath) {
@@ -45,16 +39,22 @@ async function _7zList(zipFilePath) {
 }
 testProp('zip', [
     arbitraryZip,
-], async (t, zip) => {
+    fc.record({
+        dataDescriptor: fc.boolean(),
+    }),
+], async (t, zip, options) => {
+    const zipUnparser = createZipUnparser(options);
+    {
+        const zipStream = runUnparser(zipUnparser, zip, uint8ArrayUnparserOutputCompanion);
+        await fsPromises.writeFile('/tmp/zip.zip', zipStream);
+    }
     const actualStream = runUnparser(zipUnparser, zip, uint8ArrayUnparserOutputCompanion);
     const actual = await runParser(zipParser, actualStream, uint8ArrayParserInputCompanion);
-    const isDeepEqual = t.deepEqual(actual, zip);
-    if (!isDeepEqual) {
-        return;
-    }
+    t.deepEqual(actual, zip);
     const hasZipinfo = await hasZipinfoPromise;
     const has7z = await has7zPromise;
     if (!hasZipinfo && !has7z) {
+        t.pass('no zipinfo or 7z, skipping zipfile check');
         return;
     }
     const temporaryFilePath = temporaryFile({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@futpib/parser",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "main": "build/index.js",
   "license": "GPL-3.0-only",
   "type": "module",
@@ -24,20 +24,24 @@
     "@ava/typescript": "^5.0.0",
     "@fast-check/ava": "^2.0.1",
     "@types/invariant": "^2.2.37",
-    "@types/node": "^22.10.1",
+    "@types/node": "^22.10.2",
     "ava": "^6.2.0",
-    "bson": "^6.10.0",
-    "c8": "^10.1.2",
+    "bson": "^6.10.1",
+    "c8": "^10.1.3",
     "coveralls": "^3.1.1",
+    "env-paths": "^3.0.0",
     "eslint-config-xo-typescript-overrides": "^1.6.1",
     "execa": "^9.5.2",
-    "fast-check": "^3.23.1",
+    "fast-check": "^3.23.2",
     "invariant": "^2.2.4",
     "jszip": "^3.10.1",
+    "leb128": "^0.0.5",
+    "mutf-8": "^1.1.4",
+    "p-memoize": "^7.1.1",
     "tempy": "^3.1.0",
-    "type-fest": "^4.29.0",
+    "type-fest": "^4.30.1",
     "typescript": "^5.7.2",
-    "xo": "^0.59.3"
+    "xo": "^0.60.0"
   },
   "xo": {
     "extends": [
@@ -49,6 +53,18 @@
   },
   "packageManager": "yarn@4.5.3",
   "dependencies": {
+    "fp-ts": "^2.16.9",
+    "mem": "^10.0.0",
+    "monocle-ts": "^2.3.13",
+    "newtype-ts": "^0.3.5",
     "p-mutex": "^0.1.0"
+  },
+  "peerDependencies": {
+    "mutf-8": "^1.1.4"
+  },
+  "peerDependenciesMeta": {
+    "mutf-8": {
+      "optional": true
+    }
   }
 }

package/src/allSettledStream.test.ts

@@ -57,3 +57,43 @@ test('allSettledStream', async t => {
 	t.snapshot(results);
 	t.is(results.length, 8);
 });
+
+test('allSettledStream reader cancel', async t => {
+	const stream = allSettledStream<number, number>([
+		10,
+		20,
+		0,
+		40,
+		30,
+	].map((delay) => ({
+		promise: new Promise<number>((resolve) => {
+			setTimeout(() => {
+				resolve(delay);
+			}, delay);
+		}),
+		context: delay,
+	})));
+
+	const results: any[] = [];
+
+	for await (const value of stream) {
+		results.push(value);
+
+		if (value.context === 30) {
+			break;
+		}
+	}
+
+	await new Promise<void>(resolve => {
+		setTimeout(resolve, 50);
+	});
+
+	stream.cancel();
+
+	for await (const value of stream) {
+		t.fail();
+		results.push(value);
+	}
+
+	t.deepEqual(results.map(({ context }) => context), [ 0, 10, 20, 30 ]);
+});

package/src/allSettledStream.ts

@@ -1,3 +1,5 @@
+import { promiseSettled } from "./promiseSettled.js";
+
 type AllSettledStreamFulfilledResult<T, Context> = {
 	status: 'fulfilled';
 	value: T;
@@ -18,27 +20,57 @@ type AllSettledStreamTask<T, Context> = {
 };
 
 export function allSettledStream<T, Context>(tasks: Array<AllSettledStreamTask<T, Context>>): ReadableStream<AllSettedStreamResult<T, Context>> {
+	let cancelled = false;
+
 	return new ReadableStream({
-		async start(controller) {
-			let settledCount = 0;
-			for (const { promise, context } of tasks) {
-				(async () => {
-					const [ promiseSettledResult ] = await Promise.allSettled([ promise ]);
+		start(controller) {
+			const startStack = (
+				(new Error('allSettledStream ReadableStream start stack holder').stack ?? '')
+					.split('\n')
+					.slice(1)
+					.join('\n')
+			);
+
+			const allSettledStreamTaskAwaiter = async ({
+				promise,
+				context,
+			}: AllSettledStreamTask<T, Context>) => {
+				const promiseSettledResult = await promiseSettled(promise);
+
+				if (
+					promiseSettledResult.status === 'rejected'
+						&& promiseSettledResult.reason instanceof Error
+						&& promiseSettledResult.reason.stack
+				) {
+					promiseSettledResult.reason.stack += `\n${startStack}`;
+				}
 
-					settledCount++;
+				settledCount++;
 
-					const allSettedStreamResult: AllSettedStreamResult<T, Context> = {
-						...promiseSettledResult,
-						context,
-					};
+				if (cancelled) {
+					return;
+				}
 
-					controller.enqueue(allSettedStreamResult);
+				const allSettedStreamResult: AllSettedStreamResult<T, Context> = {
+					...promiseSettledResult,
+					context,
+				};
 
-					if (settledCount === tasks.length) {
-						controller.close();
-					}
-				})();
+				controller.enqueue(allSettedStreamResult);
+
+				if (settledCount === tasks.length) {
+					controller.close();
+				}
+			};
+
+			let settledCount = 0;
+			for (const task of tasks) {
+				allSettledStreamTaskAwaiter(task);
 			}
 		},
+
+		cancel() {
+			cancelled = true;
+		},
 	});
 }

package/src/androidPackage.ts

@@ -0,0 +1,48 @@
+import { type Zip } from './zip.js';
+
+export type AndroidPackageSigningBlockPair = {
+	id: number;
+	value: Uint8Array;
+};
+
+export type AndroidPackageSignatureV2Digest = {
+	signatureAlgorithmId: number;
+	digest: Uint8Array;
+};
+
+export type AndroidPackageSignatureV2AdditionalAttribute = {
+	id: number;
+	value: Uint8Array;
+};
+
+export type AndroidPackageSignatureV2SignedData = {
+	digests: AndroidPackageSignatureV2Digest[];
+	certificates: Uint8Array[];
+	additionalAttributes: AndroidPackageSignatureV2AdditionalAttribute[];
+	zeroPaddingLength?: number;
+};
+
+export type AndroidPackageSignatureV2Signature = {
+	signatureAlgorithmId: number;
+	signature: Uint8Array;
+};
+
+export type AndroidPackageSignatureV2Signer = {
+	signedData: AndroidPackageSignatureV2SignedData;
+	signatures: AndroidPackageSignatureV2Signature[];
+	publicKey: Uint8Array;
+};
+
+export type AndroidPackageSignatureV2 = {
+	signers: AndroidPackageSignatureV2Signer[];
+};
+
+export type AndroidPackageSigningBlock = {
+	pairs: AndroidPackageSigningBlockPair[];
+	signatureV2?: AndroidPackageSignatureV2;
+	zeroPaddingLength?: number;
+};
+
+export type AndroidPackage = Zip & {
+	signingBlock?: AndroidPackageSigningBlock;
+};

package/src/{apkParser.test.ts → androidPackageParser.test.ts}

@@ -1,20 +1,19 @@
 import test from 'ava';
 import { uint8ArrayParserInputCompanion } from './parserInputCompanion.js';
 import { runParser } from './parser.js';
-import { apkParser } from './apkParser.js';
+import { androidPackageParser } from './androidPackageParser.js';
+import { fetchCid } from './fetchCid.js';
 
-for (const apkCid of [
+for (const androidPackageCid of [
 	'bafkreicckcmzrdxwoc3w2in3tivpyxrdtcfpct4zwauq3igew3nkpvfapu',
 	'bafkreidqycbgrtp7ywhevsgtm464dqpamxsijpaqfcfz2k3ymc3wrzsb5m',
 ]) {
 	test(
-		'apk ' + apkCid,
+		'androidPackage ' + androidPackageCid,
 		async t => {
-			const apkResponse = await fetch('https://ipfs.io/ipfs/' + apkCid);
+			const androidPackageStream = await fetchCid(androidPackageCid);
 
-			const apkStream = apkResponse.body!;
-
-			const actual = await runParser(apkParser, apkStream, uint8ArrayParserInputCompanion, {
+			const actual = await runParser(androidPackageParser, androidPackageStream, uint8ArrayParserInputCompanion, {
 				errorJoinMode: 'all',
 			});
 

package/src/{apkParser.test.ts.md → androidPackageParser.test.ts.md}

@@ -1,10 +1,10 @@
-# Snapshot report for `src/apkParser.test.ts`
+# Snapshot report for `src/androidPackageParser.test.ts`
 
-The actual snapshot is saved in `apkParser.test.ts.snap`.
+The actual snapshot is saved in `androidPackageParser.test.ts.snap`.
 
 Generated by [AVA](https://avajs.dev).
 
-## apk bafkreicckcmzrdxwoc3w2in3tivpyxrdtcfpct4zwauq3igew3nkpvfapu
+## androidPackage bafkreicckcmzrdxwoc3w2in3tivpyxrdtcfpct4zwauq3igew3nkpvfapu
 
 > Snapshot 1
 
@@ -189,7 +189,7 @@ Generated by [AVA](https://avajs.dev).
       },
     }
 
-## apk bafkreidqycbgrtp7ywhevsgtm464dqpamxsijpaqfcfz2k3ymc3wrzsb5m
+## androidPackage bafkreidqycbgrtp7ywhevsgtm464dqpamxsijpaqfcfz2k3ymc3wrzsb5m
 
 > Snapshot 1