@php-wasm/stream-compression 0.0.1

package/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+	"extends": ["../../../.eslintrc.json"],
+	"ignorePatterns": ["!**/*"],
+	"overrides": [
+		{
+			"files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+			"rules": {}
+		},
+		{
+			"files": ["*.ts", "*.tsx"],
+			"rules": {}
+		},
+		{
+			"files": ["*.js", "*.jsx"],
+			"rules": {}
+		}
+	]
+}

package/README.md

@@ -0,0 +1,11 @@
+# php-wasm-stream-compression
+
+This library was generated with [Nx](https://nx.dev).
+
+## Building
+
+Run `nx build php-wasm-stream-compression` to build the library.
+
+## Running unit tests
+
+Run `nx test php-wasm-stream-compression` to execute the unit tests via [Jest](https://jestjs.io).

package/package.json

@@ -0,0 +1,31 @@
+{
+	"name": "@php-wasm/stream-compression",
+	"version": "0.0.1",
+	"description": "Stream-based compression bindings.",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/WordPress/wordpress-playground"
+	},
+	"homepage": "https://developer.wordpress.org/playground",
+	"author": "The WordPress contributors",
+	"contributors": [
+		{
+			"name": "Adam Zielinski",
+			"email": "adam@adamziel.com",
+			"url": "https://github.com/adamziel"
+		}
+	],
+	"exports": {
+		".": {
+			"import": "./index.js",
+			"require": "./index.cjs"
+		},
+		"./package.json": "./package.json"
+	},
+	"publishConfig": {
+		"access": "public",
+		"directory": "../../../dist/packages/php-wasm/stream-compression"
+	},
+	"license": "GPL-2.0-or-later",
+	"type": "module"
+}

package/project.json

@@ -0,0 +1,34 @@
+{
+	"name": "php-wasm-stream-compression",
+	"$schema": "../../../node_modules/nx/schemas/project-schema.json",
+	"sourceRoot": "packages/php-wasm/stream-compression/src",
+	"projectType": "library",
+	"targets": {
+		"build": {
+			"executor": "@nx/vite:build",
+			"outputs": ["{options.outputPath}"],
+			"options": {
+				"outputPath": "dist/packages/php-wasm/stream-compression"
+			}
+		},
+		"test": {
+			"executor": "@nx/vite:test",
+			"outputs": ["{options.reportsDirectory}"],
+			"options": {
+				"passWithNoTests": true,
+				"reportsDirectory": "../../../coverage/packages/php-wasm/stream-compression"
+			}
+		},
+		"lint": {
+			"executor": "@nx/linter:eslint",
+			"outputs": ["{options.outputFile}"],
+			"options": {
+				"lintFilePatterns": [
+					"packages/php-wasm/stream-compression/**/*.ts",
+					"packages/php-wasm/stream-compression/package.json"
+				]
+			}
+		}
+	},
+	"tags": ["scope:independent-from-php-binaries"]
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+import '@php-wasm/node-polyfills';
+
+export { collectBytes } from './utils/collect-bytes';
+export { collectFile } from './utils/collect-file';
+export { iteratorToStream } from './utils/iterator-to-stream';
+export { StreamedFile } from './utils/streamed-file';
+export { encodeZip, decodeZip, decodeRemoteZip } from './zip';

package/src/test/append-bytes.spec.ts

@@ -0,0 +1,25 @@
+import { appendBytes } from '../utils/append-bytes';
+
+describe('appendBytes', () => {
+	it('Should append the specified number of bytes', async () => {
+		const stream = new ReadableStream<Uint8Array>({
+			type: 'bytes',
+			start(controller) {
+				controller.enqueue(new Uint8Array([1, 2, 3]));
+				controller.close();
+			},
+		}).pipeThrough(appendBytes(new Uint8Array([4, 5])));
+
+		const reader = stream.getReader();
+		const result1 = await reader.read();
+		expect(result1.value).toEqual(new Uint8Array([1, 2, 3]));
+		expect(result1.done).toBe(false);
+
+		const result2 = await reader.read();
+		expect(result2.value).toEqual(new Uint8Array([4, 5]));
+		expect(result2.done).toBe(false);
+
+		const result3 = await reader.read();
+		expect(result3.done).toBe(true);
+	});
+});

package/src/test/decode-zip.spec.ts

@@ -0,0 +1,22 @@
+import { decodeZip } from '../zip/decode-zip';
+import { readFile } from 'fs/promises';
+
+describe('decodeZip', () => {
+	it('Should uncompress compress files', async () => {
+		const zipBytes = await readFile(
+			__dirname + '/fixtures/hello-dolly.zip'
+		);
+		const zipStream = decodeZip(new Blob([zipBytes]).stream());
+
+		const files = [];
+		for await (const file of zipStream) {
+			files.push(file);
+		}
+		expect(files.length).toBe(3);
+		expect(files[0].name).toBe('hello-dolly/');
+		expect(files[1].name).toBe('hello-dolly/hello.php');
+		expect(files[1].size).toBe(2593);
+		expect(files[2].name).toBe('hello-dolly/readme.txt');
+		expect(files[2].size).toBe(624);
+	});
+});

package/src/test/encode-zip.spec.ts

@@ -0,0 +1,47 @@
+import { collectBytes } from '../utils/collect-bytes';
+import { encodeZip } from '../zip/encode-zip';
+import { decodeZip } from '../zip/decode-zip';
+
+describe('compressFiles', () => {
+	it('Should compress files into a zip archive', async () => {
+		const files: File[] = [
+			new File(
+				[new Uint8Array([1, 2, 3, 4, 5])],
+				'wp-content/plugins/hello.php'
+			),
+			new File(
+				[new Uint8Array([1, 2, 3, 4, 5])],
+				'wp-content/plugins/hello/hello.php'
+			),
+			new File(
+				[new Uint8Array([1, 2, 3, 4, 5])],
+				'wp-content/plugins/hello/hello2.php'
+			),
+			new File(
+				[new Uint8Array([1, 2, 3, 4, 5])],
+				'wp-content/plugins/hello/hello3.php'
+			),
+		];
+
+		const zipBytes = await collectBytes(
+			encodeZip(files[Symbol.iterator]())
+		);
+		const zipStream = decodeZip(new Blob([zipBytes!]).stream());
+
+		const reader = zipStream.getReader();
+		let i = 0;
+		for (i = 0; i < files.length; i++) {
+			const { value: receivedFile, done } = await reader.read();
+			const receivedBytes = new Uint8Array(
+				await receivedFile!.arrayBuffer()
+			);
+			const expectedBytes = new Uint8Array(await files[i].arrayBuffer());
+			expect(receivedBytes).toEqual(expectedBytes);
+			expect(done).toBe(false);
+		}
+		expect(i).toBe(files.length);
+
+		const { done } = await reader.read();
+		expect(done).toBe(true);
+	});
+});

package/src/test/prepend-bytes.spec.ts

@@ -0,0 +1,25 @@
+import { prependBytes } from '../utils/prepend-bytes';
+
+describe('prependBytes', () => {
+	it('Should prepend the specified number of bytes', async () => {
+		const stream = new ReadableStream<Uint8Array>({
+			type: 'bytes',
+			start(controller) {
+				controller.enqueue(new Uint8Array([4, 5]));
+				controller.close();
+			},
+		}).pipeThrough(prependBytes(new Uint8Array([1, 2, 3])));
+
+		const reader = stream.getReader();
+		const result1 = await reader.read();
+		expect(result1.value).toEqual(new Uint8Array([1, 2, 3]));
+		expect(result1.done).toBe(false);
+
+		const result2 = await reader.read();
+		expect(result2.value).toEqual(new Uint8Array([4, 5]));
+		expect(result2.done).toBe(false);
+
+		const result3 = await reader.read();
+		expect(result3.done).toBe(true);
+	});
+});

package/src/test/skip-first-bytes.spec.ts

@@ -0,0 +1,41 @@
+import { skipFirstBytes } from '../utils/skip-first-bytes';
+
+describe('skipFirstBytes', () => {
+	it('Should skip the specified number of bytes', async () => {
+		const stream = new ReadableStream<Uint8Array>({
+			type: 'bytes',
+			start(controller) {
+				controller.enqueue(new Uint8Array([1, 2, 3, 4, 5]));
+				controller.close();
+			},
+		}).pipeThrough(skipFirstBytes(3));
+
+		const reader = stream.getReader();
+		const result1 = await reader.read();
+		expect(result1.value).toEqual(new Uint8Array([4, 5]));
+		expect(result1.done).toBe(false);
+
+		const result2 = await reader.read();
+		expect(result2.done).toBe(true);
+	});
+
+	it('Should skip the specified number of bytes across multiple pulls', async () => {
+		const stream = new ReadableStream<Uint8Array>({
+			type: 'bytes',
+			start(controller) {
+				controller.enqueue(new Uint8Array([1]));
+				controller.enqueue(new Uint8Array([2, 3]));
+				controller.enqueue(new Uint8Array([4, 5, 6]));
+				controller.close();
+			},
+		}).pipeThrough(skipFirstBytes(4));
+
+		const reader = stream.getReader();
+		const result1 = await reader.read();
+		expect(result1.value).toEqual(new Uint8Array([5, 6]));
+		expect(result1.done).toBe(false);
+
+		const result2 = await reader.read();
+		expect(result2.done).toBe(true);
+	});
+});

package/src/test/skip-last-bytes.spec.ts

@@ -0,0 +1,27 @@
+import { skipLastBytes } from '../utils/skip-last-bytes';
+
+describe('skipLastBytes', () => {
+	it('Should skip the specified number of bytes', async () => {
+		const stream = new ReadableStream<Uint8Array>({
+			type: 'bytes',
+			start(controller) {
+				controller.enqueue(new Uint8Array([1, 2, 3, 4, 5]));
+				controller.enqueue(new Uint8Array([6, 7]));
+				controller.enqueue(new Uint8Array([8, 9]));
+				controller.close();
+			},
+		}).pipeThrough(skipLastBytes(3, 9));
+
+		const reader = stream.getReader();
+		const result1 = await reader.read();
+		expect(result1.value).toEqual(new Uint8Array([1, 2, 3, 4, 5]));
+		expect(result1.done).toBe(false);
+
+		const result2 = await reader.read();
+		expect(result2.value).toEqual(new Uint8Array([6]));
+		expect(result2.done).toBe(false);
+
+		const result3 = await reader.read();
+		expect(result3.done).toBe(true);
+	});
+});

package/src/test/vitest-setup-file.ts

@@ -0,0 +1,7 @@
+/**
+ * This file is executed before vitests are run, and ensures
+ * the required polyfills are loaded.
+ *
+ * @see tests.setupFiles in vite.config.ts
+ */
+import '@php-wasm/node-polyfills';

package/src/utils/append-bytes.ts

@@ -0,0 +1,16 @@
+/**
+ * Appends bytes to a stream.
+ *
+ * @param bytes The bytes to append.
+ * @returns A transform stream that will append the specified bytes.
+ */
+export function appendBytes(bytes: Uint8Array) {
+	return new TransformStream<Uint8Array, Uint8Array>({
+		async transform(chunk, controller) {
+			controller.enqueue(chunk);
+		},
+		async flush(controller) {
+			controller.enqueue(bytes);
+		},
+	});
+}

package/src/utils/collect-bytes.ts

@@ -0,0 +1,24 @@
+import { concatBytes } from './concat-bytes';
+import { limitBytes } from './limit-bytes';
+
+/**
+ * Collects the contents of the entire stream into a single Uint8Array.
+ *
+ * @param stream The stream to collect.
+ * @param bytes Optional. The number of bytes to read from the stream.
+ * @returns The string contents of the stream.
+ */
+export async function collectBytes(
+	stream: ReadableStream<Uint8Array>,
+	bytes?: number
+) {
+	if (bytes !== undefined) {
+		stream = limitBytes(stream, bytes);
+	}
+
+	return await stream
+		.pipeThrough(concatBytes(bytes))
+		.getReader()
+		.read()
+		.then(({ value }) => value!);
+}

package/src/utils/collect-file.ts

@@ -0,0 +1,16 @@
+import { collectBytes } from './collect-bytes';
+
+/**
+ * Collects the contents of the entire stream into a single File object.
+ *
+ * @param stream The stream to collect.
+ * @param fileName The name of the file
+ * @returns The string contents of the stream.
+ */
+export async function collectFile(
+	fileName: string,
+	stream: ReadableStream<Uint8Array>
+) {
+	// @TODO: use StreamingFile
+	return new File([await collectBytes(stream)], fileName);
+}

package/src/utils/collect-string.ts

@@ -0,0 +1,25 @@
+import { concatString } from './concat-string';
+import { limitBytes } from './limit-bytes';
+
+/**
+ * Collects the contents of the entire stream into a single string.
+ *
+ * @param stream The stream to collect.
+ * @param bytes Optional. The number of bytes to read from the stream.
+ * @returns The string contents of the stream.
+ */
+export async function collectString(
+	stream: ReadableStream<Uint8Array>,
+	bytes?: number
+) {
+	if (bytes !== undefined) {
+		stream = limitBytes(stream, bytes);
+	}
+
+	return await stream
+		.pipeThrough(new TextDecoderStream())
+		.pipeThrough(concatString())
+		.getReader()
+		.read()
+		.then(({ value }) => value);
+}

package/src/utils/concat-bytes.ts

@@ -0,0 +1,38 @@
+import { concatUint8Array } from './concat-uint8-array';
+
+/**
+ * Concatenates the contents of the stream into a single Uint8Array.
+ *
+ * @param totalBytes Optional. The number of bytes to concatenate. Used to
+ *  				 pre-allocate the buffer. If not provided, the buffer will
+ * 				     be dynamically resized as needed.
+ * @returns A stream that will emit a single UInt8Array entry before closing.
+ */
+export function concatBytes(totalBytes?: number) {
+	if (totalBytes === undefined) {
+		let acc = new Uint8Array();
+		return new TransformStream<Uint8Array, Uint8Array>({
+			transform(chunk) {
+				acc = concatUint8Array(acc, chunk);
+			},
+
+			flush(controller) {
+				controller.enqueue(acc);
+			},
+		});
+	} else {
+		const buffer = new ArrayBuffer(totalBytes || 0);
+		let offset = 0;
+		return new TransformStream<Uint8Array, Uint8Array>({
+			transform(chunk) {
+				const view = new Uint8Array(buffer);
+				view.set(chunk, offset);
+				offset += chunk.byteLength;
+			},
+
+			flush(controller) {
+				controller.enqueue(new Uint8Array(buffer));
+			},
+		});
+	}
+}

package/src/utils/concat-string.ts

@@ -0,0 +1,17 @@
+/**
+ * Concatenate all chunks into a single string.
+ *
+ * @returns A stream that will emit a single string entry before closing.
+ */
+export function concatString() {
+	const chunks: string[] = [];
+	return new TransformStream<string, string>({
+		transform(chunk) {
+			chunks.push(chunk);
+		},
+
+		flush(controller) {
+			controller.enqueue(chunks.join(''));
+		},
+	});
+}

package/src/utils/concat-uint8-array.ts

@@ -0,0 +1,17 @@
+/**
+ * Concatenates multiple Uint8Arrays into a single Uint8Array.
+ *
+ * @param arrays The arrays to concatenate.
+ * @returns A new Uint8Array containing the contents of all the arrays.
+ */
+export function concatUint8Array(...arrays: Uint8Array[]) {
+	const result = new Uint8Array(
+		arrays.reduce((sum, array) => sum + array.length, 0)
+	);
+	let offset = 0;
+	for (const array of arrays) {
+		result.set(array, offset);
+		offset += array.length;
+	}
+	return result;
+}

package/src/utils/filter-stream.ts

@@ -0,0 +1,15 @@
+/**
+ * Filter the stream based on a predicate.
+ *
+ * @param predicate The predicate to filter the stream with.
+ * @returns A new stream that will only contain chunks that pass the predicate.
+ */
+export function filterStream<T>(predicate: (chunk: T) => boolean) {
+	return new TransformStream<T, T>({
+		transform(chunk, controller) {
+			if (predicate(chunk)) {
+				controller.enqueue(chunk);
+			}
+		},
+	});
+}

package/src/utils/iterable-stream-polyfill.ts

@@ -0,0 +1,35 @@
+/**
+ * Polyfill for ReadableStream[Symbol.asyncIterator]
+ * This enables the use of for-await-of loops with ReadableStreams
+ *
+ * @example
+ * ```ts
+ * for await (const entry of stream) {
+ * 	   // ...
+ * }
+ * ```
+ */
+// @ts-ignore
+if (!ReadableStream.prototype[Symbol.asyncIterator]) {
+	// @ts-ignore
+	ReadableStream.prototype[Symbol.asyncIterator] = async function* () {
+		const reader = this.getReader();
+		try {
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) {
+					return;
+				}
+				yield value;
+			}
+		} finally {
+			reader.releaseLock();
+		}
+	};
+	// @ts-ignore
+	ReadableStream.prototype.iterate =
+		// @ts-ignore
+		ReadableStream.prototype[Symbol.asyncIterator];
+}
+
+export type IterableReadableStream<R> = ReadableStream<R> & AsyncIterable<R>;

package/src/utils/iterator-to-stream.ts

@@ -0,0 +1,39 @@
+import { IterableReadableStream } from './iterable-stream-polyfill';
+
+/**
+ * Converts an iterator or iterable to a stream.
+ *
+ * @param iteratorOrIterable The iterator or iterable to convert.
+ * @returns A stream that will yield the values from the iterator or iterable.
+ */
+export function iteratorToStream<T>(
+	iteratorOrIterable:
+		| AsyncIterator<T>
+		| Iterator<T>
+		| AsyncIterable<T>
+		| Iterable<T>
+) {
+	if (iteratorOrIterable instanceof ReadableStream) {
+		return iteratorOrIterable as IterableReadableStream<T>;
+	}
+
+	let iterator: AsyncIterator<T> | Iterator<T>;
+	if (Symbol.asyncIterator in iteratorOrIterable) {
+		iterator = iteratorOrIterable[Symbol.asyncIterator]();
+	} else if (Symbol.iterator in iteratorOrIterable) {
+		iterator = iteratorOrIterable[Symbol.iterator]();
+	} else {
+		iterator = iteratorOrIterable;
+	}
+
+	return new ReadableStream<T>({
+		async pull(controller) {
+			const { done, value } = await iterator.next();
+			if (done) {
+				controller.close();
+				return;
+			}
+			controller.enqueue(value);
+		},
+	}) as IterableReadableStream<T>;
+}

package/src/utils/limit-bytes.ts

@@ -0,0 +1,40 @@
+/**
+ * Limit the number of bytes read from a stream.
+ *
+ * @param stream The stream to limit.
+ * @param bytes The number of bytes to read from the stream.
+ * @returns A new stream that will read at most `bytes` bytes from `stream`.
+ */
+export function limitBytes(stream: ReadableStream<Uint8Array>, bytes: number) {
+	if (bytes === 0) {
+		return new ReadableStream({
+			start(controller) {
+				controller.close();
+			},
+		});
+	}
+	const reader = stream.getReader({ mode: 'byob' });
+	let offset = 0;
+	return new ReadableStream({
+		async pull(controller) {
+			const { value, done } = await reader.read(
+				new Uint8Array(bytes - offset)
+			);
+			if (done) {
+				reader.releaseLock();
+				controller.close();
+				return;
+			}
+			offset += value.length;
+			controller.enqueue(value);
+
+			if (offset >= bytes) {
+				reader.releaseLock();
+				controller.close();
+			}
+		},
+		cancel() {
+			reader.cancel();
+		},
+	});
+}

package/src/utils/prepend-bytes.ts

@@ -0,0 +1,18 @@
+/**
+ * Prepend bytes to a stream.
+ *
+ * @param bytes The bytes to prepend.
+ * @returns A transform stream that will prepend the specified bytes.
+ */
+export function prependBytes(bytes: Uint8Array) {
+	let isPrepended = false;
+	return new TransformStream<Uint8Array, Uint8Array>({
+		async transform(chunk, controller) {
+			if (!isPrepended) {
+				isPrepended = true;
+				controller.enqueue(bytes);
+			}
+			controller.enqueue(chunk);
+		},
+	});
+}

package/src/utils/skip-first-bytes.ts

@@ -0,0 +1,21 @@
+/**
+ * Skips the first `length` bytes of a stream.
+ *
+ * @param length The number of bytes to skip.
+ * @returns A transform stream that will skip the specified number of bytes.
+ */
+export function skipFirstBytes(length: number) {
+	let totalBytesSkipped = 0;
+	return new TransformStream<Uint8Array, Uint8Array>({
+		async transform(chunk, controller) {
+			if (totalBytesSkipped + chunk.byteLength < length) {
+				totalBytesSkipped += chunk.byteLength;
+				return;
+			}
+
+			const bytesToSkip = length - totalBytesSkipped;
+			totalBytesSkipped = length;
+			controller.enqueue(chunk.slice(bytesToSkip));
+		},
+	});
+}

package/src/utils/skip-last-bytes.ts

@@ -0,0 +1,24 @@
+/**
+ * Skips the first `length` bytes of a stream.
+ *
+ * @param length The number of bytes to skip.
+ * @returns A transform stream that will skip the specified number of bytes.
+ */
+export function skipLastBytes(skip: number, streamLength: number) {
+	let currentOffset = 0;
+	const lastOffset = streamLength - skip;
+	return new TransformStream({
+		async transform(chunk, controller) {
+			if (currentOffset + chunk.byteLength >= lastOffset) {
+				const lastChunkOffset = lastOffset - currentOffset;
+				if (lastChunkOffset === 0) {
+					return;
+				}
+				chunk = chunk.slice(0, lastChunkOffset);
+			}
+
+			currentOffset += chunk.byteLength;
+			controller.enqueue(chunk);
+		},
+	});
+}