@futpib/parser 1.0.0 → 1.0.2

package/src/elementTerminatedSequenceArrayParser.ts

@@ -0,0 +1,52 @@
+import { Parser, setParserName } from "./parser.js";
+import { DeriveSequenceElement } from "./sequence.js";
+
+export const createElementTerminatedSequenceArrayParser = <Sequence, Element = DeriveSequenceElement<Sequence>>(
+	terminatorElement: Element,
+): Parser<Sequence[], Sequence, Element> => {
+	const elementTerminatedSequenceArrayParser: Parser<Sequence[], Sequence, Element> = async parserContext => {
+		const sequences: Sequence[] = [];
+
+		let start = 0;
+		let window = 1;
+
+		while (true) {
+			const sequence = await parserContext.peekSequence(start, start + window);
+
+			if (sequence === undefined) {
+				window = Math.floor(window / 2);
+
+				if (start === 0 && window === 0) {
+					break;
+				}
+
+				parserContext.invariant(window > 0, 'Unexpected end of input without terminator');
+
+				continue;
+			}
+
+			const terminatorIndex = parserContext.indexOf(sequence, terminatorElement);
+
+			if (terminatorIndex === -1) {
+				start += window;
+				window *= 2;
+
+				continue;
+			}
+
+			const sequence_ = await parserContext.readSequence(0, start + terminatorIndex);
+
+			parserContext.skip(1);
+
+			sequences.push(sequence_);
+
+			start = 0;
+		}
+
+		return sequences;
+	};
+
+	setParserName(elementTerminatedSequenceArrayParser, `(.*?${JSON.stringify(terminatorElement)})*`);
+
+	return elementTerminatedSequenceArrayParser;
+};

package/src/elementTerminatedSequenceParser.test.ts

@@ -0,0 +1,54 @@
+import * as fc from 'fast-check';
+import { testProp } from '@fast-check/ava';
+import { Parser, runParser } from './parser.js';
+import { stringParserInputCompanion } from './parserInputCompanion.js';
+import { createElementTerminatedSequenceParser } from './elementTerminatedSequenceParser.js';
+
+testProp.serial(
+	'elementTerminatedSequenceParser',
+	[
+		fc
+			.string({
+				minLength: 1,
+			})
+			.map(string => ({
+				string,
+				terminator: string.slice(-1)
+			}))
+			.filter(({ string, terminator }) => string.split(terminator).length === 2)
+	],
+	async (t, { string, terminator }) => {
+		const elementTerminatedSequenceParser = createElementTerminatedSequenceParser<string>(
+			terminator,
+		);
+
+		const createTestWrapperParser = (innerParser: typeof elementTerminatedSequenceParser): Parser<{
+			string: string;
+			nextPeek: string | undefined;
+			position: number;
+		}, string> => async parserContext => {
+			const string = await innerParser(parserContext);
+
+			return {
+				string,
+				nextPeek: await parserContext.peek(0),
+				position: parserContext.position,
+			};
+		};
+
+		const actual = await runParser(
+			createTestWrapperParser(elementTerminatedSequenceParser),
+			string,
+			stringParserInputCompanion,
+		);
+
+		t.deepEqual(actual, {
+			string: string.split(terminator)[0],
+			nextPeek: undefined,
+			position: string.length,
+		});
+	},
+	{
+		verbose: true,
+	},
+);

package/src/elementTerminatedSequenceParser.ts

@@ -0,0 +1,43 @@
+import { Parser, setParserName } from "./parser.js";
+import { DeriveSequenceElement } from "./sequence.js";
+
+export const createElementTerminatedSequenceParser = <Sequence, Element = DeriveSequenceElement<Sequence>>(
+	terminatorElement: Element,
+): Parser<Sequence, Sequence, Element> => {
+	const elementTerminatedSequenceParser: Parser<Sequence, Sequence, Element> = async parserContext => {
+		let start = 0;
+		let window = 1;
+
+		while (true) {
+			const sequence = await parserContext.peekSequence(start, start + window);
+
+			if (sequence === undefined) {
+				window = Math.floor(window / 2);
+
+				parserContext.invariant(!(start === 0 && window === 0), 'Unexpected end of input without terminated sequence');
+				parserContext.invariant(window > 0, 'Unexpected end of input without terminator');
+
+				continue;
+			}
+
+			const terminatorIndex = parserContext.indexOf(sequence, terminatorElement);
+
+			if (terminatorIndex === -1) {
+				start += window;
+				window *= 2;
+
+				continue;
+			}
+
+			const sequence_ = await parserContext.readSequence(0, start + terminatorIndex);
+
+			parserContext.skip(1);
+
+			return sequence_;
+		}
+	};
+
+	setParserName(elementTerminatedSequenceParser, `.*?${JSON.stringify(terminatorElement)}`);
+
+	return elementTerminatedSequenceParser;
+};

package/src/endOfInputParser.ts

@@ -1,11 +1,11 @@
-import { Parser } from "./parser.js";
-import { DeriveSequenceElement } from "./sequence.js";
+import { type Parser } from './parser.js';
+import { type DeriveSequenceElement } from './sequence.js';
 
 export const createEndOfInputParser = <Sequence, Element = DeriveSequenceElement<Sequence>>(): Parser<void, Sequence, Element> => async parserContext => {
 	parserContext.invariant(
 		await parserContext.peek(0) === undefined,
-		'Expected end of input.'
+		'Expected end of input.',
 	);
-}
+};
 
 export const endOfInputParser = createEndOfInputParser<any, any>();

package/src/exactElementParser.ts

@@ -1,20 +1,26 @@
-import { type Parser } from './parser.js';
-import { DeriveSequenceElement } from './sequence.js';
+import { setParserName, type Parser } from './parser.js';
+import { type DeriveSequenceElement } from './sequence.js';
 
 export const createExactElementParser = <
 	Sequence,
 	Element = DeriveSequenceElement<Sequence>,
->(element: Element): Parser<Element, Sequence, Element> => async parserContext => {
-	const actualElement = await parserContext.peek(0);
+>(element: Element): Parser<Element, Sequence, Element> => {
+	const exactElementParser: Parser<Element, Sequence, Element> = async parserContext => {
+		const actualElement = await parserContext.peek(0);
 
-	parserContext.invariant(
-		actualElement === element,
-		'Expected %s, got %s',
-		element,
-		actualElement,
-	);
+		parserContext.invariant(
+			actualElement === element,
+			'Expected %s, got %s',
+			element,
+			actualElement,
+		);
 
-	parserContext.skip(1);
+		parserContext.skip(1);
 
-	return element;
+		return element;
+	};
+
+	setParserName(exactElementParser, `createExactElementParser(${element})`);
+
+	return exactElementParser;
 };

package/src/exactSequenceParser.ts

@@ -1,7 +1,7 @@
-import { inspect } from 'util';
 import { setParserName, type Parser } from './parser.js';
+import { inspect } from './inspect.js';
 
-export const createExactSequenceParser = <Sequence>(sequence: Sequence) => {
+export const createExactSequenceNaiveParser = <Sequence>(sequence: Sequence) => {
 	const exactSequenceParser: Parser<Sequence, Sequence, unknown> = async parserContext => {
 		const length = parserContext.length(sequence);
 
@@ -24,4 +24,25 @@ export const createExactSequenceParser = <Sequence>(sequence: Sequence) => {
 	setParserName(exactSequenceParser, inspect(sequence));
 
 	return exactSequenceParser;
-}
+};
+
+export const createExactSequenceParser = <Sequence>(expectedSequence: Sequence) => {
+	const exactSequenceParser: Parser<Sequence, Sequence, unknown> = async parserContext => {
+		const length = parserContext.length(expectedSequence);
+
+		const actualSequence = await parserContext.readSequence(0, length);
+
+		parserContext.invariant(
+			parserContext.equals(actualSequence, expectedSequence),
+			'Expected "%s", got "%s"',
+			() => inspect(expectedSequence),
+			() => inspect(actualSequence),
+		);
+
+		return expectedSequence;
+	};
+
+	setParserName(exactSequenceParser, inspect(expectedSequence));
+
+	return exactSequenceParser;
+};

package/src/fetchCid.ts

@@ -0,0 +1,125 @@
+import fsPromises from 'node:fs/promises';
+import path from 'node:path';
+import pMemoize from 'p-memoize';
+import envPaths from 'env-paths';
+
+const paths = envPaths('parser.futpib.github.io');
+
+function readableWebStreamOnFinish<T>(readableWebStream: ReadableStream<T>, onFinish: () => void): ReadableStream<T> {
+	const reader = readableWebStream.getReader();
+
+	const stream = new ReadableStream<T>({
+		async start(controller) {
+			try {
+				while (true) {
+					const { done, value } = await reader.read();
+					if (done) {
+						break;
+					}
+
+					controller.enqueue(value);
+				}
+			} finally {
+				controller.close();
+				reader.releaseLock();
+				onFinish();
+			}
+		},
+		async cancel(reason) {
+			await reader.cancel(reason);
+			onFinish();
+		},
+	});
+
+	return stream;
+}
+
+class FsCache {
+	private get _basePath() {
+		return path.join(paths.cache, 'fetchCid');
+	}
+
+	private _getKeyPath(key: string) {
+		return path.join(this._basePath, key.replaceAll('/', '_'));
+	}
+
+	async get(key: string): Promise<[ ReadableStream<Uint8Array>, ReadableStream<Uint8Array> ] | undefined> {
+		try {
+			const file = await fsPromises.open(this._getKeyPath(key), 'r');
+
+			const stream = file.readableWebStream({
+				type: 'bytes',
+			}) as ReadableStream<Uint8Array>;
+
+			const streamWithClose = readableWebStreamOnFinish(stream, () => {
+				file.close();
+			});
+
+			return [ streamWithClose, undefined as any ];
+		} catch (error) {
+			if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+				return undefined;
+			}
+
+			throw error;
+		}
+	}
+
+	async has(key: string) {
+		const streams = await this.get(key);
+		try {
+			return streams !== undefined;
+		} finally {
+			for (const stream of streams ?? []) {
+				await stream?.cancel();
+			}
+		}
+	}
+
+	async set(key: string, [ _, value ]: [ ReadableStream<Uint8Array>, ReadableStream<Uint8Array> ]) {
+		await fsPromises.mkdir(this._basePath, {
+			recursive: true,
+		});
+		const file = await fsPromises.open(this._getKeyPath(key), 'w');
+		try {
+			for await (const chunk of value) {
+				await file.write(chunk);
+			}
+		} finally {
+			await file.close();
+		}
+	}
+
+	async delete(key: string) {
+		await fsPromises.unlink(this._getKeyPath(key));
+	}
+}
+
+async function reallyFetchCid(cid: string): Promise<[ ReadableStream<Uint8Array>, ReadableStream<Uint8Array> ]> {
+	const response = await fetch('https://ipfs.io/ipfs/' + cid);
+	return response.body!.tee();
+}
+
+const cachedReallyFetchCid = pMemoize(reallyFetchCid, {
+	cache: new FsCache(),
+});
+
+export async function fetchCid(cidOrPath: string): Promise<AsyncIterable<Uint8Array>> {
+	if (cidOrPath.includes('/')) {
+		const file = await fsPromises.open(cidOrPath, 'r');
+
+		const stream = file.readableWebStream({
+			type: 'bytes',
+		}) as ReadableStream<Uint8Array>;
+
+		const streamWithClose = readableWebStreamOnFinish(stream, () => {
+			file.close();
+		});
+
+		return streamWithClose;
+	}
+
+	const [ readable, unused ] = await cachedReallyFetchCid(cidOrPath);
+	await unused?.cancel();
+	return readable;
+}

package/src/fixedLengthSequenceParser.test.ts

@@ -0,0 +1,77 @@
+import test from 'ava';
+import * as fc from 'fast-check';
+import { testProp } from '@fast-check/ava';
+import { createFixedLengthSequenceParser, createFixedLengthSequenceParserNaive } from './fixedLengthSequenceParser.js';
+import { Parser, runParserWithRemainingInput } from './parser.js';
+import { stringParserInputCompanion } from './parserInputCompanion.js';
+import { HighResolutionTotalTimer } from './highResolutionTimer.js';
+import { arbitrarilySlicedAsyncIterable } from './arbitrarilySlicedAsyncInterable.js';
+
+const naiveTotalTimer = new HighResolutionTotalTimer();
+const totalTimer = new HighResolutionTotalTimer();
+
+testProp.serial(
+	'fixedLengthSequenceParser',
+	[
+		fc
+			.bigInt({
+				min: 1n,
+				max: 2n ** 14n,
+			})
+			.chain(length => fc.tuple(
+				arbitrarilySlicedAsyncIterable(
+					fc.string({
+						minLength: Number(length),
+						maxLength: Number(length) * 2,
+					}),
+				),
+				fc.constant(length),
+			)),
+	],
+	async (t, [ [ _, sequence ], length ]) => {
+		const fixedLengthSequenceParserNaive = createFixedLengthSequenceParserNaive<string>(length);
+		const fixedLengthSequenceParser = createFixedLengthSequenceParser<string>(length);
+
+		const createTestWrapperParser = (innerParser: typeof fixedLengthSequenceParser): Parser<{
+			result: string,
+			position: number,
+		}, string> => async parserContext => {
+			const result = await innerParser(parserContext);
+
+			return {
+				result,
+				nextPeek: await parserContext.peek(0),
+				position: parserContext.position,
+			};
+		};
+
+		const { output: actualNaive } = await naiveTotalTimer.measureAsync(() => runParserWithRemainingInput(
+			createTestWrapperParser(fixedLengthSequenceParserNaive),
+			sequence,
+			stringParserInputCompanion,
+		));
+
+		t.true(actualNaive.result.length === Number(length));
+
+		const { output: actual } = await totalTimer.measureAsync(() => runParserWithRemainingInput(
+			createTestWrapperParser(fixedLengthSequenceParser),
+			sequence,
+			stringParserInputCompanion,
+		));
+
+		t.deepEqual(actual, actualNaive);
+	},
+	{
+		verbose: true,
+	},
+);
+
+test.serial(
+	'fixedLengthSequenceParser performance',
+	t => {
+		t.true(
+			totalTimer.time * 10n < naiveTotalTimer.time,
+			`Naive: ${naiveTotalTimer.time / 1000000n}ms, Optimized: ${totalTimer.time / 1000000n}ms`,
+		);
+	},
+);

package/src/fixedLengthSequenceParser.ts

@@ -1,7 +1,7 @@
 import { setParserName, type Parser } from './parser.js';
 import { parserImplementationInvariant } from './parserImplementationInvariant.js';
 
-export const createFixedLengthSequenceParser = <Sequence>(lengthInput: bigint | number) => {
+export const createFixedLengthSequenceParserNaive = <Sequence>(lengthInput: bigint | number) => {
 	const length = BigInt(lengthInput);
 
 	parserImplementationInvariant(length >= 0n, 'Length must be non-negative got %s.', length);
@@ -22,3 +22,30 @@ export const createFixedLengthSequenceParser = <Sequence>(lengthInput: bigint |
 
 	return fixedLengthSequenceParser;
 };
+
+const MAX_SAFE_INTEGER = BigInt(Number.MAX_SAFE_INTEGER);
+
+export const createFixedLengthSequenceParser = <Sequence>(lengthInput: bigint | number) => {
+	const length = BigInt(lengthInput);
+
+	parserImplementationInvariant(length >= 0n, 'Length must be non-negative got %s.', length);
+
+	const fixedLengthSequenceParser: Parser<Sequence, Sequence, unknown> = async parserContext => {
+		const safeIntegerSequences: Sequence[] = [];
+		let remainingLength = length;
+
+		while (remainingLength > 0n) {
+			const safeIntegerLength = length > MAX_SAFE_INTEGER ? Number.MAX_SAFE_INTEGER : Number(length);
+
+			safeIntegerSequences.push(await parserContext.readSequence(0, safeIntegerLength));
+
+			remainingLength -= BigInt(safeIntegerLength);
+		}
+
+		return parserContext.concat(safeIntegerSequences);
+	};
+
+	setParserName(fixedLengthSequenceParser, `.{${length}}`);
+
+	return fixedLengthSequenceParser;
+};

package/src/hasExecutable.ts

@@ -0,0 +1,11 @@
+import { execa } from 'execa';
+
+export async function hasExecutable(executable: string) {
+	const hasExecutable = execa(executable).then(() => true, () => false);
+
+	if (!hasExecutable) {
+		console.warn(`Executable %o not found`, executable);
+	}
+
+	return hasExecutable;
+}

package/src/highResolutionTimer.ts

@@ -0,0 +1,49 @@
+import invariant from 'invariant';
+import process from 'node:process';
+
+class HighResolutionSubTimer {
+	private _ended = false;
+
+	constructor(
+		private readonly _parent: HighResolutionTotalTimer,
+		private readonly _start: bigint,
+	) {}
+
+	end() {
+		invariant(!this._ended, 'Timer already ended.');
+		const end = process.hrtime.bigint();
+		const time = end - this._start;
+		this._parent.add(time);
+		this._ended = true;
+	}
+}
+
+export class HighResolutionTotalTimer {
+	private _total = 0n;
+
+	start(): HighResolutionSubTimer {
+		return new HighResolutionSubTimer(this, process.hrtime.bigint());
+	}
+
+	add(time: bigint) {
+		this._total += time;
+	}
+
+	get time() {
+		return this._total;
+	}
+
+	measure<T>(fn: () => T): T {
+		const timer = this.start();
+		const result = fn();
+		timer.end();
+		return result;
+	}
+
+	async measureAsync<T>(fn: () => Promise<T>): Promise<T> {
+		const timer = this.start();
+		const result = await fn();
+		timer.end();
+		return result;
+	}
+}

package/src/index.ts

@@ -1,13 +1,13 @@
 
 export {
-	Parser,
+	type Parser,
 	runParser,
 
 	setParserName,
 	getParserName,
 } from './parser.js';
 
-export {
+export type {
 	ParserContext,
 } from './parserContext.js';
 
@@ -74,3 +74,16 @@ export {
 export {
 	createDebugLogParser,
 } from './debugLogParser.js';
+
+export {
+	type Unparser,
+	runUnparser,
+} from './unparser.js';
+
+export {
+	createArrayUnparser,
+} from './arrayUnparser.js';
+
+export {
+	createSequenceUnparser,
+} from './sequenceUnparser.js';