@futpib/parser 1.0.1 → 1.0.2

package/src/inputReader.test.ts

@@ -2,6 +2,8 @@ import test from 'ava';
 import { InputReaderImplementation } from './inputReader.js';
 import { stringParserInputCompanion } from './parserInputCompanion.js';
 import { ParserImplementationError } from './parserError.js';
+import { toAsyncIterable } from './toAsyncIterable.js';
+import { InputReaderState } from './inputReaderState.js';
 
 test('inputReader', async t => {
 	const inputReader = new InputReaderImplementation(stringParserInputCompanion, (async function * () {
@@ -46,6 +48,53 @@ test('inputReader', async t => {
 	t.is(await inputReader.peek(0), undefined);
 });
 
+test('inputReader peekSequence', async t => {
+	const inputReader = new InputReaderImplementation(stringParserInputCompanion, (async function * () {
+		yield '';
+		yield 'abc';
+		yield 'def';
+		yield '';
+		yield 'gh';
+	})());
+
+	t.is(await inputReader.peekSequence(0, 0), '');
+	t.is(await inputReader.peekSequence(0, 1), 'a');
+
+	inputReader.skip(0);
+
+	t.is(await inputReader.peekSequence(1, 2), 'b');
+
+	inputReader.skip(1);
+
+	t.is(await inputReader.peekSequence(1, 7), 'bcdefgh'.slice(1, 7));
+	t.is(await inputReader.peekSequence(1, 8), undefined);
+
+	inputReader.skip(6);
+
+	t.is(await inputReader.peekSequence(0, 0), '');
+	t.is(await inputReader.peekSequence(0, 1), 'h');
+	t.is(await inputReader.peekSequence(1, 2), undefined);
+	t.is(await inputReader.peekSequence(2, 2), undefined);
+
+	inputReader.skip(1);
+
+	t.is(await inputReader.peekSequence(0, 0), '');
+	t.is(await inputReader.peekSequence(0, 1), undefined);
+	t.is(await inputReader.peekSequence(1, 1), undefined);
+
+	inputReader.skip(0);
+
+	t.is(await inputReader.peekSequence(0, 0), '');
+	t.is(await inputReader.peekSequence(0, 1), undefined);
+	t.is(await inputReader.peekSequence(1, 1), undefined);
+
+	inputReader.skip(1);
+
+	t.is(await inputReader.peekSequence(0, 0), undefined);
+	t.is(await inputReader.peekSequence(0, 1), undefined);
+	t.is(await inputReader.peekSequence(1, 1), undefined);
+});
+
 test('inputReader.peek concurrent', async t => {
 	const inputReader = new InputReaderImplementation(stringParserInputCompanion, (async function * () {
 		yield * 'abcdefgh';
@@ -68,6 +117,64 @@ test('inputReader.peek concurrent', async t => {
 	t.deepEqual(peeks, [ 'a', 'a', 'a', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h' ]);
 });
 
+test('inputReader.peekSequence concurrent', async t => {
+	const inputReader = new InputReaderImplementation(stringParserInputCompanion, (async function * () {
+		yield * 'abcdefgh';
+	})());
+
+	const peeks = await Promise.all([1, 2, 3].flatMap(length => [
+		inputReader.peekSequence(0, 0 + length),
+		inputReader.peekSequence(0, 0 + length),
+		inputReader.peekSequence(0, 0 + length),
+		inputReader.peekSequence(0, 0 + length),
+		inputReader.peekSequence(1, 1 + length),
+		inputReader.peekSequence(2, 2 + length),
+		inputReader.peekSequence(3, 3 + length),
+		inputReader.peekSequence(4, 4 + length),
+		inputReader.peekSequence(5, 5 + length),
+		inputReader.peekSequence(6, 6 + length),
+		inputReader.peekSequence(7, 7 + length),
+	]));
+
+	t.deepEqual(peeks, [
+		'a',
+		'a',
+		'a',
+		'a',
+		'b',
+		'c',
+		'd',
+		'e',
+		'f',
+		'g',
+		'h',
+
+		'ab',
+		'ab',
+		'ab',
+		'ab',
+		'bc',
+		'cd',
+		'de',
+		'ef',
+		'fg',
+		'gh',
+		undefined,
+
+		'abc',
+		'abc',
+		'abc',
+		'abc',
+		'bcd',
+		'cde',
+		'def',
+		'efg',
+		'fgh',
+		undefined,
+		undefined,
+	]);
+});
+
 test('inputReader skip while peeking', async t => {
 	const inputReader = new InputReaderImplementation(stringParserInputCompanion, (async function * () {
 		yield * 'abcdefgh';
@@ -77,8 +184,33 @@ test('inputReader skip while peeking', async t => {
 
 	inputReader.skip(1);
 
+	const peekAfterSkipPromise = inputReader.peek(0);
+	const peekSequenceAfterSkipPromise = inputReader.peekSequence(0, 5);
+
 	t.is(await peekPromise, 'a');
 	t.is(await inputReader.peek(0), 'b');
+
+	t.is(await peekAfterSkipPromise, 'b');
+	t.is(await peekSequenceAfterSkipPromise, 'bcdef');
+});
+
+test('inputReader skip while peeking sequence', async t => {
+	const inputReader = new InputReaderImplementation(stringParserInputCompanion, (async function * () {
+		yield * 'abcdefgh';
+	})());
+
+	const peekPromise = inputReader.peekSequence(1, 5);
+
+	inputReader.skip(1);
+
+	const peekAfterSkipPromise = inputReader.peek(0);
+	const peekSequenceAfterSkipPromise = inputReader.peekSequence(0, 4);
+
+	t.is(await peekPromise, 'bcde', 'peekPromise');
+	t.is(await inputReader.peekSequence(0, 4), 'bcde', 'inputReader.peekSequence(0, 4)');
+
+	t.is(await peekAfterSkipPromise, 'b', 'peekAfterSkipPromise');
+	t.is(await peekSequenceAfterSkipPromise, 'bcde', 'peekSequenceAfterSkipPromise');
 });
 
 test('inputReader.lookahead', async t => {
@@ -236,3 +368,80 @@ test('inputReader.lookahead skip position', async t => {
 	t.is(await lookahead.peek(0), 'c');
 	t.is(await lookahead1.peek(0), 'd');
 });
+
+const END_OF_CONSUMED_SEQUENCES = Symbol('END_OF_CONSUMED_SEQUENCES');
+const END_OF_BUFFERED_SEQUENCES = Symbol('END_OF_BUFFERED_SEQUENCES');
+
+async function inputReaderStateToArray<Sequence>({
+	unconsumedBufferedSequences,
+	consumedBufferedSequences,
+	unbufferedSequences,
+}: InputReaderState<Sequence>): Promise<(Sequence | typeof END_OF_CONSUMED_SEQUENCES | typeof END_OF_BUFFERED_SEQUENCES)[]> {
+	const unconsumedBufferedSequencesArray = unconsumedBufferedSequences.slice();
+	const consumedBufferedSequencesArray = consumedBufferedSequences.slice();
+
+	const unbufferedSequencesArray = [];
+	if (unbufferedSequences) {
+		for await (const sequence of toAsyncIterable(unbufferedSequences)) {
+			unbufferedSequencesArray.push(sequence);
+		}
+	}
+
+	return [
+		...consumedBufferedSequencesArray,
+		END_OF_CONSUMED_SEQUENCES,
+		...unconsumedBufferedSequencesArray,
+		END_OF_BUFFERED_SEQUENCES,
+		...unbufferedSequencesArray,
+	];
+}
+
+for (const [
+	input,
+	position,
+	expected,
+] of [
+	[
+		[ '', 'abc', 'def', '', 'gh' ],
+		0,
+		[ END_OF_CONSUMED_SEQUENCES, 'abc', END_OF_BUFFERED_SEQUENCES, 'def', '', 'gh' ],
+	],
+	[
+		[ '', 'abc', 'def', '', 'gh' ],
+		1,
+		[ 'a', END_OF_CONSUMED_SEQUENCES, 'bc', END_OF_BUFFERED_SEQUENCES, 'def', '', 'gh' ],
+	],
+	[
+		[ '', 'abc', 'def', '', 'gh' ],
+		2,
+		[ 'ab', END_OF_CONSUMED_SEQUENCES, 'c', END_OF_BUFFERED_SEQUENCES, 'def', '', 'gh' ],
+	],
+	[
+		[ '', 'abc', 'def', '', 'gh' ],
+		3,
+		[ END_OF_CONSUMED_SEQUENCES, 'def', END_OF_BUFFERED_SEQUENCES, '', 'gh' ],
+	],
+	[
+		[ '', 'abc', 'def', '', 'gh' ],
+		4,
+		[ 'd', END_OF_CONSUMED_SEQUENCES, 'ef', END_OF_BUFFERED_SEQUENCES, '', 'gh' ],
+	],
+	[
+		[ '', 'abc', 'def', '', 'gh' ],
+		8,
+		[ END_OF_CONSUMED_SEQUENCES, END_OF_BUFFERED_SEQUENCES ],
+	],
+] as const) {
+	test('inputReader.toInputReaderState ' + JSON.stringify({ input, position }), async t => {
+		const inputReader = new InputReaderImplementation(stringParserInputCompanion, (async function * () {
+			yield * input;
+		})());
+
+		inputReader.skip(position);
+		await inputReader.peek(0);
+
+		const actual = await inputReaderStateToArray(inputReader.toInputReaderState());
+
+		t.deepEqual(actual, expected);
+	});
+}

package/src/inputReader.ts

@@ -3,14 +3,18 @@ import invariant from 'invariant';
 import { type SequenceBuffer, SequenceBufferImplementation } from './sequenceBuffer.js';
 import { type ParserInputCompanion } from './parserInputCompanion.js';
 import { parserImplementationInvariant } from './parserImplementationInvariant.js';
+import { InputReaderState } from './inputReaderState.js';
 
 export type InputReader<Sequence, Element> = {
 	get position(): number;
 
 	peek(offset: number): Promise<Element | undefined>;
+	peekSequence(start: number, end: number): Promise<Sequence | undefined>;
 	skip(offset: number): void;
 
 	lookahead(): InputReader<Sequence, Element>;
+
+	toInputReaderState(): InputReaderState<Sequence>;
 };
 
 let inputReaderId = 0;
@@ -20,6 +24,7 @@ export class InputReaderImplementation<Sequence, Element> implements InputReader
 
 	private _position = 0;
 	private _uncommitedSkipOffset = 0;
+	private _inputAsyncIteratorDone = false;
 
 	private readonly _promiseMutex = new PromiseMutex();
 
@@ -66,6 +71,8 @@ export class InputReaderImplementation<Sequence, Element> implements InputReader
 				const inputIteratorResult = await this._inputAsyncIterator.next();
 
 				if (inputIteratorResult.done) {
+					this._inputAsyncIteratorDone = true;
+
 					return undefined;
 				}
 
@@ -85,6 +92,47 @@ export class InputReaderImplementation<Sequence, Element> implements InputReader
 		});
 	}
 
+	async peekSequence(start: number, end: number): Promise<Sequence | undefined> {
+		parserImplementationInvariant(
+			start >= 0,
+			'start (%s) >= 0',
+			start,
+		);
+		parserImplementationInvariant(
+			end >= start,
+			'end (%s) >= start (%s)',
+			start,
+			end,
+		);
+		parserImplementationInvariant(
+			Number.isSafeInteger(start),
+			'start (%s) is not a safe integer',
+			start,
+		);
+		parserImplementationInvariant(
+			Number.isSafeInteger(end),
+			'end (%s) is not a safe integer',
+			end,
+		);
+
+		start += this._uncommitedSkipOffset;
+		end += this._uncommitedSkipOffset;
+
+		const sequence = this._sequenceBuffer.peekSequence(start, end);
+
+		if (sequence !== undefined) {
+			return sequence;
+		}
+
+		const lastElement = await this.peek(Math.max(0, end - 1));
+
+		if (lastElement === undefined) {
+			return undefined;
+		}
+
+		return this._sequenceBuffer.peekSequence(start, end);
+	}
+
 	skip(offset: number) {
 		parserImplementationInvariant(offset >= 0, 'offset >= 0');
 
@@ -101,6 +149,20 @@ export class InputReaderImplementation<Sequence, Element> implements InputReader
 	lookahead(): InputReader<Sequence, Element> {
 		return new InputReaderLookaheadImplementation(this);
 	}
+
+	toInputReaderState() {
+		const {
+			consumedBufferedSequences,
+			unconsumedBufferedSequences,
+		} = this._sequenceBuffer.toSequenceBufferState();
+
+		return {
+			consumedBufferedSequences,
+			unconsumedBufferedSequences,
+			unbufferedSequences: this._inputAsyncIteratorDone ? undefined : this._inputAsyncIterator,
+			position: this._position,
+		};
+	}
 }
 
 class InputReaderLookaheadImplementation<Sequence, Element> implements InputReader<Sequence, Element> {
@@ -134,6 +196,15 @@ class InputReaderLookaheadImplementation<Sequence, Element> implements InputRead
 		return this._inputReader.peek(this._offset + offset);
 	}
 
+	async peekSequence(start: number, end: number): Promise<Sequence | undefined> {
+		const inputReaderMovedForward = this._inputReader.position - this._initialInputReaderPosition;
+
+		this._initialInputReaderPosition = this._inputReader.position;
+		this._offset -= inputReaderMovedForward;
+
+		return this._inputReader.peekSequence(this._offset + start, this._offset + end);
+	}
+
 	skip(offset: number) {
 		parserImplementationInvariant(offset >= 0, 'offset >= 0');
 
@@ -156,4 +227,8 @@ class InputReaderLookaheadImplementation<Sequence, Element> implements InputRead
 
 		return lookahead;
 	}
+
+	toInputReaderState() {
+		return invariant(false, 'Not implemented');
+	}
 }

package/src/inputReaderState.ts

@@ -0,0 +1,33 @@
+export type InputReaderState<Sequence> = {
+	position: number;
+	consumedBufferedSequences: Sequence[];
+	unconsumedBufferedSequences: Sequence[];
+	unbufferedSequences: undefined | AsyncIterator<Sequence>;
+};
+
+export const inputReaderStateCompanion = {
+	isDone<Sequence>({
+		unconsumedBufferedSequences,
+		unbufferedSequences,
+	}: InputReaderState<Sequence>): boolean {
+		return (
+			unconsumedBufferedSequences.length === 0
+				&& unbufferedSequences === undefined
+		);
+	},
+
+	async * toRemainingInputAsyncIterator<Sequence>({
+		unconsumedBufferedSequences,
+		unbufferedSequences,
+	}: InputReaderState<Sequence>): AsyncIterable<Sequence> {
+		yield * unconsumedBufferedSequences;
+
+		if (unbufferedSequences !== undefined) {
+			yield * {
+				[Symbol.asyncIterator]() {
+					return unbufferedSequences;
+				},
+			};
+		}
+	},
+};

package/src/inspect.ts

@@ -0,0 +1,9 @@
+import { inspect as nodeInspect } from 'node:util';
+
+export const inspect = (value: unknown) => {
+	if (value instanceof Uint8Array) {
+		value = Buffer.from(value);
+	}
+
+	return nodeInspect(value);
+}

package/src/javaKeyStoreParser.test.ts

@@ -2,6 +2,7 @@ import test from 'ava';
 import { uint8ArrayParserInputCompanion } from './parserInputCompanion.js';
 import { runParser } from './parser.js';
 import { javaKeyStoreParser } from './javaKeyStoreParser.js';
+import { fetchCid } from './fetchCid.js';
 
 for (const javaKeyStoreCid of [
 	'bafkreig6k53b6p7bdvfjxc5mcb4qv3mffqls5ymqerxkqd6ih2xy5cs3n4',
@@ -9,9 +10,7 @@ for (const javaKeyStoreCid of [
 	test(
 		'javaKeyStore ' + javaKeyStoreCid,
 		async t => {
-			const javaKeyStoreResponse = await fetch('https://ipfs.io/ipfs/' + javaKeyStoreCid);
-
-			const javaKeyStoreStream = javaKeyStoreResponse.body!;
+			const javaKeyStoreStream = await fetchCid(javaKeyStoreCid);
 
 			const actual = await runParser(javaKeyStoreParser, javaKeyStoreStream, uint8ArrayParserInputCompanion, {
 				errorJoinMode: 'all',

package/src/jsonParser.ts

@@ -13,6 +13,7 @@ import { createArrayParser } from './arrayParser.js';
 import { createParserAccessorParser } from './parserAccessorParser.js';
 import { createElementParser } from './elementParser.js';
 import { parserCreatorCompose } from './parserCreatorCompose.js';
+import { createSeparatedArrayParser } from './separatedArrayParser.js';
 
 const whitespaceParser: Parser<unknown, string> = createArrayParser(
 	createUnionParser([
@@ -65,7 +66,7 @@ const jsonStringCharacterParser: Parser<string, string> = createDisjunctionParse
 	)(),
 ]);
 
-const jsonStringParser: Parser<string, string> = promiseCompose(
+export const jsonStringParser: Parser<string, string> = promiseCompose(
 	createTupleParser([
 		createExactSequenceParser('"'),
 		promiseCompose(
@@ -79,7 +80,7 @@ const jsonStringParser: Parser<string, string> = promiseCompose(
 	([ , string ]) => string,
 );
 
-const jsonNumberParser: Parser<number, string> = parserCreatorCompose(
+export const jsonNumberParser: Parser<number, string> = parserCreatorCompose(
 	() => createArrayParser(
 		parserCreatorCompose(
 			() => elementParser,
@@ -189,30 +190,17 @@ const jsonArrayParser: Parser<JsonArray, string> = promiseCompose(
 	createTupleParser([
 		createExactSequenceParser('['),
 		whitespaceParser,
-		promiseCompose(
-			createTerminatedArrayParser(
-				createDisjunctionParser<JsonValue, string>([
-					promiseCompose(
-						createTupleParser([
-							createParserAccessorParser(() => jsonValueParser),
-							whitespaceParser,
-							createExactSequenceParser(','),
-							whitespaceParser,
-						]),
-						([ value ]) => value,
-					),
-					promiseCompose(
-						createTupleParser([
-							createParserAccessorParser(() => jsonValueParser),
-							whitespaceParser,
-						]),
-						([ value ]) => value,
-					),
-				]),
-				createExactSequenceParser(']'),
-			),
-			([ values ]) => values,
+		createSeparatedArrayParser(
+			createParserAccessorParser(() => jsonValueParser),
+			createTupleParser([
+				whitespaceParser,
+				createExactSequenceParser(','),
+				whitespaceParser,
+			]),
 		),
+		whitespaceParser,
+		createExactSequenceParser(']'),
+		whitespaceParser,
 	]),
 	([ _bracket, _whitespace, values ]) => values,
 );

package/src/leb128Parser.test.ts

@@ -0,0 +1,171 @@
+import { testProp, fc } from '@fast-check/ava';
+import test from 'ava';
+// @ts-expect-error
+import leb128 from 'leb128';
+import { uint8ArrayParserInputCompanion } from './parserInputCompanion.js';
+import { runParser } from './parser.js';
+import { uleb128Parser, sleb128Parser, uleb128NumberParser, sleb128NumberParser } from './leb128Parser.js';
+
+test('sleb128, uleb128, uleb128p1', async t => {
+	for (const [
+		input, expectedSleb128, expectedUleb128,
+	] of [
+		[ Buffer.from('00', 'hex'), 0n, 0n ],
+		[ Buffer.from('01', 'hex'), 1n, 1n ],
+		[ Buffer.from('7f', 'hex'), -1n, 127n ],
+		[ Buffer.from('807f', 'hex'), -128n, 16256n ],
+	] as const) {
+		const actualSleb128 = await runParser(sleb128Parser, input, uint8ArrayParserInputCompanion);
+		const actualUleb128 = await runParser(uleb128Parser, input, uint8ArrayParserInputCompanion);
+
+		t.is(actualSleb128, expectedSleb128, 'sleb128');
+		t.is(actualUleb128, expectedUleb128, 'uleb128');
+	}
+
+	for (const [
+		input, expectedUleb128,
+	] of [
+		[ new Uint8Array([ 2 ]), 2n ],
+		[ new Uint8Array([ 127 ]), 127n ],
+		[ new Uint8Array([ 0 + 0x80, 1 ]), 128n ],
+		[ new Uint8Array([ 1 + 0x80, 1 ]), 129n ],
+		[ new Uint8Array([ 57 + 0x80, 100 ]), 12857n ],
+	] as const) {
+		const actualUleb128 = await runParser(uleb128Parser, input, uint8ArrayParserInputCompanion);
+
+		t.is(actualUleb128, expectedUleb128, 'uleb128');
+	}
+
+	for (const [
+		input, expectedSleb128,
+	] of [
+		[ new Uint8Array([ 2 ]), 2n ],
+		[ new Uint8Array([ 0x7e ]), -2n ],
+		[ new Uint8Array([ 127 + 0x80, 0 ]), 127n ],
+		[ new Uint8Array([ 1 + 0x80, 0x7f ]), -127n ],
+		[ new Uint8Array([ 0 + 0x80, 1 ]), 128n ],
+		[ new Uint8Array([ 0 + 0x80, 0x7f ]), -128n ],
+		[ new Uint8Array([ 1 + 0x80, 1 ]), 129n ],
+		[ new Uint8Array([ 0x7f + 0x80, 0x7e ]), -129n ],
+	] as const) {
+		const actualSleb128 = await runParser(sleb128Parser, input, uint8ArrayParserInputCompanion);
+
+		t.is(actualSleb128, expectedSleb128, 'sleb128');
+	}
+});
+
+testProp(
+	'uleb128NumberParser on small number',
+	[
+		fc.nat(),
+	],
+	async (t, natural) => {
+		const uleb128 = leb128.unsigned.encode(natural);
+
+		const actualUleb128 = await runParser(uleb128NumberParser, uleb128, uint8ArrayParserInputCompanion);
+
+		t.is(actualUleb128, natural, 'uleb128');
+	},
+	{
+		verbose: true,
+	},
+);
+
+testProp(
+	'sleb128NumberParser on small number',
+	[
+		fc.integer(),
+	],
+	async (t, integer) => {
+		const sleb128 = leb128.signed.encode(integer);
+
+		const actualSleb128 = await runParser(sleb128NumberParser, sleb128, uint8ArrayParserInputCompanion);
+
+		t.is(actualSleb128, integer, 'sleb128');
+	},
+	{
+		verbose: true,
+	},
+);
+
+// TODO?
+testProp.skip(
+	'uleb128NumberParser on large number',
+	[
+		fc.maxSafeNat(),
+	],
+	async (t, natural) => {
+		const uleb128 = leb128.unsigned.encode(natural);
+
+		if (natural > (2 ** 32) - 1) {
+			await t.throwsAsync(() => runParser(uleb128NumberParser, uleb128, uint8ArrayParserInputCompanion));
+
+			return;
+		}
+
+		const actualUleb128 = await runParser(uleb128NumberParser, uleb128, uint8ArrayParserInputCompanion);
+
+		t.is(actualUleb128, natural, 'uleb128');
+	},
+	{
+		verbose: true,
+	},
+);
+
+// TODO?
+testProp.skip(
+	'sleb128NumberParser on large number',
+	[
+		fc.maxSafeInteger(),
+	],
+	async (t, integer) => {
+		const sleb128 = leb128.signed.encode(integer);
+
+		if (integer > (2 ** 32) - 1 || integer < -(2 ** 32)) {
+			await t.throwsAsync(() => runParser(sleb128NumberParser, sleb128, uint8ArrayParserInputCompanion));
+
+			return;
+		}
+
+		const actualSleb128 = await runParser(sleb128NumberParser, sleb128, uint8ArrayParserInputCompanion);
+
+		t.is(actualSleb128, integer, 'sleb128');
+	},
+	{
+		verbose: true,
+	},
+);
+
+testProp(
+	'uleb128Parser on bigint',
+	[
+		fc.bigInt({ min: 0n }),
+	],
+	async (t, bigInt) => {
+		const uleb128 = leb128.unsigned.encode(bigInt);
+
+		const actualUleb128 = await runParser(uleb128Parser, uleb128, uint8ArrayParserInputCompanion);
+
+		t.is(actualUleb128, bigInt, 'uleb128');
+	},
+	{
+		verbose: true,
+	},
+);
+
+testProp(
+	'sleb128Parser on bigint',
+	[
+		fc.bigInt(),
+	],
+	async (t, bigInt) => {
+		const sleb128 = leb128.signed.encode(bigInt);
+
+		const actualSleb128 = await runParser(sleb128Parser, sleb128, uint8ArrayParserInputCompanion);
+
+		t.is(actualSleb128, bigInt, 'sleb128');
+	},
+	{
+		verbose: true,
+	},
+);