grammar-composer 0.3.0 → 0.4.0

package/src/parser-generator/TopDownParser.ts

@@ -1,3 +1,4 @@
+import { isNumber } from '../utilities/Utilities.js'
 import { Grammar, GrammarElement, Terminal } from './Grammar.js'
 
 export function parse(inputString: string, grammar: Grammar<any>) {
@@ -18,37 +19,30 @@ export function parse(inputString: string, grammar: Grammar<any>) {
 	}
 
 	function tryParse(grammarElement: GrammarElement, startOffset: number): ParseResult | null {
-		if (grammarElement.cached === true) {
+		if (isNumber(grammarElement.cacheId)) {
 			return tryParseCached(grammarElement, startOffset)
 		} else {
 			return tryParseUncached(grammarElement, startOffset)
 		}
 	}
 
-	type Slot = Map<GrammarElement, ParseResult | null> | undefined
+	const offsetMultiplier = grammar.maxCacheId + 1
 
-	const cachedParseResults: Slot[] = new Array(inputLength)
+	const parseResultsCache = new Map<number, ParseResult | null>()
 
 	function tryParseCached(grammarElement: GrammarElement, startOffset: number): ParseResult | null {
-		let slot = cachedParseResults[startOffset]
+		const cacheId = grammarElement.cacheId!
+		const cacheKey = (startOffset * offsetMultiplier) + cacheId
 
-		if (slot === undefined) {
-			slot = new Map<GrammarElement, ParseResult | null>()
-
-			cachedParseResults[startOffset] = slot
+		if (parseResultsCache.has(cacheKey)) {
+			return parseResultsCache.get(cacheKey)!
 		} else {
-			const cachedResult = slot.get(grammarElement)
-
-			if (cachedResult !== undefined) {
-				return cachedResult
-			}
-		}
-
-		const parseResult = tryParseUncached(grammarElement, startOffset)
+			const parseResult = tryParseUncached(grammarElement, startOffset)
 
-		slot.set(grammarElement, parseResult)
+			parseResultsCache.set(cacheKey, parseResult)
 
-		return parseResult
+			return parseResult
+		}
 	}
 
 	function tryParseUncached(grammarElement: GrammarElement, startOffset: number): ParseResult | null {

package/src/tests/Test.ts

@@ -4,6 +4,8 @@ import { anyOf, buildGrammar } from '../exports/Exports.js'
 import { JsonGrammar } from './test-grammars/JsonGrammar.js'
 import { XmlGrammar } from './test-grammars/XmlGrammar.js'
 import { RegExpGrammar } from './test-grammars/RegExpGrammar.js'
+import { writeFile } from 'fs/promises'
+import { SimpleTestGrammar1 } from './test-grammars/SimpleTestGrammar1.js'
 
 const log = console.log
 
@@ -83,9 +85,11 @@ function testXmlParser() {
 
 async function testRegExpParser() {
 	const regExpString = /^([+]?[1]?(1 )?[-.+]?\(?\d{1}[- .+]*\d{1}[- .+]*\d{1}\)?[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1})$/.source
-	//const regExpString = /^asdf{$/.source
+	//const regExpString = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/.source
+	//const regExpString = /(?=.*[!@#$%^&*])^mongodb:\/\/(?<user>[a-zA-Z0-9]+):(?<pass>[a-zA-Z0-9!@#$%^&*]{8,})@(?<host>[a-z0-9.-]+):(?<port>\d{2,5})$/.source
+	//const regExpString = /(abcd)*ef+g/.source
 
-	const grammar = buildGrammar(RegExpGrammar, 'disjunction')
+	const grammar = buildGrammar(RegExpGrammar, 'root')
 
 	const parseTree = grammar.parse(regExpString)
 
@@ -93,13 +97,10 @@ async function testRegExpParser() {
 
 	log(parseTreeJson)
 
-	const { writeFile } = await import('fs/promises')
-
 	await writeFile('out/out.json', parseTreeJson)
 }
 
-
-async function testParserError1() {
+function testParserError1() {
 	const xmlData = `<hello> wo rld <!!! `
 
 	const grammar = buildGrammar(XmlGrammar, 'document')
@@ -109,7 +110,7 @@ async function testParserError1() {
 	console.log(JSON.stringify(result, undefined, 4))
 }
 
-async function testParserError2() {
+function testParserError2() {
 	const jsonData = `{ "asdf": 12.5 `
 
 	const grammar = buildGrammar(JsonGrammar, 'expression')
@@ -119,6 +120,19 @@ async function testParserError2() {
 	console.log(JSON.stringify(result, undefined, 4))
 }
 
+function test1() {
+	const input = `abcdefg`
+
+	const grammar = buildGrammar(SimpleTestGrammar1, 'root')
+
+	const result = grammar.parse(input)
+
+	console.log(JSON.stringify(result, undefined, 4))
+}
+
+
 //testJsonParser()
 
 testRegExpParser()
+
+//test1()

package/src/tests/test-grammars/RegExpGrammar.ts

@@ -3,101 +3,132 @@ import * as R from 'regexp-composer'
 
 export class RegExpGrammar {
 	//////////////////////////////////////////////////////////////////////////////////////////////
-	// High level productions
+	// High-level productions
 	//////////////////////////////////////////////////////////////////////////////////////////////
+	root = () => this.sequenceOrDisjunction
+
 	disjunction = () => [
 		this.sequence,
 
-		G.possibly([
-			this.disjunctionSeparator,
-			this.disjunction,
+		G.oneOrMore([
+			'|',
+			this.sequence,
 		])
 	]
 
-	sequence = () => G.oneOrMore(this.element)
-
-	element = () => G.anyOf(
-		this.anchor,
-		this.lookaround,
-		this.possiblyQuantifiedExpression,
+	sequence = () => G.cached(
+		G.zeroOrMore(this.sequenceElement)
 	)
 
+	sequenceOrDisjunction = [
+		G.anyOf(
+			this.disjunction,
+			this.sequence,
+		)
+	]
+
 	//////////////////////////////////////////////////////////////////////////////////////////////
-	// Quantifed expressions
+	// Quantifier expressions
 	//////////////////////////////////////////////////////////////////////////////////////////////
-	possiblyQuantifiedExpression = () => [
-		G.anyOf(
-			this.group,
-			this.backReference,
-			this.singleCharExpression
-		),
+	starQuantifier = () => [
+		'*',
+		G.possibly(this.nongreedyQuantifier)
+	]
 
-		G.possibly(G.anyOf(
-			this.starQuantifier,
-			this.plusQuantifier,
-			this.exactCountQuantifier,
-			this.countRangeQuantifier,
-		)),
+	plusQuantifier = () => [
+		'+',
+		G.possibly(this.nongreedyQuantifier)
+	]
 
-		G.possibly(this.nongreedyQuantifier), // Nongreedy quantifier
+	optionalQuantifier = () => [
+		'?',
+		G.possibly(this.nongreedyQuantifier)
 	]
 
-	exactCountQuantifier = () => G.pattern([
-		'{',
-		R.captureAs('count', R.oneOrMore(digit)),
-		'}',
-	])
+	exactCountQuantifier = () => [
+		G.pattern([
+			'{',
+			R.captureAs('count',
+				R.oneOrMore(digit)
+			),
+			'}',
+		]),
+		G.possibly(this.nongreedyQuantifier)
+	]
 
-	countRangeQuantifier = () => G.pattern([
-		'{',
-		R.captureAs('start', R.oneOrMore(digit)),
-		',',
-		R.possibly(R.captureAs('end', R.oneOrMore(digit))),
-		'}',
-	])
+	countRangeQuantifier = () => [
+		G.pattern([
+			'{',
+			R.captureAs('start',
+				R.oneOrMore(digit)
+			),
+			',',
+			R.possibly(R.captureAs('end',
+				R.oneOrMore(digit))
+			),
+			'}',
+		]),
+		G.possibly(this.nongreedyQuantifier)
+	]
+
+	nongreedyQuantifier = () => '?'
+
+	quantifier = () => G.anyOf(
+		this.starQuantifier,
+		this.plusQuantifier,
+		this.optionalQuantifier,
+		this.exactCountQuantifier,
+		this.countRangeQuantifier,
+	)
 
 	//////////////////////////////////////////////////////////////////////////////////////////////
 	// Group expressions
 	//////////////////////////////////////////////////////////////////////////////////////////////
-	group = () => G.anyOf(
-		this.uncapturedGroup,
-		this.namedCaptureGroup,
-		this.unnamedCaptureGroup,
-	)
-
 	uncapturedGroup = () => [
-		'(:',
-		this.disjunction,
+		'(?:',
+		this.sequenceOrDisjunction,
 		')',
 	]
 
 	namedCaptureGroup = () => [
 		'(?<',
-		G.pattern(R.captureAs('name', R.oneOrMore(letter))),
+		G.pattern(R.captureAs('name',
+			R.oneOrMore(identifierChar))
+		),
 		'>',
-		this.disjunction,
+		this.sequenceOrDisjunction,
 		')',
 	]
 
 	unnamedCaptureGroup = () => [
 		'(',
-		this.disjunction,
+		this.sequenceOrDisjunction,
 		')',
 	]
 
+	group = G.anyOf(
+		this.uncapturedGroup,
+		this.namedCaptureGroup,
+		this.unnamedCaptureGroup,
+	)
+
 	//////////////////////////////////////////////////////////////////////////////////////////////
 	// Backreferences
 	//////////////////////////////////////////////////////////////////////////////////////////////
 	unnamedBackreference = () =>
 		G.pattern([
 			'\\',
-			R.captureAs('index', digit),
+			R.captureAs('index',
+				digit
+			),
 		])
 
 	namedBackreference = () =>
 		G.pattern([
 			'\\k<',
-			R.captureAs('name', digit),
+			R.captureAs('name',
+				R.oneOrMore(identifierChar)
+			),
 			'>',
 		])
 
@@ -111,25 +142,25 @@ export class RegExpGrammar {
 	//////////////////////////////////////////////////////////////////////////////////////////////
 	positiveLookahead = () => [
 		'(?=',
-		this.disjunction,
+		this.sequenceOrDisjunction,
 		')',
 	]
 
 	negativeLookahead = () => [
 		'(?!',
-		this.disjunction,
+		this.sequenceOrDisjunction,
 		')',
 	]
 
 	positiveLookbehind = () => [
 		'(?<=',
-		this.disjunction,
+		this.sequenceOrDisjunction,
 		')',
 	]
 
 	negativeLookbehind = () => [
 		'(?<!',
-		this.disjunction,
+		this.sequenceOrDisjunction,
 		')',
 	]
 
@@ -148,44 +179,68 @@ export class RegExpGrammar {
 		this.charClass,
 		this.unicodeProperty,
 		this.notUnicodeProperty,
+		this.escapedCharacterClass,
 		this.charcodeOrEscapedChar,
 		this.unreservedCharLiteral
 	)
 
 	unreservedCharLiteral = () => G.pattern(
-		R.notAnyOfChars('[', '.', '*', '+', '?', '^', '$', '{', '}', '(', ')', '|', ']', '\\')
+		R.captureAs('char',
+			R.notAnyOfChars('[', '.', '*', '+', '?', '^', '$', '{', '}', '(', ')', '|', ']', '\\')
+		)
 	)
 
 	escapedCharLiteral = () => G.pattern([
 		'\\',
-		R.anyChar
+		R.captureAs('char',
+			R.anyChar
+		)
 	])
 
 	hexCharcode = () =>
 		G.pattern([
 			'\\x',
-			hexDigit,
-			hexDigit
+			R.captureAs('value',
+				R.repeated(2, hexDigit)
+			)
 		])
 
 	controlCharcode = () => G.pattern([
 		'\\c',
-		R.charRange('A', 'Z'),
+		R.captureAs('value',
+			R.anyOf(R.charRange('A', 'Z'), R.charRange('a', 'z'))
+		)
 	])
 
-	codepoint = () => G.pattern([
-		'\\u',
-		'{',
-		R.captureAs('value', R.oneOrMore(hexDigit)),
-		'}',
-	])
+	nullCharcode = () => G.pattern('\\0')
+
+	codepoint = () => G.anyOf(
+		G.pattern([
+			'\\u',
+			'{',
+			R.captureAs('value',
+				R.repeated([1, 6], hexDigit)
+			),
+			'}',
+		]),
+		G.pattern([
+			'\\u',
+			R.captureAs('value',
+				R.repeated(4, hexDigit)
+			)
+		])
+	)
 
 	codepointRange = () => G.pattern([
 		'\\u',
 		'{',
-		R.captureAs('start', R.oneOrMore(hexDigit)),
+		R.captureAs('start',
+			R.oneOrMore(hexDigit)
+		),
 		'-',
-		R.captureAs('end', R.oneOrMore(hexDigit)),
+		R.captureAs('end',
+			R.oneOrMore(hexDigit)
+		),
 		'}',
 	])
 
@@ -201,27 +256,29 @@ export class RegExpGrammar {
 
 	unicodePropertyBodyPattern: R.Pattern = [
 		'{',
-		R.captureAs('property', R.oneOrMore(letterOrDigit)),
+		R.captureAs('property',
+			R.oneOrMore(R.anyOf(letter, digit, '_', '-'))
+		),
 		R.possibly([
 			'=',
-			R.captureAs('value', R.oneOrMore(letterOrDigit)),
+			R.captureAs('value',
+				R.oneOrMore(R.anyOf(letter, digit, '_', '-'))
+			),
 		]),
 		'}',
 	]
 
-	charClassChar = () => G.pattern(R.notAnyOfChars(']'))
-
 	charClass = () => [
 		'[',
 		G.possibly(this.charClassNegator),
 		G.oneOrMore(G.anyOf(
 			this.charRange,
 			this.codepointRange,
-			this.charClass,
 			this.unicodeProperty,
 			this.notUnicodeProperty,
+			this.escapedCharacterClass,
 			this.charcodeOrEscapedChar,
-			this.charClassChar
+			this.charClassLiteral
 		)),
 		']',
 	]
@@ -230,27 +287,33 @@ export class RegExpGrammar {
 		this.codepoint,
 		this.hexCharcode,
 		this.controlCharcode,
+		this.nullCharcode,
 		this.escapedCharLiteral,
 	)
 
+	charClassLiteral = () => G.pattern(
+		R.captureAs('char',
+			R.notAnyOfChars(']')
+		)
+	)
+
 	charRangeElement = G.anyOf(
 		this.charcodeOrEscapedChar,
-		G.pattern(R.anyChar),
+		this.charClassLiteral,
 	)
 
+	charRangeStart = () => this.charRangeElement
+	charRangeEnd = () => this.charRangeElement
+
 	charRange = () => [
-		this.charRangeElement,
+		this.charRangeStart,
 		'-',
-		this.charRangeElement,
+		this.charRangeEnd,
 	]
 
 	//////////////////////////////////////////////////////////////////////////////////////////////
 	// Special symbols
 	//////////////////////////////////////////////////////////////////////////////////////////////
-	starQuantifier = () => '*'
-	plusQuantifier = () => '+'
-	nongreedyQuantifier = () => '?'
-
 	charClassNegator = () => '^'
 
 	inputStartAnchor = () => '^'
@@ -258,38 +321,67 @@ export class RegExpGrammar {
 
 	anyCharWildcard = () => '.'
 
-	disjunctionSeparator = () => '|'
-
 	anchor = G.anyOf(this.inputStartAnchor, this.inputEndAnchor)
 
 	escapedCharacterClass = () => G.pattern(R.anyOf(
-		R.captureAs('whitespace', escapedCharString.whitespace),
-		R.captureAs('nonWhitespace', escapedCharString.nonWhitespace),
-		R.captureAs('digit', escapedCharString.digit),
-		R.captureAs('nonDigit', escapedCharString.nonDigit),
-		R.captureAs('wordBoundary', escapedCharString.wordBoundary),
-		R.captureAs('nonWordBoundary', escapedCharString.nonWordBoundary),
-		R.captureAs('formFeed', escapedCharString.formFeed),
-		R.captureAs('carriageReturn', escapedCharString.carriageReturn),
-		R.captureAs('lineFeed', escapedCharString.lineFeed),
-		R.captureAs('tab', escapedCharString.tab),
-		R.captureAs('verticalTab', escapedCharString.verticalTab),
-		R.captureAs('backwardSlash', escapedCharString.backwardSlash),
+		R.captureAs('whitespace', escapedChars.whitespace),
+		R.captureAs('nonWhitespace', escapedChars.nonWhitespace),
+		R.captureAs('digit', escapedChars.digit),
+		R.captureAs('nonDigit', escapedChars.nonDigit),
+		R.captureAs('word', escapedChars.word),
+		R.captureAs('nonWord', escapedChars.nonWord),
+		R.captureAs('wordBoundary', escapedChars.wordBoundary),
+		R.captureAs('nonWordBoundary', escapedChars.nonWordBoundary),
+		R.captureAs('formFeed', escapedChars.formFeed),
+		R.captureAs('carriageReturn', escapedChars.carriageReturn),
+		R.captureAs('lineFeed', escapedChars.lineFeed),
+		R.captureAs('tab', escapedChars.tab),
+		R.captureAs('verticalTab', escapedChars.verticalTab),
+		R.captureAs('backwardSlash', escapedChars.backwardSlash),
 	))
+
+	//////////////////////////////////////////////////////////////////////////////////////////////
+	// Sequence elements (positioned here due to TypeScript class member ordering requirements)
+	//////////////////////////////////////////////////////////////////////////////////////////////
+	quantifiableExpression = G.cached(
+		G.anyOf(
+			this.group,
+			this.backReference,
+			this.singleCharExpression
+		)
+	)
+
+	quantifiedExpression = () => [
+		this.quantifiableExpression,
+		this.quantifier,
+	]
+
+	sequenceElement = G.anyOf(
+		this.anchor,
+		this.lookaround,
+		this.quantifiedExpression,
+		this.quantifiableExpression,
+	)
 }
 
+//////////////////////////////////////////////////////////////////////////////////////////////
+// Shared regular expressions
+//////////////////////////////////////////////////////////////////////////////////////////////
 const digit = R.charRange('0', '9')
 const letter = R.anyOf(R.charRange('a', 'z'), R.charRange('A', 'Z'))
-const letterOrDigit = R.anyOf(letter, digit)
 const hexDigit = R.anyOf(digit, R.charRange('a', 'f'), R.charRange('A', 'F'))
+const identifierChar = R.anyOf(letter, digit, '_', '$')
 
-const escapedCharString = {
+//////////////////////////////////////////////////////////////////////////////////////////////
+// Escaped characters lookup
+//////////////////////////////////////////////////////////////////////////////////////////////
+const escapedChars = {
 	whitespace: '\\s',
 	nonWhitespace: '\\S',
 	digit: '\\d',
 	nonDigit: '\\D',
-	word: '\\d',
-	nonWord: '\\D',
+	word: '\\w',
+	nonWord: '\\W',
 	wordBoundary: '\\b',
 	nonWordBoundary: '\\B',
 	formFeed: '\\f',

package/src/tests/test-grammars/SimpleTestGrammar1.ts

@@ -0,0 +1,10 @@
+import * as G from '../../exports/Exports.js'
+
+export class SimpleTestGrammar1 {
+	root = () => [
+		G.anyOf(
+			'abc',
+			'abcdefg',
+		)
+	]
+}

package/src/utilities/Utilities.ts

@@ -4,30 +4,30 @@ export function roundToDigits(val: number, digits = 3) {
 	return Math.round(val * multiplier) / multiplier
 }
 
-export function isNumber(value: any): value is number {
+export function isNumber(value: unknown): value is number {
 	return typeof value === 'number'
 }
 
-export function isString(value: any): value is string {
+export function isString(value: unknown): value is string {
 	return typeof value === 'string'
 }
 
-export function isBigInt(value: any): value is bigint {
+export function isBigInt(value: unknown): value is bigint {
 	return typeof value === 'bigint'
 }
 
-export function isObject(value: any): value is object {
+export function isObject(value: unknown): value is object {
 	return typeof value === 'object' && !Array.isArray(value)
 }
 
-export function isArray(value: any): value is any[] {
+export function isArray(value: unknown): value is unknown[] {
 	return Array.isArray(value)
 }
 
-export function isBoolean(value: any): value is boolean {
+export function isBoolean(value: unknown): value is boolean {
 	return typeof value === 'boolean'
 }
 
-export function isFunction(value: any): value is Function {
+export function isFunction(value: unknown): value is Function {
 	return typeof value === 'function'
 }