grammar-composer 0.3.1 → 0.5.0

package/src/parser-generator/TerminalToText.ts

@@ -0,0 +1,196 @@
+import type { Pattern, SpecialToken } from 'regexp-composer'
+import type { PatternTerminal, StringTerminal, Terminal } from './Grammar.js'
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+// Terminal to text conversion
+/////////////////////////////////////////////////////////////////////////////////////////////////
+export class TerminalToText {
+	constructor(private stringifiedLengthLimit: number) {
+	}
+
+	stringifyTerminal(terminal: Terminal): string {
+		if (terminal.type === 'StringTerminal') {
+			return this.stringifyStringTerminal(terminal)
+		} else {
+			return this.stringifyPatternTerminal(terminal)
+		}
+	}
+
+	stringifyStringTerminal(terminal: StringTerminal): string {
+		return this.stringifyPatternStringLiteral(terminal.content)
+	}
+
+	stringifyPatternTerminal(terminal: PatternTerminal) {
+		let pattern = terminal.pattern
+
+		// Pattern terminals are automatically anchored at the start of the current parse
+		// position. This anchor is an implementation detail, so remove it.
+		if (Array.isArray(pattern) && pattern.length > 1) {
+			const firstElement = pattern[0]
+
+			if (!Array.isArray(firstElement) && typeof firstElement !== 'string' &&
+				firstElement.type === 'specialToken' && firstElement.name === 'inputStart') {
+
+				pattern = pattern.slice(1)
+			}
+		}
+
+		const stringifiedPattern = this.stringifyPattern(pattern)
+		const truncatedStringifiedPattern = this.truncateString(stringifiedPattern, this.stringifiedLengthLimit)
+
+		return truncatedStringifiedPattern
+	}
+
+	stringifyPattern(pattern: Pattern): string {
+		if (typeof pattern === 'string') {
+			return this.stringifyPatternStringLiteral(pattern)
+		}
+
+		if (Array.isArray(pattern)) {
+			return `[${pattern.map((pattern) => this.stringifyPattern(pattern)).join(', ')}]`
+		}
+
+		switch (pattern.type) {
+			case 'specialToken': {
+				return this.stringifySpecialToken(pattern)
+			}
+
+			case 'possibly': {
+				return `possibly(${this.stringifyPattern(pattern.content)})`
+			}
+
+			case 'zeroOrMore': {
+				return `zeroOrMore(${this.stringifyPattern(pattern.content)})`
+			}
+
+			case 'oneOrMore': {
+				return `oneOrMore(${this.stringifyPattern(pattern.content)})`
+			}
+
+			case 'repeated': {
+				const { minCount, maxCount } = pattern
+				const stringifiedContent = this.stringifyPattern(pattern.content)
+
+				if (minCount === maxCount) {
+					return `repeated(${minCount}, ${stringifiedContent})`
+				} else if (maxCount === Number.POSITIVE_INFINITY) {
+					return `repeated([${minCount}, ...], ${stringifiedContent})`
+				} else {
+					return `repeated([${minCount}, ${maxCount}], ${stringifiedContent})`
+				}
+			}
+
+			case 'capture': {
+				if (pattern.name !== undefined) {
+					return pattern.name
+				}
+
+				return `(${this.stringifyPattern(pattern.content)})`
+			}
+
+			case 'anyOf': {
+				const stringifiedMembers = pattern.members.map((pattern) => this.stringifyPattern(pattern))
+
+				let stringified = `anyOf(${stringifiedMembers.join(', ')})`
+
+				if (stringified.length > this.stringifiedLengthLimit && stringifiedMembers.length > 2) {
+					stringified = `anyOf(${stringifiedMembers[0]}, ${stringifiedMembers[1]}, …, ${stringifiedMembers[stringifiedMembers.length - 1]})`
+				}
+
+				return stringified
+			}
+
+			case 'notAnyOfChars': {
+				const stringifiedMembers = pattern.members.map((pattern) => this.stringifyPattern(pattern))
+
+				return `notAnyOfChars(${stringifiedMembers.join(', ')})`
+			}
+
+			case 'followedBy': {
+				return `followedBy(${this.stringifyPattern(pattern.content)})`
+			}
+
+			case 'notFollowedBy': {
+				return `notFollowedBy(${this.stringifyPattern(pattern.content)})`
+			}
+
+			case 'precededBy': {
+				return `precededBy(${this.stringifyPattern(pattern.content)})`
+			}
+
+			case 'notPrecededBy': {
+				return `notPrecededBy(${this.stringifyPattern(pattern.content)})`
+			}
+
+			case 'sameAs': {
+				const captureGroupNameOrIndex = pattern.captureGroupNameOrIndex
+
+				if (typeof captureGroupNameOrIndex === 'string') {
+					return `sameAs('${captureGroupNameOrIndex}')`
+				} else {
+					return `sameAs(${captureGroupNameOrIndex})`
+				}
+			}
+
+			default: {
+				throw new Error(`Unrecognized pattern type: ${(pattern as any).type}.`)
+			}
+		}
+	}
+
+	stringifyPatternStringLiteral(text: string): string {
+		let result = ''
+
+		for (const char of text) {
+			const escapedChar = escapeChar(char)
+
+			result += escapedChar
+		}
+
+		return `'${result}'`
+	}
+
+	stringifySpecialToken(token: SpecialToken): string {
+		if (token.name === 'charRange') {
+			const dashIndex = token.rawRegExp.indexOf('-')
+
+			if (dashIndex > 0 && dashIndex < token.rawRegExp.length - 1) {
+				const startChar = token.rawRegExp.substring(0, dashIndex).replace(/^\\/, '')
+
+				if (startChar.length > 0) {
+					return `charRange('${startChar}', '${token.rawRegExp.substring(dashIndex + 1)}')`
+				}
+			}
+		} else if (token.name !== undefined) {
+			return token.name
+		}
+
+		return token.rawRegExp
+	}
+
+	truncateString(str: string, lengthLimit: number): string {
+		if (str.length <= lengthLimit) {
+			return str
+		}
+
+		return `${str.substring(0, lengthLimit - 1)}…`
+	}
+}
+
+export function escapeChar(char: string): string {
+	const charMapping: Record<string, string> = {
+		//'\\': '\\\\',
+		"'": "\\'",
+		'\n': '\\n',
+		'\r': '\\r',
+		'\t': '\\t',
+	}
+
+	const mappedChar = charMapping[char]
+
+	if (mappedChar !== undefined) {
+		return mappedChar
+	} else {
+		return char
+	}
+}

package/src/parser-generator/TopDownParser.ts

@@ -1,54 +1,60 @@
-import { Grammar, GrammarElement, Terminal } from './Grammar.js'
+import { isNumber } from '../utilities/Utilities.js'
+import { Grammar, GrammarElement, Terminal, type Nonterminal } from './Grammar.js'
+import { type FailedMatch, ParseError } from './ParseError.js'
+
+//////////////////////////////////////////////////////////////////////////////////////////////
+// Main parser function
+//////////////////////////////////////////////////////////////////////////////////////////////
+export function parse(inputString: string, grammar: Grammar<any>, options?: TopDownParserOptions) {
+	options = { ...options }
 
-export function parse(inputString: string, grammar: Grammar<any>) {
 	const inputLength = inputString.length
 
-	let bestFailedMatches: Terminal[] = []
+	const nonterminalStack: Nonterminal[] = []
+
+	let bestFailedMatches: FailedMatch[] = []
 	let bestFailedMatchesOffset = -1
 
+	const cacheKeyOffsetMultiplier = grammar.maxCacheId + 1
+	const parseResultsCache = new Map<number, ParseResult | null>()
+
 	function updateBestFailedMatchesIfNeeded(terminal: Terminal, startOffset: number) {
 		if (startOffset >= bestFailedMatchesOffset) {
+			const failedMatch: FailedMatch = {
+				terminal,
+				productionStack: [...nonterminalStack]
+			}
+
 			if (startOffset > bestFailedMatchesOffset) {
 				bestFailedMatchesOffset = startOffset
-				bestFailedMatches = [terminal]
+				bestFailedMatches = [failedMatch]
 			} else {
-				bestFailedMatches.push(terminal)
+				bestFailedMatches.push(failedMatch)
 			}
 		}
 	}
 
 	function tryParse(grammarElement: GrammarElement, startOffset: number): ParseResult | null {
-		if (grammarElement.cached === true) {
+		if (isNumber(grammarElement.cacheId)) {
 			return tryParseCached(grammarElement, startOffset)
 		} else {
 			return tryParseUncached(grammarElement, startOffset)
 		}
 	}
 
-	type Slot = Map<GrammarElement, ParseResult | null> | undefined
-
-	const cachedParseResults: Slot[] = new Array(inputLength)
-
 	function tryParseCached(grammarElement: GrammarElement, startOffset: number): ParseResult | null {
-		let slot = cachedParseResults[startOffset]
-
-		if (slot === undefined) {
-			slot = new Map<GrammarElement, ParseResult | null>()
+		const cacheId = grammarElement.cacheId!
+		const cacheKey = (startOffset * cacheKeyOffsetMultiplier) + cacheId
 
-			cachedParseResults[startOffset] = slot
+		if (parseResultsCache.has(cacheKey)) {
+			return parseResultsCache.get(cacheKey)!
 		} else {
-			const cachedResult = slot.get(grammarElement)
-
-			if (cachedResult !== undefined) {
-				return cachedResult
-			}
-		}
+			const parseResult = tryParseUncached(grammarElement, startOffset)
 
-		const parseResult = tryParseUncached(grammarElement, startOffset)
+			parseResultsCache.set(cacheKey, parseResult)
 
-		slot.set(grammarElement, parseResult)
-
-		return parseResult
+			return parseResult
+		}
 	}
 
 	function tryParseUncached(grammarElement: GrammarElement, startOffset: number): ParseResult | null {
@@ -100,10 +106,6 @@ export function parse(inputString: string, grammar: Grammar<any>) {
 
 					if (groupsIndices.groups) {
 						namedGroupIndicesIdentifiers = Object.keys(groupsIndices.groups)
-
-						if (namedGroupIndicesIdentifiers.length !== groupsIndices.length - 1) {
-							throw new Error(`The regular expression /${grammarElement.regExp.source}/ contains a combination of named and unnamed groups. Due to limitations of the JavaScript RegExp engine, it is impossible to reliably identify the ordering of this combination, please use either all unnamed or named groups, but not both.`)
-						}
 					}
 
 					const children: ParseTreeNode[] = []
@@ -139,26 +141,44 @@ export function parse(inputString: string, grammar: Grammar<any>) {
 			}
 
 			case 'Nonterminal': {
+				nonterminalStack.push(grammarElement)
+
 				const result = tryParse(grammarElement.content, startOffset)
 
+				nonterminalStack.pop()
+
 				if (result === null) {
 					return null
 				}
 
-				let newNode: ParseTreeNode = {
-					name: grammarElement.name,
-					startOffset,
-					endOffset: result.endOffset,
-					sourceText: inputString.substring(startOffset, result.endOffset),
-					children: result.nodes,
-				}
+				const grammarElementName = grammarElement.name
 
-				const newResult: ParseResult = {
-					endOffset: result.endOffset,
-					nodes: [newNode]
-				}
+				if (grammarElement.unwrapped) {
+					const newResult: ParseResult = {
+						endOffset: result.endOffset,
+						nodes: result.nodes
+					}
+
+					return newResult
+				} else {
+					const newNode: ParseTreeNode = {
+						name: grammarElementName,
 
-				return newResult
+						startOffset,
+						endOffset: result.endOffset,
+
+						sourceText: inputString.substring(startOffset, result.endOffset),
+
+						children: result.nodes,
+					}
+
+					const newResult: ParseResult = {
+						endOffset: result.endOffset,
+						nodes: [newNode]
+					}
+
+					return newResult
+				}
 			}
 
 			case 'Sequence': {
@@ -258,36 +278,22 @@ export function parse(inputString: string, grammar: Grammar<any>) {
 	if (result && result.endOffset >= inputLength) {
 		return result.nodes ?? []
 	} else {
-		if (bestFailedMatches.length > 0) {
-			const possibleMatches = bestFailedMatches.map(match => {
-				if (match.type === 'StringTerminal') {
-					return `'${match.content}'`
-				} else if (match.type === 'PatternTerminal') {
-					return `/${match.regExp.source}/`
-				} else {
-					throw new Error(`Invalid match type: '${(match as any).type}'`)
-				}
-			})
-
-			const possibleMatchesWithoutDuplicates = [...(new Set(possibleMatches))]
+		const failureOffset = bestFailedMatches.length > 0 ? bestFailedMatchesOffset : (result?.endOffset ?? 0)
 
-			let possibleMatchesString: string
-
-			if (possibleMatchesWithoutDuplicates.length > 1) {
-				possibleMatchesString = `any of ${possibleMatchesWithoutDuplicates.join(', ')}`
-			} else {
-				possibleMatchesString = possibleMatchesWithoutDuplicates[0]
-			}
-
-			throw new Error(`Failed parsing the input text. Expected ${possibleMatchesString} at position ${bestFailedMatchesOffset}.`)
+		if (bestFailedMatches.length > 0) {
+			throw ParseError.createFailedParseError(inputString, failureOffset, bestFailedMatches)
 		} else {
 			const lastNode = result?.nodes?.[result.nodes.length - 1]
+			const parsedLength = lastNode?.endOffset ?? result?.endOffset ?? 0
 
-			throw new Error(`Failed parsing the input text. Parsed length was ${lastNode?.endOffset ?? result?.endOffset ?? 0}. Input length was ${inputLength}.`)
+			throw ParseError.createIncompleteParseError(inputString, failureOffset, parsedLength)
 		}
 	}
 }
 
+//////////////////////////////////////////////////////////////////////////////////////////////
+// Types
+//////////////////////////////////////////////////////////////////////////////////////////////
 export interface ParseResult {
 	endOffset: number
 	nodes: ParseTreeNode[] | undefined
@@ -295,8 +301,14 @@ export interface ParseResult {
 
 export interface ParseTreeNode {
 	name: string
+
 	startOffset: number
 	endOffset: number
+
 	sourceText: string
-	children: ParseTreeNode[] | undefined
+
+	children?: ParseTreeNode[]
+}
+
+export interface TopDownParserOptions {
 }

package/src/tests/Test.ts

@@ -1,9 +1,11 @@
 import { Timer } from '../utilities/Timer.js'
 import { jsonSample1, jsonSample2 } from './test-data/TestData.js'
 import { anyOf, buildGrammar } from '../exports/Exports.js'
-import { JsonGrammar } from './test-grammars/JsonGrammar.js'
-import { XmlGrammar } from './test-grammars/XmlGrammar.js'
-import { RegExpGrammar } from './test-grammars/RegExpGrammar.js'
+import { JsonGrammar, jsonGrammarUnwrappedNonterminalNames } from './test-grammars/JsonGrammar.js'
+import { XmlGrammar, xmlGrammarUnwrappedNonterminalNames } from './test-grammars/XmlGrammar.js'
+import { RegExpGrammar, regExpGrammarUnwrappedNonterminalNames } from './test-grammars/RegExpGrammar.js'
+import { writeFile } from 'fs/promises'
+import { SimpleTestGrammar1 } from './test-grammars/SimpleTestGrammar1.js'
 
 const log = console.log
 
@@ -28,7 +30,9 @@ function testBasic() {
 function testJsonParser() {
 	const jsonString = jsonSample1
 
-	const grammar = buildGrammar(JsonGrammar, 'expression')
+	const grammar = buildGrammar(JsonGrammar, 'expression', {
+		unwrappedNonterminalNames: jsonGrammarUnwrappedNonterminalNames
+	})
 
 	const iterations = 1000
 
@@ -50,7 +54,7 @@ function testJsonParser() {
 	log(JSON.stringify(result1, undefined, 4))
 }
 
-function testXmlParser() {
+async function testXmlParser() {
 	const xmlString = `
 <!DOCTYPE web-app>
 
@@ -58,34 +62,33 @@ function testXmlParser() {
     <header>Adobe SVG Viewer</header>
     <item action="Open" id="Open">Open</item>
     <item action="OpenNew" id="OpenNew">Open New</item>
-    <separator/>
-    <item action="ZoomIn" id="ZoomIn">Zoom In</item>
-    <item action="ZoomOut" id="ZoomOut">Zoom Out</item>
-    <separator/>
-    <item action="Quality" id="Quality">Quality</item>
-    <item action="Pause" id="Pause">Pause</item>
-    <item action="Mute" id="Mute">Mute</item>
-    <separator/>
-    <item action="Find" id="Find">Find...</item>
-    <item action="FindAgain" id="FindAgain">Find Again</item>
-    <item action="Copy" id="Copy">Copy</item>
 </menu>
-
 `
 	// Build the grammar. 'document' is the starting production
-	const grammar = buildGrammar(XmlGrammar, 'document')
+	const grammar = buildGrammar(XmlGrammar, 'document', {
+		unwrappedNonterminalNames: xmlGrammarUnwrappedNonterminalNames
+	})
 
 	// Parse the XML string
 	const parseTree = grammar.parse(xmlString)
 
-	log(JSON.stringify(parseTree, undefined, 4))
+	const parseTreeJson = JSON.stringify(parseTree, undefined, 4)
+
+	log(parseTreeJson)
+
+	await writeFile('out/out.json', parseTreeJson)
 }
 
 async function testRegExpParser() {
-	const regExpString = /^([+]?[1]?(1 )?[-.+]?\(?\d{1}[- .+]*\d{1}[- .+]*\d{1}\)?[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1})$/.source
-	//const regExpString = /^asdf{$/.source
+	//const regExpString = /^([+]?[1]?(1 )?[-.+]?\(?\d{1}[- .+]*\d{1}[- .+]*\d{1}\)?[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1}[- .+]*\d{1})$/.source
+	//const regExpString = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/.source
+	//const regExpString = /(?=.*[!@#$%^&*])^mongodb:\/\/(?<user>[a-zA-Z0-9]+):(?<pass>[a-zA-Z0-9!@#$%^&*]{8,})@(?<host>[a-z0-9.-]+):(?<port>\d{2,5})$/.source
+	const regExpString = /^(abcd)*ef+g/.source
+	//const regExpString = '(abcd)(aa))'
 
-	const grammar = buildGrammar(RegExpGrammar, 'disjunction')
+	const grammar = buildGrammar(RegExpGrammar, 'root', {
+		unwrappedNonterminalNames: regExpGrammarUnwrappedNonterminalNames
+	})
 
 	const parseTree = grammar.parse(regExpString)
 
@@ -93,13 +96,10 @@ async function testRegExpParser() {
 
 	log(parseTreeJson)
 
-	const { writeFile } = await import('fs/promises')
-
 	await writeFile('out/out.json', parseTreeJson)
 }
 
-
-async function testParserError1() {
+function testParserError1() {
 	const xmlData = `<hello> wo rld <!!! `
 
 	const grammar = buildGrammar(XmlGrammar, 'document')
@@ -109,7 +109,7 @@ async function testParserError1() {
 	console.log(JSON.stringify(result, undefined, 4))
 }
 
-async function testParserError2() {
+function testParserError2() {
 	const jsonData = `{ "asdf": 12.5 `
 
 	const grammar = buildGrammar(JsonGrammar, 'expression')
@@ -119,6 +119,22 @@ async function testParserError2() {
 	console.log(JSON.stringify(result, undefined, 4))
 }
 
+function test1() {
+	const input = `abcdefg`
+
+	const grammar = buildGrammar(SimpleTestGrammar1, 'root')
+
+	const result = grammar.parse(input)
+
+	console.log(JSON.stringify(result, undefined, 4))
+}
+
+
+//testParserError1()
+//testParserError2()
+
 //testJsonParser()
+testXmlParser()
+//testRegExpParser()
 
-testRegExpParser()
+//test1()

package/src/tests/test-grammars/JsonGrammar.ts

@@ -11,7 +11,7 @@ export class JsonGrammar {
 		this.arrayExpression
 	)
 
-	stringLiteral = G.pattern([
+	stringLiteral = () => G.pattern([
 		zeroOrMoreWhitespace,
 
 		'"',
@@ -27,7 +27,7 @@ export class JsonGrammar {
 		zeroOrMoreWhitespace,
 	])
 
-	numberLiteral = G.pattern([
+	numberLiteral = () => G.pattern([
 		zeroOrMoreWhitespace,
 
 		R.captureAs('value', [
@@ -46,7 +46,7 @@ export class JsonGrammar {
 		zeroOrMoreWhitespace,
 	])
 
-	booleanLiteral = G.pattern([
+	booleanLiteral = () => G.pattern([
 		zeroOrMoreWhitespace,
 
 		R.captureAs('value',
@@ -56,7 +56,7 @@ export class JsonGrammar {
 		zeroOrMoreWhitespace,
 	])
 
-	nullLiteral = G.pattern([
+	nullLiteral = () => G.pattern([
 		zeroOrMoreWhitespace,
 
 		'null',
@@ -92,14 +92,14 @@ export class JsonGrammar {
 		this.closingSquareBracket
 	]
 
-	openingCurlyBrace = createPatternWithClearedWhitespace('{')
-	closingCurlyBrace = createPatternWithClearedWhitespace('}')
+	openingCurlyBrace = () => createPatternWithClearedWhitespace('{')
+	closingCurlyBrace = () => createPatternWithClearedWhitespace('}')
 
-	openingSquareBracket = createPatternWithClearedWhitespace('[')
-	closingSquareBracket = createPatternWithClearedWhitespace(']')
+	openingSquareBracket = () => createPatternWithClearedWhitespace('[')
+	closingSquareBracket = () => createPatternWithClearedWhitespace(']')
 
-	comma = createPatternWithClearedWhitespace(',')
-	colons = createPatternWithClearedWhitespace(':')
+	comma = () => createPatternWithClearedWhitespace(',')
+	colons = () => createPatternWithClearedWhitespace(':')
 }
 
 function createPatternWithClearedWhitespace(subpattern: R.Pattern) {
@@ -113,3 +113,21 @@ function createPatternWithClearedWhitespace(subpattern: R.Pattern) {
 }
 
 const zeroOrMoreWhitespace = R.zeroOrMore(R.whitespace)
+
+//////////////////////////////////////////////////////////////////////////////////////////////
+// Wrapped nonterminal names
+//////////////////////////////////////////////////////////////////////////////////////////////
+export const jsonGrammarUnwrappedNonterminalNames: G.GrammarNonterminalNames<JsonGrammar> = [
+	//'stringLiteral',
+	//'numberLiteral',
+	//'booleanLiteral',
+	//'nullLiteral',
+
+	'openingCurlyBrace',
+	'closingCurlyBrace',
+	'openingSquareBracket',
+	'closingSquareBracket',
+
+	'comma',
+	'colons',
+]