grammar-composer 0.4.0 → 0.6.0

package/src/parser-generator/ParseError.ts

@@ -0,0 +1,337 @@
+import type { Nonterminal, Terminal } from './Grammar.js'
+import { buildCaretSpacing, expandTabs, getLineAndColumn } from '../utilities/LineAndColumn.js'
+import { escapeChar, TerminalToText } from './TerminalToText.js'
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+// Parse error class
+/////////////////////////////////////////////////////////////////////////////////////////////////
+export class ParseError extends Error {
+	readonly input: string
+
+	readonly offset: number
+	readonly line: number
+	readonly column: number
+
+	readonly found?: string
+	readonly expected?: readonly ExpectedTerminal[]
+
+	readonly contextProduction?: string
+
+	constructor(details: ParseErrorDetails, maxDisplayedLineLength = 100) {
+		const { line, column } = getLineAndColumn(details.input, details.offset)
+
+		const errorMessage = buildErrorMessage(details, line, column, maxDisplayedLineLength)
+
+		super(errorMessage)
+
+		this.name = 'ParseError'
+
+		this.input = details.input
+
+		this.offset = details.offset
+		this.line = line
+		this.column = column
+
+
+		if (details.offset < details.input.length) {
+			this.found = details.input[details.offset]
+		}
+		this.expected = details.expected
+
+		this.contextProduction = details.contextProductionName
+	}
+
+	// Dedicated factory for the case where parsing failed because no expected
+	// terminal matched at some position.
+	static createFailedParseError(input: string, offset: number, bestFailedMatches: FailedMatch[]): ParseError {
+		const { expected, contextProduction } = analyzeFailedMatches(bestFailedMatches)
+
+		return new ParseError({
+			input,
+			offset,
+			expected,
+			contextProductionName: contextProduction.name
+		})
+	}
+
+	// Dedicated factory for the case where parsing consumed only part of the input
+	// without any specific terminal failing.
+	static createIncompleteParseError(input: string, offset: number, parsedLength: number): ParseError {
+		return new ParseError({ input, offset, parsedLength })
+	}
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+// Failed match analysis
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+// Computes the data needed to describe a parse failure: the deduplicated list of expected
+// terminals (each retaining the production it belongs to) and the shared context production.
+// The actual message formatting is owned by ParseError.
+export function analyzeFailedMatches(failedMatches: FailedMatch[]): FailedMatchAnalysisResult {
+	const productionStacks = failedMatches.map(failedMatch => failedMatch.productionStack)
+
+	const commonProductionPrefix = findCommonProductionPrefix(productionStacks)
+	const contextProduction = commonProductionPrefix[commonProductionPrefix.length - 1]
+
+	// Collect every production each distinct terminal could be part of, together with
+	// the production stack that led to it. The production is the one directly below
+	// the shared context production.
+	const candidatesByTerminal = new Map<string, TerminalCandidate[]>()
+
+	for (const failedMatch of failedMatches) {
+		const subProduction = failedMatch.productionStack[commonProductionPrefix.length] ?? contextProduction
+
+		if (subProduction === undefined) {
+			continue
+		}
+
+		const stringifiedFailedTerminal = stringifyFailedTerminal(failedMatch)
+
+		const candidates = candidatesByTerminal.get(stringifiedFailedTerminal) ?? []
+
+		candidates.push({ production: subProduction, productionStack: failedMatch.productionStack })
+		candidatesByTerminal.set(stringifiedFailedTerminal, candidates)
+	}
+
+	// When the same terminal can be reached through several productions and one of
+	// them is an ancestor of another (e.g. the same pattern is referenced both
+	// directly and through a parent production), keep only the innermost production.
+	const expected: ExpectedTerminal[] = []
+
+	for (const [stringifiedFailedTerminal, candidates] of candidatesByTerminal) {
+		for (const candidate of candidates) {
+			if (isRedundantCandidate(candidate, candidates)) {
+				continue
+			}
+
+			if (!expected.some(entry =>
+				entry.productionName === candidate.production.name && entry.terminalString === stringifiedFailedTerminal)) {
+
+				expected.push({ productionName: candidate.production.name, terminalString: stringifiedFailedTerminal })
+			}
+		}
+	}
+
+	return {
+		expected,
+		contextProduction,
+	}
+}
+
+// A candidate is redundant when the same terminal is also expected by a deeper
+// production within the same context, i.e. when another candidate's production
+// appears below this candidate's production in its own stack.
+function isRedundantCandidate(candidate: TerminalCandidate, candidates: TerminalCandidate[]): boolean {
+	const candidateIndex = stackIndexOf(candidate.productionStack, candidate.production)
+
+	for (const other of candidates) {
+		if (other === candidate) {
+			continue
+		}
+
+		const otherIndex = stackIndexOf(candidate.productionStack, other.production)
+
+		if (otherIndex > candidateIndex) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Compares nonterminals by their canonical grammar reference, because the same
+// production can be represented by multiple cloned instances during a parse.
+function stackIndexOf(productionStack: Nonterminal[], production: Nonterminal): number {
+	const canonical = production.grammarNonterminal ?? production
+
+	return productionStack.findIndex(nonterminal => (nonterminal.grammarNonterminal ?? nonterminal) === canonical)
+}
+
+function stringifyFailedTerminal(failedMatch: FailedMatch): string {
+	const { terminal, productionStack } = failedMatch
+
+	if (terminal.name !== undefined) {
+		return terminal.name
+	}
+
+	const innermostProduction = productionStack[productionStack.length - 1]
+
+	if (innermostProduction !== undefined && innermostProduction.content === terminal) {
+		return innermostProduction.name
+	}
+
+	const stringifiedLengthLimit = 100
+	const terminalToText = new TerminalToText(stringifiedLengthLimit)
+
+	return terminalToText.stringifyTerminal(terminal)
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+// Production grouping
+/////////////////////////////////////////////////////////////////////////////////////////////////
+
+// The stacks are compared by canonical nonterminal identity rather than object identity,
+// because the same production can be represented by multiple distinct object instances
+// during a single parse: the grammar builder creates spread copies of the canonical
+// nonterminal for optional references and for references wrapped in cached(). Each clone
+// keeps a reference to the original in 'grammarNonterminal'.
+export function findCommonProductionPrefix(productionStacks: Nonterminal[][]): Nonterminal[] {
+	if (productionStacks.length === 0) {
+		return []
+	}
+
+	let commonPrefix = productionStacks[0]
+
+	for (let i = 1; i < productionStacks.length; i++) {
+		const stack = productionStacks[i]
+		let commonLength = 0
+
+		while (commonLength < commonPrefix.length &&
+			commonLength < stack.length &&
+			areSameNonterminal(commonPrefix[commonLength], stack[commonLength])) {
+
+			commonLength += 1
+		}
+
+		commonPrefix = commonPrefix.slice(0, commonLength)
+
+		if (commonPrefix.length === 0) {
+			break
+		}
+	}
+
+	return commonPrefix
+}
+
+function areSameNonterminal(a: Nonterminal, b: Nonterminal): boolean {
+	return (a.grammarNonterminal ?? a) === (b.grammarNonterminal ?? b)
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+// Parse error helpers
+/////////////////////////////////////////////////////////////////////////////////////////////////
+function buildErrorMessage(details: ParseErrorDetails, line: number, column: number, maxDisplayedLineLength: number): string {
+	const context = formatErrorContext(details.input, details.offset, maxDisplayedLineLength)
+
+	if (details.parsedLength !== undefined) {
+		return `Failed parsing the input text ${context}\n\n` +
+			`Stopped parsing at line ${line}, column ${column} (${details.parsedLength} of ${details.input.length} characters consumed).`
+	}
+
+	const expected = details.expected ?? []
+	const expectedMessage = expected.length > 0
+		? buildExpectedGroupsMessage(expected)
+		: ''
+
+	const contextMessage = details.contextProductionName !== undefined
+		? `While parsing '${details.contextProductionName}', expected one of the following:\n\n${expectedMessage}`
+		: `Expected one of the following:\n\n${expectedMessage}`
+
+	return `Failed parsing the input text ${context}\n\n${contextMessage}`
+}
+
+// Formats the expected terminals grouped by production, with a blank line between
+// the groups so the message stays readable.
+function buildExpectedGroupsMessage(expected: readonly ExpectedTerminal[]): string {
+	const expectedByProduction = new Map<string, string[]>()
+
+	for (const { productionName, terminalString } of expected) {
+		const stringifiedTerminals = expectedByProduction.get(productionName) ?? []
+
+		if (!stringifiedTerminals.includes(terminalString)) {
+			stringifiedTerminals.push(terminalString)
+		}
+
+		expectedByProduction.set(productionName, stringifiedTerminals)
+	}
+
+	return [...expectedByProduction.entries()]
+		.map(([productionName, stringifiedTerminals]) => `In '${productionName}': ${stringifiedTerminals.join(', ')}`)
+		.join('\n\n')
+}
+
+export function formatErrorContext(input: string, offset: number, maxDisplayedLineLength: number): string {
+	const { line, column } = getLineAndColumn(input, offset)
+
+	let lineStartOffset = offset
+	while (lineStartOffset > 0 && input[lineStartOffset - 1] !== '\n') {
+		lineStartOffset -= 1
+	}
+
+	let lineEndOffset = offset
+	while (lineEndOffset < input.length && input[lineEndOffset] !== '\n') {
+		lineEndOffset += 1
+	}
+
+	const lineText = input.substring(lineStartOffset, lineEndOffset).replace(/\r$/, '')
+
+	let displayedText = lineText
+	let displayedCaretColumn = column
+
+	// If the failing line is very long (e.g. minified input), show a window around the
+	// error position instead of the entire line.
+	if (lineText.length > maxDisplayedLineLength) {
+		const rawCaretOffset = Math.min(column - 1, lineText.length)
+		const radius = Math.floor(maxDisplayedLineLength / 2)
+
+		const contextStart = Math.max(0, rawCaretOffset - radius)
+		const contextEnd = Math.min(lineText.length, rawCaretOffset + radius)
+
+		const hasTruncatedStart = contextStart > 0
+		const hasTruncatedEnd = contextEnd < lineText.length
+
+		const truncatedLine = lineText.substring(contextStart, contextEnd)
+
+		displayedText = `${hasTruncatedStart ? '…' : ''}${truncatedLine}${hasTruncatedEnd ? '…' : ''}`
+		displayedCaretColumn = (rawCaretOffset - contextStart) + (hasTruncatedStart ? 1 : 0) + 1
+	}
+
+	let unexpectedTargetText: string
+
+	if (offset < input.length) {
+		const targetChar = input[offset]
+		const escapedTargetChar = escapeChar(targetChar)
+
+		unexpectedTargetText = ` (unexpected character '${escapedTargetChar}')`
+	} else {
+		unexpectedTargetText = ' (unexpected end of input)'
+	}
+
+	const text = `at line ${line}, column ${column}${unexpectedTargetText}:\n\n\t${expandTabs(displayedText)}\n\t${buildCaretSpacing(displayedText, displayedCaretColumn)}^`
+
+	return text
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+// Types
+/////////////////////////////////////////////////////////////////////////////////////////////////
+interface TerminalCandidate {
+	production: Nonterminal
+	productionStack: Nonterminal[]
+}
+
+interface FailedMatchAnalysisResult {
+	expected: ExpectedTerminal[]
+	contextProduction: Nonterminal
+}
+
+export interface FailedMatch {
+	terminal: Terminal
+	productionStack: Nonterminal[]
+}
+
+// A single expected terminal, retaining the production it belongs to.
+export interface ExpectedTerminal {
+	productionName: string
+	terminalString: string
+}
+
+export interface ParseErrorDetails {
+	input: string
+	offset: number
+
+	expected?: readonly ExpectedTerminal[]
+	contextProductionName?: string
+	parsedLength?: number
+}

package/src/parser-generator/StaticAnalysis.ts

@@ -0,0 +1,340 @@
+import { Pattern } from 'regexp-composer'
+import { isArray, isBoolean, isString } from '../utilities/Utilities.js'
+import { GrammarElement } from './Grammar.js'
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+// Internal static analysis methods
+/////////////////////////////////////////////////////////////////////////////////////////////////
+export function detectAndAnnotateOptionalNodes(rootNode: GrammarElement) {
+	const visitedNodes = new Set<GrammarElement>()
+
+	const resolvedNodes = new Map<GrammarElement, boolean>()
+	const unresolvedNodes = new Map<GrammarElement, { dependencies: Set<GrammarElement>, isChoice: boolean }>()
+
+	function processDepthFirst(node: GrammarElement): boolean | undefined {
+		if (visitedNodes.has(node)) {
+			return resolvedNodes.get(node)
+		}
+
+		visitedNodes.add(node)
+
+		switch (node.type) {
+			case 'StringTerminal':
+			case 'PatternTerminal': {
+				resolvedNodes.set(node, node.optional)
+
+				return node.optional
+			}
+
+			case 'Nonterminal':
+			case 'Repetition': {
+				const result = processDepthFirst(node.content)
+
+				if (node.optional) {
+					resolvedNodes.set(node, true)
+
+					return true
+				} else if (isBoolean(result)) {
+					resolvedNodes.set(node, result)
+
+					return result
+				} else {
+					unresolvedNodes.set(node, { dependencies: new Set([node.content]), isChoice: false })
+
+					return undefined
+				}
+			}
+
+			case 'Sequence': {
+				const dependencies = new Set<GrammarElement>()
+
+				let hasNonOptionalResolvedMember = false
+
+				for (const element of node.members) {
+					const result = processDepthFirst(element)
+
+					if (isBoolean(result)) {
+						if (result === false) {
+							hasNonOptionalResolvedMember = true
+						}
+					} else {
+						dependencies.add(element)
+					}
+				}
+
+				if (node.optional === true) {
+					resolvedNodes.set(node, true)
+
+					return true
+				} else if (hasNonOptionalResolvedMember) {
+					resolvedNodes.set(node, false)
+
+					return false
+				} else if (dependencies.size === 0) {
+					resolvedNodes.set(node, true)
+
+					return true
+				} else {
+					unresolvedNodes.set(node, { dependencies, isChoice: false })
+
+					return undefined
+				}
+			}
+
+			case 'Choice': {
+				const dependencies = new Set<GrammarElement>()
+				let hasOptionalResolvedMember = false
+
+				for (const element of node.members) {
+					const result = processDepthFirst(element)
+
+					if (isBoolean(result)) {
+						if (result === true) {
+							hasOptionalResolvedMember = true
+						}
+					} else {
+						dependencies.add(element)
+					}
+				}
+
+				if (node.optional === true) {
+					resolvedNodes.set(node, true)
+
+					return true
+				} else if (hasOptionalResolvedMember) {
+					resolvedNodes.set(node, true)
+
+					return true
+				} else if (dependencies.size === 0) {
+					resolvedNodes.set(node, false)
+
+					return false
+				} else {
+					unresolvedNodes.set(node, { dependencies, isChoice: true })
+
+					return undefined
+				}
+			}
+		}
+
+		return undefined
+	}
+
+	// Process depth first to resolve the easy cases, for productions that contain
+	// no cyclic references:
+	processDepthFirst(rootNode)
+
+	// Now the remainder consists of nodes containing cyclic references that have not yet been resolved.
+	// Use a form of iterative elimination and substitution to resolve them:
+	while (unresolvedNodes.size > 0) {
+		// This variable tracks whether at least one dependency was resolved, in any node.
+		// If it stays false, it means that no improvement was made during the iteration,
+		// and we should exit the loop.
+		let atLastOneDependencyResolvedInAnyNode = false
+		
+		const nodesToDelete: GrammarElement[] = []
+
+		// Scan the unresolved nodes to locate any new resolved dependencies
+		for (const [node, { dependencies, isChoice }] of unresolvedNodes) {
+			let nodeResolved = false
+
+			// Iterate over all unresolved dependencies for the node
+			for (const dependency of Array.from(dependencies)) {
+				// Check if the dependency has been resolved
+				const value = resolvedNodes.get(dependency)
+
+				if (value !== undefined) {
+					// If it did, record that some dependencies were resolved
+					atLastOneDependencyResolvedInAnyNode = true
+
+					if (isChoice) {
+						if (value === true) {
+							resolvedNodes.set(node, true)
+							nodeResolved = true
+							break
+						}
+					} else {
+						if (value === false) {
+							resolvedNodes.set(node, false)
+							nodeResolved = true
+							break
+						}
+					}
+
+					dependencies.delete(dependency)
+				}
+			}
+
+			if (nodeResolved) {
+				nodesToDelete.push(node)
+			} else if (dependencies.size === 0) {
+				if (isChoice) {
+					resolvedNodes.set(node, false)
+				} else {
+					resolvedNodes.set(node, true)
+				}
+
+				nodesToDelete.push(node)
+			}
+		}
+
+		for (const node of nodesToDelete) {
+			unresolvedNodes.delete(node)
+		}
+
+		// If not even one dependency was eliminated for any node,
+		// it means that only mutually cyclic nodes are left unresolved, so exit the loop.
+		if (!atLastOneDependencyResolvedInAnyNode) {
+			break
+		}
+	}
+
+	// All remaining unresolved nodes must now be optional,
+	// since they are all mutually cyclic and all their non-cyclic grammar elements are known to be optional.
+	for (const node of unresolvedNodes.keys()) {
+		resolvedNodes.set(node, true)
+		unresolvedNodes.delete(node)
+	}
+
+	// Finally set the 'optional' property of all nodes based on the detected values.
+	for (const [node, isOptional] of resolvedNodes) {
+		node.optional = isOptional
+	}
+}
+
+export function detectAndErrorOnLeftRecursion(rootNode: GrammarElement) {
+	const currentlyIteratedNodes = new Set<GrammarElement>()
+
+	function detect(node: GrammarElement) {
+		if (currentlyIteratedNodes.has(node)) {
+			if (node.type === 'Nonterminal') {
+				throw new Error(`Detected left recursion for nonterminal '${node.name}'.`)
+			} else {
+				throw new Error(`Detected left recursion for node: ${JSON.stringify(node, undefined, 4)}`)
+			}
+		}
+
+		currentlyIteratedNodes.add(node)
+
+		switch (node.type) {
+			case 'Nonterminal':
+			case 'Repetition': {
+				detect(node.content)
+
+				break
+			}
+
+			case 'Sequence': {
+				for (const member of node.members) {
+					detect(member)
+
+					if (!member.optional) {
+						break
+					}
+				}
+
+				break
+			}
+
+			case 'Choice': {
+				for (const member of node.members) {
+					detect(member)
+				}
+
+				break
+			}
+		}
+
+		currentlyIteratedNodes.delete(node)
+	}
+
+	detect(rootNode)
+}
+
+// Walks the regexp-composer Pattern AST to validate its capture groups.
+//
+// The JavaScript RegExp engine collapses multiple named groups that share a name
+// into a single entry, and cannot reliably report the ordering of a mix of named
+// and unnamed groups. Both cases are problematic for the parser, so we detect them
+// here, once, at grammar build time (before the RegExp is even compiled), instead of
+// during parsing.
+//
+// Mirrors the AST traversal performed by regexp-composer's own 'isPatternOptional'.
+export function validatePatternCaptureGroups(pattern: Pattern): void {
+	const seenNamedCaptureGroupNames = new Set<string>()
+	let hasNamedCaptureGroup = false
+	let hasUnnamedCaptureGroup = false
+
+	function walk(node: Pattern): void {
+		if (isString(node)) {
+			return
+		}
+
+		if (isArray(node)) {
+			for (const element of node) {
+				walk(element)
+			}
+
+			return
+		}
+
+		switch (node.type) {
+			case 'capture': {
+				if (node.name !== undefined) {
+					if (seenNamedCaptureGroupNames.has(node.name)) {
+						throw new Error(`The regular expression pattern contains multiple named capture groups with the same name '${node.name}'. Named capture groups must have unique names.`)
+					}
+
+					seenNamedCaptureGroupNames.add(node.name)
+					hasNamedCaptureGroup = true
+				} else {
+					hasUnnamedCaptureGroup = true
+				}
+
+				walk(node.content)
+
+				return
+			}
+
+			// Composite nodes that wrap content without creating a capturing group:
+			case 'possibly':
+			case 'zeroOrMore':
+			case 'oneOrMore':
+			case 'repeated':
+			case 'followedBy':
+			case 'notFollowedBy':
+			case 'precededBy':
+			case 'notPrecededBy': {
+				walk(node.content)
+
+				return
+			}
+
+			case 'anyOf': {
+				for (const member of node.members) {
+					walk(member)
+				}
+
+				return
+			}
+
+			// These nodes never introduce capturing groups and contain only
+			// character-level tokens or backreferences:
+			case 'notAnyOfChars':
+			case 'specialToken':
+			case 'sameAs': {
+				return
+			}
+
+			default: {
+				throw new Error(`Unrecognized pattern type: ${(node as any).type}`)
+			}
+		}
+	}
+
+	walk(pattern)
+
+	if (hasNamedCaptureGroup && hasUnnamedCaptureGroup) {
+		throw new Error(`The regular expression pattern contains a combination of named and unnamed capture groups. Due to limitations of the JavaScript RegExp engine, it is impossible to reliably identify the ordering of this combination, please use either all unnamed or all named capture groups, but not both.`)
+	}
+}