grammar-composer 0.3.1 → 0.5.0

package/src/parser-generator/Grammar.ts

@@ -2,47 +2,67 @@ import { Pattern, buildRegExp, inputStart, isPatternOptional } from 'regexp-comp
 import { isArray, isBoolean, isFunction, isString } from '../utilities/Utilities.js'
 
 import { parse } from './TopDownParser.js'
+import { detectAndAnnotateOptionalNodes, detectAndErrorOnLeftRecursion, validatePatternCaptureGroups } from './StaticAnalysis.js'
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 // Grammar builder method
 /////////////////////////////////////////////////////////////////////////////////////////////////
-export function buildGrammar<T extends { [key: string]: any }>(obj: T | (new () => T), startProductionName: keyof T): Grammar<T> {
-	if (isFunction(obj)) {
-		obj = new obj()
+export function buildGrammar<T extends { [key: string]: any }>(
+	definitionObject: T | (new () => T),
+	startProductionName: keyof T,
+	options?: GrammarBuilderOptions<T>): Grammar<T> {
+
+	options = {
+		unwrappedNonterminalNames: [],
+		...options
+	}
+
+	if (isFunction(definitionObject)) {
+		definitionObject = new definitionObject()
 	}
 
 	const nameLookup = new Map<any, keyof T>()
 	const nonterminalLookup = new Map<Function, Nonterminal>()
 	const optionalNonterminalLookup = new Map<Function, Nonterminal>()
 
-	for (const key in obj) {
-		const objectProperty = obj[key]
+	for (const key in definitionObject) {
+		const objectProperty = definitionObject[key]
 
 		nameLookup.set(objectProperty, key)
 
 		if (!isFunction(objectProperty)) {
-			continue
+			throw new Error(`Grammar definition contains a property '${key}', which is not a function. All object properties must be productions, declared as functions.`)
 		}
 
-		const productionContent = objectProperty.call(obj)
+		const productionContent = objectProperty.call(definitionObject)
 		const normalizedProduction = productionToGrammarElement(productionContent)
+		const isUnwrappedNonterminal = options.unwrappedNonterminalNames?.includes(key) === true
 
-		const newNonterminal = nonterminal(key, normalizedProduction)
-		const newOptionalNonterminal = { ...newNonterminal, optional: true }
+		const newNonterminal = nonterminal(key, normalizedProduction, isUnwrappedNonterminal)
+		const newOptionalNonterminal = {
+			...newNonterminal,
+			optional: true,
+			grammarNonterminal: newNonterminal,
+		}
 
 		nonterminalLookup.set(objectProperty, newNonterminal)
 		optionalNonterminalLookup.set(objectProperty, newOptionalNonterminal)
 	}
 
-	let uniqueIdCounter = 0
-	const uniqueIdSource = () => uniqueIdCounter++
+	let cacheIdCounter = 0
+	const getNewCacheId = () => {
+		const newCacheId = cacheIdCounter
+		cacheIdCounter += 1
+
+		return newCacheId
+	}
 
 	for (const [func, nonterminal] of nonterminalLookup) {
 		const preparedContent = prepareGrammarElement(
 			nonterminal.content,
 			nonterminalLookup,
 			optionalNonterminalLookup,
-			uniqueIdSource)
+			getNewCacheId)
 
 		nonterminal.content = preparedContent
 
@@ -50,7 +70,7 @@ export function buildGrammar<T extends { [key: string]: any }>(obj: T | (new ()
 		optionalNonterminal.content = preparedContent
 	}
 
-	const startNonterminal = nonterminalLookup.get(obj[startProductionName] as Function)
+	const startNonterminal = nonterminalLookup.get(definitionObject[startProductionName] as Function)
 
 	if (!startNonterminal) {
 		throw new Error(`Couldn't find a start production named '${startProductionName as string}'.`)
@@ -68,7 +88,7 @@ export function buildGrammar<T extends { [key: string]: any }>(obj: T | (new ()
 	return new Grammar<T>(
 		nonterminals,
 		startProductionName,
-		uniqueIdCounter
+		cacheIdCounter
 	)
 }
 
@@ -76,41 +96,44 @@ function prepareGrammarElement(
 	rootElement: GrammarElement,
 	nonterminalLookup: Map<Function, Nonterminal>,
 	optionalNonterminalLookup: Map<Function, Nonterminal>,
-	getUniqueId: () => number
+	getNewCacheId: () => number
 ): GrammarElement {
 	function prepare(element: GrammarElement): GrammarElement {
+		const setCacheIdIfNeeded = (element: GrammarElement) => {
+			if (element.cached === true) {
+				element.cacheId = getNewCacheId()
+			}
+
+			return element
+		}
+
 		switch (element.type) {
 			case 'StringTerminal':
 			case 'Nonterminal': {
-				if (element.uniqueId === undefined) {
-					element.uniqueId = getUniqueId()
-				}
-
-				return element
+				return setCacheIdIfNeeded({
+					...element
+				})
 			}
 
 			case 'PatternTerminal': {
-				return {
+				return setCacheIdIfNeeded({
 					...element,
-					uniqueId: getUniqueId()
-				}
+				})
 			}
 
 			case 'Repetition': {
-				return {
+				return setCacheIdIfNeeded({
 					...element,
 					content: prepare(element.content),
-					uniqueId: getUniqueId()
-				}
+				})
 			}
 
 			case 'Sequence':
 			case 'Choice': {
-				return {
+				return setCacheIdIfNeeded({
 					...element,
 					members: element.members.map(element => prepare(element)),
-					uniqueId: getUniqueId()
-				}
+				})
 			}
 
 			case 'NonterminalReference': {
@@ -128,286 +151,34 @@ function prepareGrammarElement(
 					throw new Error(`Couldn't resolve function reference in grammar element: ${JSON.stringify(element)}`)
 				}
 
-				if (element.cached !== undefined || element.uniqueId !== undefined) {
-					const clonedNonterminal = { ...nonterminal }
-
-					if (element.cached !== undefined) {
-						clonedNonterminal.cached = element.cached
-					}
-
-					clonedNonterminal.uniqueId = getUniqueId()
-
-					return clonedNonterminal
-				}
-
-				if (nonterminal.uniqueId === undefined) {
-					nonterminal.uniqueId = getUniqueId()
-				}
-
-				return nonterminal
-			}
-		}
-	}
-
-	return prepare(rootElement)
-}
-
-/////////////////////////////////////////////////////////////////////////////////////////////////
-// Internal static analysis methods
-/////////////////////////////////////////////////////////////////////////////////////////////////
-function detectAndAnnotateOptionalNodes(rootNode: GrammarElement) {
-	const visitedNodes = new Set<GrammarElement>()
-
-	const resolvedNodes = new Map<GrammarElement, boolean>()
-	const unresolvedNodes = new Map<GrammarElement, { dependencies: Set<GrammarElement>, isChoice: boolean }>()
-
-	function processDepthFirst(node: GrammarElement): boolean | undefined {
-		if (visitedNodes.has(node)) {
-			return resolvedNodes.get(node)
-		}
-
-		visitedNodes.add(node)
-
-		switch (node.type) {
-			case 'StringTerminal':
-			case 'PatternTerminal': {
-				resolvedNodes.set(node, node.optional)
-
-				return node.optional
-			}
-
-			case 'Nonterminal':
-			case 'Repetition': {
-				const result = processDepthFirst(node.content)
-
-				if (node.optional) {
-					resolvedNodes.set(node, true)
-
-					return true
-				} else if (isBoolean(result)) {
-					resolvedNodes.set(node, result)
-
-					return result
-				} else {
-					unresolvedNodes.set(node, { dependencies: new Set([node.content]), isChoice: false })
-
-					return undefined
-				}
-			}
-
-			case 'Sequence': {
-				const dependencies = new Set<GrammarElement>()
-
-				let hasNonOptionalResolvedMember = false
+				if (element.cached === true) {
+					return {
+						...nonterminal,
 
-				for (const element of node.members) {
-					const result = processDepthFirst(element)
+						cached: true,
+						cacheId: getNewCacheId(),
 
-					if (isBoolean(result)) {
-						if (result === false) {
-							hasNonOptionalResolvedMember = true
-						}
-					} else {
-						dependencies.add(element)
+						grammarNonterminal: nonterminal.grammarNonterminal ?? nonterminal
 					}
-				}
-
-				if (node.optional == true) {
-					resolvedNodes.set(node, true)
-
-					return true
-				} else if (hasNonOptionalResolvedMember) {
-					resolvedNodes.set(node, false)
-
-					return false
-				} else if (dependencies.size == 0) {
-					resolvedNodes.set(node, true)
-
-					return true
 				} else {
-					unresolvedNodes.set(node, { dependencies, isChoice: false })
-
-					return undefined
-				}
-			}
-
-			case 'Choice': {
-				const dependencies = new Set<GrammarElement>()
-				let hasOptionalResolvedMember = false
-
-				for (const element of node.members) {
-					const result = processDepthFirst(element)
-
-					if (isBoolean(result)) {
-						if (result === true) {
-							hasOptionalResolvedMember = true
-						}
-					} else {
-						dependencies.add(element)
-					}
-				}
-
-				if (node.optional == true) {
-					resolvedNodes.set(node, true)
-
-					return true
-				} else if (hasOptionalResolvedMember) {
-					resolvedNodes.set(node, true)
-
-					return true
-				} else if (dependencies.size == 0) {
-					resolvedNodes.set(node, false)
-
-					return false
-				} else {
-					unresolvedNodes.set(node, { dependencies, isChoice: true })
-
-					return undefined
+					return nonterminal
 				}
 			}
 		}
-
-		return undefined
 	}
 
-	// Process depth first to resolve the easy cases, for productions that contain
-	// no cyclic references:
-	processDepthFirst(rootNode)
-
-	// Now the remainder consists of nodes containing cyclic references that have not yet been resolved.
-	// Use a form of iterative elimination and substitution to resolve them:
-	while (unresolvedNodes.size > 0) {
-		// This variable tracks whether at least one dependency was resolved, in any node.
-		// If it stays false, it means that no improvement was made during the iteration,
-		// and we should exit the loop.
-		let atLastOneDependencyResolvedInAnyNode = false
-		const nodesToDelete: GrammarElement[] = []
-
-		// Scan the unresolved nodes to locate any new resolved dependencies
-		for (const [node, { dependencies, isChoice }] of unresolvedNodes) {
-			let nodeResolved = false
-
-			// Iterate over all unresolved dependencies for the node
-			for (const dependency of Array.from(dependencies)) {
-				// Check if the dependency has been resolved
-				const value = resolvedNodes.get(dependency)
-
-				if (value !== undefined) {
-					// If it did, record that some dependencies were resolved
-					atLastOneDependencyResolvedInAnyNode = true
-
-					if (isChoice) {
-						if (value === true) {
-							resolvedNodes.set(node, true)
-							nodeResolved = true
-							break
-						}
-					} else {
-						if (value === false) {
-							resolvedNodes.set(node, false)
-							nodeResolved = true
-							break
-						}
-					}
-
-					dependencies.delete(dependency)
-				}
-			}
-
-			if (nodeResolved) {
-				nodesToDelete.push(node)
-			} else if (dependencies.size === 0) {
-				if (isChoice) {
-					resolvedNodes.set(node, false)
-				} else {
-					resolvedNodes.set(node, true)
-				}
-
-				nodesToDelete.push(node)
-			}
-		}
-
-		for (const node of nodesToDelete) {
-			unresolvedNodes.delete(node)
-		}
-
-		// If not even one dependency was eliminated for any node,
-		// it means that only mutually cyclic nodes are left unresolved, so exit the loop.
-		if (!atLastOneDependencyResolvedInAnyNode) {
-			break
-		}
-	}
-
-	// All remaining unresolved nodes must now be optional,
-	// since they are all mutually cyclic and all their non-cyclic grammar elements are known to be optional.
-	for (const node of unresolvedNodes.keys()) {
-		resolvedNodes.set(node, true)
-		unresolvedNodes.delete(node)
-	}
-
-	// Finally set the 'optional' property of all nodes based on the detected values.
-	for (const [node, isOptional] of resolvedNodes) {
-		node.optional = isOptional
-	}
-}
-
-function detectAndErrorOnLeftRecursion(rootNode: GrammarElement) {
-	const currentlyIteratedNodes = new Set<GrammarElement>()
-
-	function detect(node: GrammarElement) {
-		if (currentlyIteratedNodes.has(node)) {
-			if (node.type === 'Nonterminal') {
-				throw new Error(`Detected left recursion for nonterminal '${node.name}'.`)
-			} else {
-				throw new Error(`Detected left recursion for node: ${JSON.stringify(node, undefined, 4)}`)
-			}
-		}
-
-		currentlyIteratedNodes.add(node)
-
-		switch (node.type) {
-			case 'Nonterminal':
-			case 'Repetition': {
-				detect(node.content)
-
-				break
-			}
-
-			case 'Sequence': {
-				for (const member of node.members) {
-					detect(member)
-
-					if (!member.optional) {
-						break
-					}
-				}
-
-				break
-			}
-
-			case 'Choice': {
-				for (const member of node.members) {
-					detect(member)
-				}
-
-				break
-			}
-		}
-
-		currentlyIteratedNodes.delete(node)
-	}
-
-	detect(rootNode)
+	return prepare(rootElement)
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
-// Exported builder methods
+// Exported AST builder functions
 /////////////////////////////////////////////////////////////////////////////////////////////////
 export function zeroOrMore(content: Production): Repetition {
 	return {
 		type: 'Repetition',
 		content: productionToGrammarElement(content),
-		optional: true
+		optional: true,
+		cached: false,
 	}
 }
 
@@ -415,12 +186,13 @@ export function oneOrMore(content: Production): Repetition {
 	return {
 		type: 'Repetition',
 		content: productionToGrammarElement(content),
-		optional: false
+		optional: false,
+		cached: false,
 	}
 }
 
 export function anyOf(...members: Production[]): Choice {
-	if (members.length == 0) {
+	if (members.length === 0) {
 		throw new Error(`'anyOf' requires at least one member.`)
 	}
 
@@ -430,12 +202,13 @@ export function anyOf(...members: Production[]): Choice {
 		type: 'Choice',
 		members: normalizedMembers,
 		optional: false,
-		exhaustive: false
+		exhaustive: false,
+		cached: false,
 	}
 }
 
 export function bestOf(...members: Production[]): Choice {
-	if (members.length == 0) {
+	if (members.length === 0) {
 		throw new Error(`'bestOf' requires at least one member.`)
 	}
 
@@ -445,7 +218,9 @@ export function bestOf(...members: Production[]): Choice {
 		type: 'Choice',
 		members: normalizedMembers,
 		optional: false,
-		exhaustive: true
+		exhaustive: true,
+
+		cached: false,
 	}
 }
 
@@ -460,6 +235,8 @@ export function pattern(pattern: Pattern): PatternTerminal {
 		pattern = [inputStart, pattern]
 	}
 
+	validatePatternCaptureGroups(pattern)
+
 	const regExp = buildRegExp(pattern)
 	const optional = isPatternOptional(pattern)
 
@@ -468,6 +245,8 @@ export function pattern(pattern: Pattern): PatternTerminal {
 		pattern,
 		regExp,
 		optional,
+
+		cached: false,
 	}
 }
 
@@ -480,7 +259,7 @@ export function uncached<T extends Production>(content: Production): T {
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
-// Internal builder methods
+// Internal AST builder methods
 /////////////////////////////////////////////////////////////////////////////////////////////////
 function stringTerminal(content: string): StringTerminal {
 	if (content.length < 1) {
@@ -490,11 +269,13 @@ function stringTerminal(content: string): StringTerminal {
 	return {
 		type: 'StringTerminal',
 		content,
-		optional: false
+		optional: false,
+
+		cached: false,
 	}
 }
 
-function nonterminal(name: string, content: GrammarElement): Nonterminal {
+function nonterminal(name: string, content: GrammarElement, unwrapped: boolean): Nonterminal {
 	if (name.length < 1) {
 		throw new Error(`A nonterminal name must include at least 1 character.`)
 	}
@@ -503,7 +284,10 @@ function nonterminal(name: string, content: GrammarElement): Nonterminal {
 		type: 'Nonterminal',
 		name,
 		content,
+
 		optional: false,
+		cached: false,
+		unwrapped,
 	}
 }
 
@@ -511,7 +295,9 @@ function sequence(members: GrammarElement[]): Sequence {
 	return {
 		type: 'Sequence',
 		members,
-		optional: false
+		optional: false,
+
+		cached: false,
 	}
 }
 
@@ -519,7 +305,9 @@ function unresolvedReference(reference: Function): NonterminalReference {
 	return {
 		type: 'NonterminalReference',
 		reference,
-		optional: false
+		optional: false,
+
+		cached: false,
 	}
 }
 
@@ -538,17 +326,18 @@ function productionToGrammarElement(production: Production): GrammarElement {
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
-// Type definitions
+// Grammar class
 /////////////////////////////////////////////////////////////////////////////////////////////////
 export class Grammar<T> {
 	readonly productions: Record<keyof T, any>
 	readonly startProductionName: keyof T
-	readonly maxElementId: number
 
-	constructor(productions: Record<keyof T, any>, startProductionName: keyof T, maxElementId: number) {
+	readonly maxCacheId: number
+
+	constructor(productions: Record<keyof T, any>, startProductionName: keyof T, maxCacheId: number) {
 		this.startProductionName = startProductionName
 		this.productions = productions
-		this.maxElementId = maxElementId
+		this.maxCacheId = maxCacheId
 	}
 
 	parse(text: string) {
@@ -560,6 +349,9 @@ export class Grammar<T> {
 	}
 }
 
+/////////////////////////////////////////////////////////////////////////////////////////////////
+// Type definitions
+/////////////////////////////////////////////////////////////////////////////////////////////////
 export type Production = string | GrammarElement | (() => Production) | Production[]
 
 export type GrammarElement =
@@ -574,8 +366,13 @@ export type GrammarElement =
 interface GrammarElementBase {
 	type: string
 	optional: boolean
-	uniqueId?: number
-	cached?: boolean
+
+	cached: boolean
+	cacheId?: number
+
+	// The name of the grammar property this element was assigned to, if any.
+	// Used by the parser to refer to the element by name in error messages.
+	name?: string
 }
 
 export type Terminal = StringTerminal | PatternTerminal
@@ -595,6 +392,12 @@ export interface Nonterminal extends GrammarElementBase {
 	type: 'Nonterminal'
 	name: string
 	content: GrammarElement
+	unwrapped: boolean
+
+	// The canonical nonterminal as defined in the grammar. Clones created for
+	// optional references and cached references keep a reference to the original,
+	// so identity comparisons work across clones.
+	grammarNonterminal?: Nonterminal
 }
 
 export interface Sequence extends GrammarElementBase {
@@ -617,3 +420,12 @@ export interface NonterminalReference extends GrammarElementBase {
 	type: 'NonterminalReference'
 	reference: Function
 }
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+// Builder type definitions
+/////////////////////////////////////////////////////////////////////////////////////////////////
+export interface GrammarBuilderOptions<T> {
+	unwrappedNonterminalNames?: GrammarNonterminalNames<T>
+}
+
+export type GrammarNonterminalNames<T> = (keyof T)[]