rltk 2.2.1 → 3.0.0

data/lib/rltk/cfg.rb

@@ -10,6 +10,9 @@
 # Requires #
 ############
 
+# Standard Library
+require 'set'
+
 # Ruby Language Toolkit
 require 'rltk/lexers/ebnf'
 
@@ -17,8 +20,7 @@ require 'rltk/lexers/ebnf'
 # Classes and Modules #
 #######################
 
-module RLTK # :nodoc:
-	
+module RLTK
 	# An exception class that represents a problem with a context-free
 	# grammar's definition.
 	class GrammarError < StandardError; end
@@ -71,22 +73,22 @@ module RLTK # :nodoc:
 		#
 		# @param [Proc] callback A Proc object to be called when EBNF operators are expanded.
 		def initialize(&callback)
-			@curr_lhs			= nil
-			@callback			= callback || Proc.new {}
-			@lexer			= Lexers::EBNF.new
-			@production_counter	= -1
-			@start_symbol		= nil
-			@wrapper_symbol	= nil
-			
-			@productions_id	= Hash.new
-			@productions_sym	= Hash.new { |h, k| h[k] = [] }
-			@production_buffer	= Array.new
-			
-			@terms	= Hash.new(false).update({:EOS => true})
-			@nonterms	= Hash.new(false)
-			
-			@firsts	= Hash.new
-			@follows	= Hash.new { |h,k| h[k] = Array.new }
+			@curr_lhs           = nil
+			@callback           = callback || Proc.new {}
+			@lexer              = Lexers::EBNF.new
+			@production_counter = -1
+			@start_symbol       = nil
+			@wrapper_symbol     = nil
+			
+			@productions_id     = Hash.new
+			@productions_sym    = Hash.new { |h, k| h[k] = [] }
+			@production_buffer  = Array.new
+			
+			@terms    = Set.new([:EOS])
+			@nonterms = Set.new
+			
+			@firsts   = Hash.new
+			@follows  = Hash.new { |h,k| h[k] = Array.new }
 		end
 		
 		# Adds *production* to the appropriate internal data structures.
@@ -100,13 +102,136 @@ module RLTK # :nodoc:
 			production
 		end
 		
+		# Builds a production representing a (possibly empty) list of tokens.
+		# These tokens may optionally be separated by a provided token.  This
+		# function is used to eliminate the EBNF * operator.
+		#
+		# @param [Symbol]                         name           The name of the production to add
+		# @param [String, Symbol, Array<String>]  list_elements  Expression(s) that may appear in the list
+		# @param [Symbol, String]                 separator      The list separator symbol or symbols
+		#
+		# @return [void]
+		def build_list_production(name, list_elements, separator = '')
+			# Add the items for the following productions:
+			#
+			# name: | name_prime
+			
+			name_prime = "#{name}_prime".to_sym
+			
+			# 1st Production
+			production, _ = self.production(name, '')
+			@callback.call(:elp, :empty, production)
+			
+			# 2nd Production
+			production, _ = self.production(name, name_prime)
+			@callback.call(:elp, :nonempty, production)
+			
+			# Add remaining productions via nonempty_list helper.
+			self.nonempty_list(name_prime, list_elements, separator)
+			
+			name
+		end
+		alias :list :build_list_production
+		
+		# Builds a production representing a non-empty list of tokens.  These
+		# tokens may optionally be separated by a provided token.  This
+		# function is used to eliminate the EBNF + operator.
+		#
+		# @param [Symbol]                                 name           The name of the production to add
+		# @param [String, Symbol, Array<String, Symbol>]  list_elements  Expression(s) that may appear in the list
+		# @param [Symbol, String]                         separator      The list separator symbol or symbols
+		#
+		# @return [void]
+		def build_nonempty_list_production(name, list_elements, separator = '')
+			# Add the items for the following productions:
+			#
+			# If there is only one list element:
+			#
+			#   name: list_element | name separator list_element
+			#
+			# else
+			#
+			#   name: name_list_elements | name separator name_list_elements
+			#
+			#   name_list_elements: #{list_elements.join('|')}
+			
+			build_elements_productions = false
+			
+			list_element_string =
+			if list_elements.is_a?(Array)
+				if list_elements.empty?
+					raise ArgumentError, 'Parameter list_elements must not be empty.'
+					
+				elsif list_elements.length == 1
+					list_elements.first
+					
+				else
+					build_elements_productions = true
+					"#{name}_list_elements"
+				end
+			else
+				list_elements
+			end
+			
+			list_element_selected_string = list_element_string.to_s.split.map { |s| ".#{s}" }.join(' ')
+			
+			# Single Element Production
+			production, _ = self.production(name, list_element_string)
+			@callback.call(:nelp, :single, production)
+			
+			# Multiple Element Production
+			production, selections = self.production(name, ".#{name} #{separator} #{list_element_selected_string}")
+			@callback.call(:nelp, :multiple, production, selections)
+			
+			if build_elements_productions
+				# List Element Productions
+				list_elements.each do |element|
+					production, _ = self.production(list_element_string, element)
+					@callback.call(:nelp, :elements, production)
+				end
+			end
+			
+			name
+		end
+		alias :nonempty_list :build_nonempty_list_production
+		
+		# Build a production for an optional symbol.  This is used to
+		# eliminate the EBNF ? operator.
+		#
+		# @param [Symbol]  name        The name for the new production
+		# @param [Symbol]  opt_symbol  Symbol to expand
+		#
+		# @return [Symbol]  The value of the name argument
+		def build_optional_production(name, opt_symbol)
+			if not @productions_sym.has_key?(name)
+				# Add the items for the following productions:
+				#
+				# name: | opt_symbol
+				
+				# Empty production.
+				production = self.add_production(Production.new(self.next_id, name, []))
+				@callback.call(:optional, :empty, production)
+				
+				# Nonempty production
+				production = self.add_production(Production.new(self.next_id, name, [opt_symbol]))
+				@callback.call(:optional, :nonempty, production)
+				
+				# Add the new symbol to the list of nonterminals.
+				@nonterms << name
+			end
+			
+			name
+		end
+		
 		# Sets the EBNF callback to *callback*.
 		#
 		# @param [Proc] callback A Proc object to be called when EBNF operators are expanded and list productions are added.
 		#
 		# @return [void]
 		def callback(&callback)
-			@callback = callback || Proc.new {}
+			@callback = callback if callback
+			
+			nil
 		end
 		
 		# This function MUST be called inside a CFG.production block.  It will
@@ -114,90 +239,73 @@ module RLTK # :nodoc:
 		# CFG.production call's argument.  This is the function that is
 		# responsible for removing EBNF symbols from the grammar.
 		#
-		# @param [String] expression The right-hand side of a CFG production.
+		# @param [String, Symbol] expression The right-hand side of a CFG production.
 		#
-		# @return [Production]
+		# @return [Array(Production, Array<Integer>)]
 		def clause(expression)
 			raise GrammarError, 'CFG#clause called outside of CFG#production block.' if not @curr_lhs
 			
-			lhs		= @curr_lhs.to_sym
-			rhs		= Array.new
-			tokens	= @lexer.lex(expression)
+			lhs        = @curr_lhs.to_sym
+			rhs        = Array.new
+			tokens     = @lexer.lex(expression.to_s)
+			selections = Array.new
 			
 			# Set this as the start symbol if there isn't one already
 			# defined.
 			@start_symbol ||= lhs
 			
 			# Remove EBNF tokens and replace them with new productions.
+			symbol_count = 0
 			tokens.each_index do |i|
-				ttype0	= tokens[i].type
-				tvalue0	= tokens[i].value
+				ttype0  = tokens[i].type
+				tvalue0 = tokens[i].value
 				
 				if ttype0 == :TERM or ttype0 == :NONTERM
 					
 					# Add this symbol to the correct collection.
-					(ttype0 == :TERM ? @terms : @nonterms)[tvalue0] = true
+					(ttype0 == :TERM ? @terms : @nonterms) << tvalue0
 					
 					if i + 1 < tokens.length
-						ttype1	= tokens[i + 1].type
-						tvalue1	= tokens[i + 1].value
+						ttype1  = tokens[i + 1].type
+						tvalue1 = tokens[i + 1].value
 						
 						rhs <<
 						case ttype1
-						when :'?'	then self.get_question(tvalue0)						
-						when :*	then self.get_star(tvalue0)
-						when :+	then self.get_plus(tvalue0)
-						else			tvalue0
+						when :QUESTION then self.build_optional_production("#{tvalue0.downcase}_optional".to_sym, tvalue0)
+						when :STAR     then self.build_list_production("#{tvalue0.downcase}_list".to_sym, tvalue0)
+						when :PLUS     then self.build_nonempty_list_production("#{tvalue0.downcase}_nonempty_list".to_sym, tvalue0)
+						else                tvalue0
 						end
 					else
 						rhs << tvalue0
 					end
+					
+					symbol_count += 1
+					
+				elsif ttype0 == :DOT
+					selections << symbol_count
 				end
 			end
 			
 			# Make the production.
-			@production_buffer << (production = Production.new(self.next_id, lhs, rhs))
+			@production_buffer << [(production = Production.new(self.next_id, lhs, rhs)), selections]
 			
 			# Make sure the production symbol is collected.
-			@nonterms[lhs] = true
+			@nonterms << lhs
 			
 			# Add the new production to our collections.
 			self.add_production(production)
 			
-			return production
+			return [production, selections]
 		end
 		
-		# This method adds the necessary productions for empty lists to the
-		# grammar.  These productions are named `symbol`, `symbol + '_prime'`
-		# and `symbol + '_elements'`
-		#
-		# @param [Symbol]		symbol		The name of the production to add.
-		# @param [Array<String>]	list_elements	An array of expressions that may appear in the list.
-		# @param [Symbol]		separator		The list separator symbol.
+		# This function calculates the *first* set of a series of tokens.  It
+		# uses the {CFG#first_set} helper function to find the first set of
+		# individual symbols.
 		#
-		# @return [void]
-		def empty_list_production(symbol, list_elements, separator)
-			# Add the items for the following productions:
-			#
-			# symbol: | symbol_prime
-			
-			prime = symbol.to_s + '_prime'
-			
-			# 1st Production
-			production = self.production(symbol, '').first
-			@callback.call(production, :elp, :first)
-			
-			# 2nd Production
-			production = self.production(symbol, prime.to_s).first
-			@callback.call(production, :elp, :second)
-			
-			self.nonempty_list(prime, list_elements, separator)
-		end
-		alias :empty_list :empty_list_production
-		
-		# @param [Symbol, Array<Symbol>] sentence Sentence to find the *first set* for.
+		# @param [Symbol, Array<Symbol>]  sentence  Sentence to find the *first set* for.
 		#
-		# @return [Array<Symbol>] The *first set* for the given sentence.
+		# @return [Array<Symbol>]  The *first set* for the given sentence.
 		def first_set(sentence)
 			if sentence.is_a?(Symbol)
 				first_set_prime(sentence)
@@ -217,19 +325,16 @@ module RLTK # :nodoc:
 		end
 		
 		# This function is responsible for calculating the *first* set of
-		# individual symbols.  {CFG#first_set} is a wrapper around this
-		# function to provide support for calculating the *first* set for
-		# sentences.
+		# individual symbols.
 		#
-		# @param [Symbol]		sym0			The symbol to find the *first set* of.
-		# @param [Array<Symbol>]	seen_lh_sides	Previously seen LHS symbols.
+		# @param [Symbol]         sym0           The symbol to find the *first set* of.
+		# @param [Array<Symbol>]  seen_lh_sides  Previously seen LHS symbols.
 		#
 		# @return [Array<Symbol>]
 		def first_set_prime(sym0, seen_lh_sides = [])
 			if self.symbols.include?(sym0)
 				# Memoize the result for later.
 				@firsts[sym0] ||=
-				
 				if CFG::is_terminal?(sym0)
 					# If the symbol is a terminal, it is the only symbol in
 					# its follow set.
@@ -238,7 +343,7 @@ module RLTK # :nodoc:
 					set0 = []
 					
 					@productions_sym[sym0].each do |production|
-						if production.rhs == []
+						if production.rhs.empty?
 							# If this is an empty production we should
 							# add the empty string to the First set.
 							set0 << :'ɛ'
@@ -277,8 +382,8 @@ module RLTK # :nodoc:
 		# used to avoid infinite recursion when mutually recursive rules are
 		# encountered.
 		#
-		# @param [Symbol]		sym0			The symbol to find the *follow set* for.
-		# @param [Array<Symbol>]	seen_lh_sides	Previously seen LHS symbols.
+		# @param [Symbol]         sym0           The symbol to find the *follow set* for.
+		# @param [Array<Symbol>]  seen_lh_sides  Previously seen LHS symbols.
 		#
 		# @return [Array<Symbol>]
 		def follow_set(sym0, seen_lh_sides = [])
@@ -286,7 +391,7 @@ module RLTK # :nodoc:
 			# Use the memoized set if possible.
 			return @follows[sym0] if @follows.has_key?(sym0)
 			
-			if @nonterms[sym0]
+			if @nonterms.member? sym0
 				set0 = []
 				
 				# Add EOS to the start symbol's follow set.
@@ -317,146 +422,14 @@ module RLTK # :nodoc:
 			end
 		end
 		
-		# Builds productions used to eliminate the + EBNF operator.
-		#
-		# @param [Symbol] symbol Symbol to expand.
-		#
-		# @return [Symbol]
-		def get_plus(symbol)
-			new_symbol = (symbol.to_s.downcase + '_plus').to_sym
-			
-			if not @productions_sym.has_key?(new_symbol)
-				# Add the items for the following productions:
-				#
-				# token_plus: token | token token_plus
-				
-				# 1st production
-				production = self.add_production(Production.new(self.next_id, new_symbol, [symbol]))
-				@callback.call(production, :+, :first)
-				
-				# 2nd production
-				production = self.add_production(Production.new(self.next_id, new_symbol, [new_symbol, symbol]))
-				@callback.call(production, :+, :second)
-				
-				# Add the new symbol to the list of nonterminals.
-				@nonterms[new_symbol] = true
-			end
-			
-			return new_symbol
-		end
-		
-		# Builds productions used to eliminate the ? EBNF operator.
-		#
-		# @param [Symbol] symbol Symbol to expand.
-		#
-		# @return [Symbol]
-		def get_question(symbol)
-			new_symbol = (symbol.to_s.downcase + '_question').to_sym
-			
-			if not @productions_sym.has_key?(new_symbol)
-				# Add the items for the following productions:
-				#
-				# nonterm_question: | nonterm
-				
-				# 1st (empty) production.
-				production = self.add_production(Production.new(self.next_id, new_symbol, []))
-				@callback.call(production, :'?', :first)
-				
-				# 2nd production
-				production = self.add_production(Production.new(self.next_id, new_symbol, [symbol]))
-				@callback.call(production, :'?', :second)
-				
-				# Add the new symbol to the list of nonterminals.
-				@nonterms[new_symbol] = true
-			end
-			
-			return new_symbol
-		end
-		
-		# Builds productions used to eliminate the * EBNF operator.
-		#
-		# @param [Symbol] symbol Symbol to expand.
-		#
-		# @return [Symbol]
-		def get_star(symbol)
-			new_symbol = (symbol.to_s.downcase + '_star').to_sym
-			
-			if not @productions_sym.has_key?(new_symbol)
-				# Add the items for the following productions:
-				#
-				# token_star: | token token_star
-				
-				# 1st (empty) production
-				production = self.add_production(Production.new(self.next_id, new_symbol, []))
-				@callback.call(production, :*, :first)
-				
-				# 2nd production
-				production = self.add_production(Production.new(self.next_id, new_symbol, [new_symbol, symbol]))
-				@callback.call(production, :*, :second)
-				
-				# Add the new symbol to the list of nonterminals.
-				@nonterms[new_symbol] = true
-			end
-			
-			return new_symbol
-		end
-		
-		# @return [Integer] ID for the next production to be defined.
+		# @return [Integer]  ID for the next production to be defined.
 		def next_id
 			@production_counter += 1
 		end
 		
-		# This method adds the necessary productions for non-empty lists to
-		# the grammar.  These productions are named `symbol` and
-		# `symbol + '_elements'`
-		#
-		# @param [Symbol]						symbol		The name of the production to add.
-		# @param [String, Symbol, Array<String>]	list_elements	Expression(s) that may appear in the list.
-		# @param [Symbol]						separator		The list separator symbol.
-		#
-		# @return [void]
-		def nonempty_list_production(symbol, list_elements, separator)
-			# Add the items for the following productions:
-			#
-			# symbol_elements: list_elements.join('|')
-			#
-			# symbol: symbol_elements | symbol separator symbol_elements
-			
-			if list_elements.is_a?(String) or list_elements.is_a?(Symbol)
-				list_elements = [list_elements.to_s]
-				
-			elsif list_elements.is_a?(Array)
-				if list_elements.empty?
-					raise ArgumentError, 'Parameter list_elements must not be empty.'
-				else
-					list_elements.map! { |el| el.to_s }
-				end
-				
-			else
-				raise ArgumentError, 'Parameter list_elements must be a String, Symbol, or array of Strings and Symbols.'
-			end
-			
-			el_symbol = (symbol.to_s + '_elements').to_sym
-			
-			# 1st Production
-			production = self.production(symbol, el_symbol.to_s).first
-			@callback.call(production, :nelp, :first)
-			
-			# 2nd Production
-			production = self.production(symbol, "#{symbol} #{separator} #{el_symbol}").first
-			@callback.call(production, :nelp, :second)
-			
-			# 3rd Productions
-			list_elements.each do |el|
-				production = self.production(el_symbol, el).first
-				@callback.call(production, :nelp, :third)
-			end
-		end
-		alias :nonempty_list :nonempty_list_production
-		
-		# @return [Array<Symbol>] All terminal symbols used in the grammar's definition.
+		# @return [Set<Symbol>] All terminal symbols used in the grammar's definition.
 		def nonterms
-			@nonterms.keys
+			@nonterms.clone
 		end
 		
 		# Builds a new production with the left-hand side value of *symbol*.
@@ -464,23 +437,29 @@ module RLTK # :nodoc:
 		# production.  If *expression* is nil then *block* is evaluated, and
 		# expected to make one or more calls to {CFG#clause}.
 		#
-		# @param [Symbol]	symbol		The right-hand side of a production.
-		# @param [String]	expression	The left-hand side of a production.
-		# @param [Proc]	block		Optional block for defining production clauses.
+		# @param [Symbol]          symbol      The left-hand side of a production
+		# @param [String, Symbol]  expression  The right-hand side of a production
+		# @param [Proc]            block       Optional block for defining production clauses
 		#
-		# @return [Array<Production>]
+		# @return [Production, Array<Production>]  A single production if called with an expression;
+		#   an array of productions otherwise
 		def production(symbol, expression = nil, &block)
 			@production_buffer = Array.new
+			
+			prev_lhs  = @curr_lhs
 			@curr_lhs = symbol
 			
+			ret_val =
 			if expression
 				self.clause(expression)
 			else
 				self.instance_exec(&block)
+				
+				@production_buffer.clone
 			end
 			
-			@curr_lhs = nil
-			return @production_buffer.clone
+			@curr_lhs = prev_lhs
+			return ret_val
 		end
 		
 		# If *by* is :sym, returns a hash of the grammar's productions, using
@@ -488,7 +467,7 @@ module RLTK # :nodoc:
 		# an array of productions is returned in the order of their
 		# definition.
 		#
-		# @param [:sym, :id] by The way in which productions should be returned.
+		# @param [:sym, :id]  by  The way in which productions should be returned.
 		#
 		# @return [Array<Production>, Hash{Symbol => Production}]
 		def productions(by = :sym)
@@ -503,7 +482,7 @@ module RLTK # :nodoc:
 		
 		# Sets the start symbol for this grammar.
 		#
-		# @param [Symbol] symbol The new start symbol.
+		# @param [Symbol]  symbol  The new start symbol.
 		#
 		# @return [Symbol]
 		def start(symbol)
@@ -514,34 +493,34 @@ module RLTK # :nodoc:
 			@start_symbol = symbol
 		end
 		
-		# @return [Array<Symbol>] All symbols used in the grammar's definition.
+		# @return [Array<Symbol>]  All symbols used in the grammar's definition.
 		def symbols
 			self.terms + self.nonterms
 		end
 		
-		# @return [Array<Symbol>] All terminal symbols used in the grammar's definition.
+		# @return [Set<Symbol>]  All terminal symbols used in the grammar's definition.
 		def terms
-			@terms.keys
+			@terms.clone
 		end
 		
 		# Oddly enough, the Production class represents a production in a
 		# context-free grammar.
 		class Production
-			# @return [Integer] ID of this production.
+			# @return [Integer]  ID of this production.
 			attr_reader :id
 			
-			# @return [Symbol] Left-hand side of this production.
+			# @return [Symbol]  Left-hand side of this production.
 			attr_reader :lhs
 			
-			# @return [Array<Symbol>] Right-hand side of this production.
+			# @return [Array<Symbol>]  Right-hand side of this production.
 			attr_reader :rhs
 			
 			# Instantiates a new Production object with the specified ID,
 			# and left- and right-hand sides.
 			#
-			# @param [Integer]		id	ID number of this production.
-			# @param [Symbol]		lhs	Left-hand side of the production.
-			# @param [Array<Symbol>]	rhs	Right-hand side of the production.
+			# @param [Integer]        id   ID number of this production.
+			# @param [Symbol]         lhs  Left-hand side of the production.
+			# @param [Array<Symbol>]  rhs  Right-hand side of the production.
 			def initialize(id, lhs, rhs)
 				@id	= id
 				@lhs	= lhs
@@ -551,35 +530,35 @@ module RLTK # :nodoc:
 			# Comparese on production to another.  Returns true only if the
 			# left- and right- hand sides match.
 			#
-			# @param [Production] other Another production to compare to.
+			# @param [Production]  other  Another production to compare to.
 			#
 			# @return [Boolean]
 			def ==(other)
 				self.lhs == other.lhs and self.rhs == other.rhs
 			end
 			
-			# @return [Production] A new copy of this production.
+			# @return [Production]  A new copy of this production.
 			def copy
 				Production.new(@id, @lhs, @rhs.clone)
 			end
 			
-			# @return [Symbol] The last terminal in the right-hand side of the production.
+			# @return [Symbol]  The last terminal in the right-hand side of the production.
 			def last_terminal
 				@rhs.inject(nil) { |m, sym| if CFG::is_terminal?(sym) then sym else m end }
 			end
 			
-			# @return [Item] An Item based on this production.
+			# @return [Item]  An Item based on this production.
 			def to_item
 				Item.new(0, @id, @lhs, @rhs)
 			end
 			
 			# Returns a string representation of this production.
 			#
-			# @param [Integer] padding The ammount of padding spaces to add to the beginning of the string.
+			# @param [Integer]  padding  The ammount of padding spaces to add to the beginning of the string.
 			#
 			# @return [String]
 			def to_s(padding = 0)
-				"#{format("%-#{padding}s", @lhs)} -> #{@rhs.map { |s| s.to_s }.join(' ')}"
+				"#{format("%-#{padding}s", @lhs)} -> #{@rhs.empty? ? 'ɛ' : @rhs.map { |s| s.to_s }.join(' ')}"
 			end
 		end
 		
@@ -593,8 +572,8 @@ module RLTK # :nodoc:
 			# arguments (*args*) should be as specified by
 			# {Production#initialize}.
 			#
-			# @param [Integer] dot Location of the dot in this Item.
-			# @param [Array<Object>] args (see {Production#initialize})
+			# @param [Integer]        dot   Location of the dot in this Item.
+			# @param [Array<Object>]  args  (see {Production#initialize})
 			def initialize(dot, *args)
 				super(*args)
 				
@@ -604,7 +583,7 @@ module RLTK # :nodoc:
 			
 			# Compares two items.
 			#
-			# @param [Item] other Another item to compare to.
+			# @param [Item]  other  Another item to compare to.
 			#
 			# @return [Boolean]
 			def ==(other)
@@ -628,21 +607,21 @@ module RLTK # :nodoc:
 				@dot == @rhs.length
 			end
 			
-			# @return [Item] A new copy of this item.
+			# @return [Item]  A new copy of this item.
 			def copy
 				Item.new(@dot, @id, @lhs, @rhs.clone)
 			end
 			
 			# Returns the symbol located after the dot.
 			#
-			# @return [Symbol] Symbol located after the dot (at the index indicated by the {#dot} attribute).
+			# @return [Symbol]  Symbol located after the dot (at the index indicated by the {#dot} attribute).
 			def next_symbol
 				@rhs[@dot]
 			end
 			
 			# Returns a string representation of this item.
 			#
-			# @param [Integer] padding The ammount of padding spaces to add to the beginning of the string.
+			# @param [Integer]  padding  The ammount of padding spaces to add to the beginning of the string.
 			#
 			# @return [String]
 			def to_s(padding = 0)