rltk 1.1.0

data/lib/rltk/parsers/infix_calc.rb

@@ -0,0 +1,43 @@
+# Author:		Chris Wailes <chris.wailes@gmail.com>
+# Project: 	Ruby Language Toolkit
+# Date:		2011/03/04
+# Description:	This file contains a parser for a simple infix calculator.
+
+############
+# Requires #
+############
+
+# Ruby Language Toolkit
+require 'rltk/parser'
+
+#######################
+# Classes and Modules #
+#######################
+
+module RLTK # :nodoc:
+	
+	# The RLTK::Parsers module contains the parsers that are included as part
+	# of the RLKT project.
+	module Parsers
+		
+		# A parser for a simple infix calculator.
+		class InfixCalc < Parser
+			
+			left :PLS, :SUB
+			right :MUL, :DIV
+			
+			production(:e) do
+				clause('NUM') { |n| n }
+				
+				clause('LPAREN e RPAREN') { |_, e, _| e }
+				
+				clause('e PLS e') { |e0, _, e1| e0 + e1 }
+				clause('e SUB e') { |e0, _, e1| e0 - e1 }
+				clause('e MUL e') { |e0, _, e1| e0 * e1 }
+				clause('e DIV e') { |e0, _, e1| e0 / e1 }
+			end
+			
+			finalize
+		end
+	end
+end

data/lib/rltk/parsers/postfix_calc.rb

@@ -0,0 +1,34 @@
+# Author:		Chris Wailes <chris.wailes@gmail.com>
+# Project: 	Ruby Language Toolkit
+# Date:		2011/04/06
+# Description:	This file contains a parser for a simple postfix calculator.
+
+############
+# Requires #
+############
+
+# Ruby Language Toolkit
+require 'rltk/parser'
+
+#######################
+# Classes and Modules #
+#######################
+
+module RLTK # :nodoc:
+	module Parsers # :nodoc:
+		
+		# A parser for a simple post-fix calculator.
+		class PostfixCalc < Parser
+			production(:e) do
+				clause('NUM') { |n| n }
+				
+				clause('e e PLS') { |e0, e1, _| e0 + e1 }
+				clause('e e SUB') { |e0, e1, _| e0 - e1 }
+				clause('e e MUL') { |e0, e1, _| e0 * e1 }
+				clause('e e DIV') { |e0, e1, _| e0 / e1 }
+			end
+			
+			finalize
+		end
+	end
+end

data/lib/rltk/parsers/prefix_calc.rb

@@ -0,0 +1,34 @@
+# Author:		Chris Wailes <chris.wailes@gmail.com>
+# Project: 	Ruby Language Toolkit
+# Date:		2011/04/06
+# Description:	This file contains a parser for a simple prefix calculator.
+
+############
+# Requires #
+############
+
+# Ruby Language Toolkit
+require 'rltk/parser'
+
+#######################
+# Classes and Modules #
+#######################
+
+module RLTK # :nodoc:
+	module Parsers # :nodoc:
+		
+		# A parser for a simple prefix calculator.
+		class PrefixCalc < Parser
+			production(:e) do
+				clause('NUM') { |n| n }
+				
+				clause('PLS e e') { |_, e0, e1| e0 + e1 }
+				clause('SUB e e') { |_, e0, e1| e0 - e1 }
+				clause('MUL e e') { |_, e0, e1| e0 * e1 }
+				clause('DIV e e') { |_, e0, e1| e0 / e1 }
+			end
+			
+			finalize
+		end
+	end
+end

data/lib/rltk/token.rb

@@ -0,0 +1,66 @@
+# Author:		Chris Wailes <chris.wailes@gmail.com>
+# Project: 	Ruby Language Toolkit
+# Date:		2011/01/17
+# Description:	This file contains code having to do with tokens.
+
+#######################
+# Classes and Modules #
+#######################
+
+module RLTK # :nodoc:
+	
+	# The StreamPosition class is used to indicate the position of a token or
+	# other text inside a stream.
+	class StreamPosition
+		attr_accessor :stream_offset
+		attr_accessor :line_number
+		attr_accessor :line_offset
+		attr_accessor :length
+		
+		attr_accessor :file_name
+		
+		alias :start :line_offset
+		
+		# Instantiates a new StreamPosition object with the values specified.
+		def initialize(stream_offset = 0, line_number = 0, line_offset = 0, length = 0, file_name = nil)
+			@stream_offset	= stream_offset
+			@line_number	= line_number
+			@line_offset	= line_offset
+			@length		= length
+			@file_name	= file_name
+		end
+	end
+	
+	# The Token class is used to represent the output of a RLTK::Lexer and the
+	# input of a RLTK::Parser.
+	class Token
+		attr_reader :type
+		attr_reader :value
+		
+		# The StreamPosition object associated with this token.
+		attr_reader :position
+		
+		# Instantiates a new Token object with the values specified.
+		def initialize(type, value = nil, position = nil)
+			@type	= type
+			@value	= value
+			
+			@position	= position
+		end
+		
+		# Compares one token to another.  This only tests the token's _type_
+		# and _value_ and not the location of the token in its source.
+		def ==(other)
+			self.type == other.type and self.value == other.value
+		end
+		
+		# Returns a string representing the tokens _type_ and _value_.
+		def to_s
+			if value
+				"#{self.type}(#{self.value})"
+			else
+				self.type.to_s
+			end
+		end
+	end
+end

data/test/tc_ast.rb

@@ -0,0 +1,85 @@
+# Author:		Chris Wailes <chris.wailes@gmail.com>
+# Project: 	Ruby Language Toolkit
+# Date:		2011/04/06
+# Description:	This file contains unit tests for the RLTK::ASTNode class.
+
+############
+# Requires #
+############
+
+# Standard Library
+require 'test/unit'
+require 'pp'
+
+# Ruby Language Toolkit
+require 'rltk/ast'
+
+#######################
+# Classes and Modules #
+#######################
+
+class ANode < RLTK::ASTNode
+	child :left, ANode
+	child :right, ANode
+end
+
+class BNode < ANode; end
+class CNode < ANode; end
+
+class DNode < RLTK::ASTNode; end
+
+class ASTNodeTester < Test::Unit::TestCase
+	def setup
+		@leaf0 = CNode.new(nil, nil)
+		@tree0 = ANode.new(BNode.new(@leaf0, nil), BNode.new(nil, nil))
+		
+		@tree1 = ANode.new(BNode.new(CNode.new(nil, nil), nil), BNode.new(nil, nil))
+		@tree2 = ANode.new(BNode.new(nil, nil), BNode.new(CNode.new(nil, nil), nil))
+	end
+	
+	def test_children
+		node = ANode.new(nil, nil)
+		
+		assert_equal(node.children, [nil, nil])
+		
+		node.children = (expected_children = [BNode.new(nil, nil), CNode.new(nil, nil)])
+		
+		assert_equal(node.children, expected_children)
+		
+		node.map do |child|
+			if child.is_a?(BNode)
+				CNode.new(nil, nil)
+			else
+				BNode.new(nil, nil)
+			end
+		end
+		
+		assert_equal(node.children, expected_children.reverse)
+	end
+	
+	def test_equal
+		assert_equal(@tree0, @tree1)
+		assert_not_equal(@tree0, @tree2)
+	end
+	
+	def test_initialize
+		assert_raise(Exception) { RLTK::ASTNode.new }
+		assert_nothing_raised(Exception) { ANode.new(nil, nil) }
+	end
+	
+	def test_notes
+		node = ANode.new(nil, nil)
+		
+		assert_nil(node[:a])
+		assert_equal(node[:a] = :b, :b)
+		assert_equal(node.note?(:a), true)
+		assert_equal(node.note?(:b), false)
+		assert_equal(node.delete_note(:a), :b)
+		assert_nil(node[:a])
+	end
+	
+	def test_root
+		assert_same(@tree0, @tree0.root)
+		assert_same(@tree0, @leaf0.root)
+	end
+end

data/test/tc_cfg.rb

@@ -0,0 +1,149 @@
+# Author:		Chris Wailes <chris.wailes@gmail.com>
+# Project: 	Ruby Language Toolkit
+# Date:		2011/04/06
+# Description:	This file contains unit tests for the RLTK::CFG class.
+
+############
+# Requires #
+############
+
+# Standard Library
+require 'test/unit'
+
+# Ruby Language Toolkit
+require 'rltk/cfg'
+
+#######################
+# Classes and Modules #
+#######################
+
+class CFGTester < Test::Unit::TestCase
+	def setup
+		@grammar = RLTK::CFG.new
+		
+		@grammar.production(:s) do
+			clause('A G D')
+			clause('A a C')
+			clause('B a D')
+			clause('B G C')
+		end
+		
+		@grammar.production(:a, 'b')
+		@grammar.production(:b, 'G')
+	end
+	
+	def test_callback
+		grammar = RLTK::CFG.new
+		
+		first = true
+		grammar.callback do |p, type, num|
+			assert_not_nil(p)
+			assert_equal(type, :'?')
+			
+			if first
+				assert_equal(num, :first)
+				first = false
+			else
+				assert_equal(num, :second)
+			end
+		end
+		
+		grammar.production(:a, 'A?') { |a| a }
+		
+		first = true
+		grammar.callback do |p, type, num|
+			assert_not_nil(p)
+			assert_equal(type, :*)
+			
+			if first
+				assert_equal(num, :first)
+				first = false
+			else
+				assert_equal(num, :second)
+			end
+		end
+		
+		grammar.production(:a, 'A*') { |a| a }
+		
+		first = true
+		grammar.callback do |p, type, num|
+			assert_not_nil(p)
+			assert_equal(type, :+)
+			
+			if first
+				assert_equal(num, :first)
+				first = false
+			else
+				assert_equal(num, :second)
+			end
+		end
+		
+		grammar.production(:a, 'A+') { |a| a }
+	end
+	
+	def test_first_set
+		@grammar.first_set(:s).each do |sym|
+			assert([:A, :B].include?(sym))
+		end
+		
+		assert_equal(@grammar.first_set(:b), [:G])
+		assert_equal(@grammar.first_set(:a), [:G])
+	end
+	
+	def test_follow_set
+		assert_equal(@grammar.follow_set(:s), [:EOS])
+		
+		@grammar.follow_set(:a).each do |sym|
+			assert([:C, :D].include?(sym))
+		end
+		
+		@grammar.follow_set(:b).each do |sym|
+			assert([:C, :D].include?(sym))
+		end
+	end
+	
+	def test_is_nonterminal
+		assert_equal(RLTK::CFG::is_nonterminal?(:lowercase), true)
+		assert_equal(RLTK::CFG::is_nonterminal?(:UPERCASE), false)
+	end
+	
+	def test_is_terminal
+		assert_equal(RLTK::CFG::is_terminal?(:lowercase), false)
+		assert_equal(RLTK::CFG::is_terminal?(:UPERCASE), true)
+	end
+	
+	def test_item
+		i0 = RLTK::CFG::Item.new(0, 0, :a, [:b, :C, :D, :e])
+		i1 = i0.copy
+		
+		assert_equal(i0, i1)
+		assert_equal(i0.at_end?, false)
+		assert_equal(i0.next_symbol, :b)
+		
+		i0.advance
+		
+		assert_not_equal(i0, i1)
+		assert_equal(i0.at_end?, false)
+		assert_equal(i0.next_symbol, :C)
+		
+		i0.advance
+		assert_equal(i0.at_end?, false)
+		assert_equal(i0.next_symbol, :D)
+		
+		i0.advance
+		assert_equal(i0.at_end?, false)
+		assert_equal(i0.next_symbol, :e)
+		
+		i0.advance
+		assert_equal(i0.at_end?, true)
+		assert_nil(i0.next_symbol)
+	end
+	
+	def test_production
+		p0 = RLTK::CFG::Production.new(0, :a, [:b, :C, :D, :e])
+		p1 = p0.copy
+		
+		assert_equal(p0, p1)
+		assert_equal(p0.last_terminal, :D)
+	end
+end

data/test/tc_lexer.rb

@@ -0,0 +1,217 @@
+# Author:		Chris Wailes <chris.wailes@gmail.com>
+# Project: 	Ruby Language Toolkit
+# Date:		2011/04/06
+# Description:	This file contains unit tests for the RLTK::Lexer class.
+
+############
+# Requires #
+############
+
+# Standard Library
+require 'test/unit'
+
+# Ruby Language Toolkit
+require 'rltk/token'
+require 'rltk/lexer'
+require 'rltk/lexers/calculator'
+require 'rltk/lexers/ebnf'
+
+#######################
+# Classes and Modules #
+#######################
+
+class ABLongest < RLTK::Lexer
+	rule(/a+/)	{ :APLUS }
+	rule(/b+/)	{ :BPLUS }
+	
+	rule(/a+b+/)	{ :APLUSBPLUS }
+end
+
+class ABFirst < RLTK::Lexer
+	match_first
+	
+	rule(/a+/)	{ :APLUS }
+	rule(/b+/)	{ :BPLUS }
+	
+	rule(/a+b+/)	{ :APLUSBPLUS }
+end
+
+class ENVLexer < RLTK::Lexer
+	rule(/a/)	{ [:A, next_value] }
+	
+	class Environment < Environment
+		def initialize(*args)
+			super(*args)
+			@value = -1
+		end
+		
+		def next_value
+			@value += 1
+		end
+	end
+end
+
+class FlagLexer < RLTK::Lexer
+	rule(/a/)		{ set_flag(:a); :A }
+	rule(/\s/)
+	
+	rule(/b/, :default, [:a])	{ set_flag(:b); :B }
+	rule(/c/, :default, [:a, :b])	{ :C }
+end
+
+class StateLexer < RLTK::Lexer
+	rule(/a/)		{ :A }
+	rule(/\s/)
+	
+	rule(/\(\*/)	{ push_state(:comment) }
+	
+	rule(/\(\*/, :comment)	{ push_state(:comment) }
+	rule(/\*\)/, :comment)	{ pop_state }
+	rule(/./,    :comment)
+end
+
+class LexerTester < Test::Unit::TestCase
+	def test_calc
+		expected =
+			[
+				RLTK::Token.new(:NUM, 1),
+				
+				RLTK::Token.new(:PLS),
+				RLTK::Token.new(:SUB),
+				RLTK::Token.new(:MUL),
+				RLTK::Token.new(:DIV),
+				
+				RLTK::Token.new(:LPAREN),
+				RLTK::Token.new(:RPAREN),
+				RLTK::Token.new(:EOS)
+			]
+		
+		actual = RLTK::Lexers::Calculator.lex('1 + - * / ( )')
+		
+		assert_equal(expected, actual)
+	end
+	
+	def test_ebnf
+		expected =
+			[
+				RLTK::Token.new(:NONTERM, :aaa),
+				RLTK::Token.new(:TERM, :BBB),
+				
+				RLTK::Token.new(:*),
+				RLTK::Token.new(:+),
+				RLTK::Token.new(:'?'),
+				RLTK::Token.new(:EOS)
+			]
+		
+		actual = RLTK::Lexers::EBNF.lex('aaa BBB * + ?')
+		
+		assert_equal(expected, actual)
+	end
+	
+	def test_environment
+		expected =
+			[
+				RLTK::Token.new(:A, 0),
+				RLTK::Token.new(:A, 1),
+				RLTK::Token.new(:A, 2),
+				RLTK::Token.new(:EOS)
+			]
+		
+		actual = ENVLexer.lex('aaa')
+		
+		assert_equal(expected, actual)
+		
+		lexer = ENVLexer.new
+		
+		assert_equal(expected, lexer.lex('aaa'))
+		
+		expected =
+			[
+				RLTK::Token.new(:A, 3),
+				RLTK::Token.new(:A, 4),
+				RLTK::Token.new(:A, 5),
+				RLTK::Token.new(:EOS)
+			]
+		
+		assert_equal(expected, lexer.lex('aaa'))
+	end
+	
+	def test_first_match
+		expected =
+			[
+				RLTK::Token.new(:APLUS),
+				RLTK::Token.new(:BPLUS),
+				RLTK::Token.new(:EOS)
+			]
+		
+		actual = ABFirst.lex('aaabbb')
+		
+		assert_equal(expected, actual)
+	end
+	
+	def test_flags
+		
+		assert_raise(RLTK::LexingError) { FlagLexer.lex('b') }
+		assert_raise(RLTK::LexingError) { FlagLexer.lex('ac') }
+		
+		expected =
+			[
+				RLTK::Token.new(:A),
+				RLTK::Token.new(:B),
+				RLTK::Token.new(:C),
+				RLTK::Token.new(:EOS)
+			]
+		
+		actual = FlagLexer.lex('abc')
+		assert_equal(expected, actual)
+		
+		expected =
+			[
+				RLTK::Token.new(:A),
+				RLTK::Token.new(:B),
+				RLTK::Token.new(:C),
+				RLTK::Token.new(:A),
+				RLTK::Token.new(:B),
+				RLTK::Token.new(:C),
+				RLTK::Token.new(:EOS)
+			]
+		
+		actual = FlagLexer.lex('abcabc')
+		assert_equal(expected, actual)
+	end
+	
+	def test_lex
+		assert_raise(RLTK::LexingError) { ABFirst.lex('aaabbbCCC') }
+		assert_raise(RLTK::LexingError) { ABLongest.lex('aaabbbCCC') }
+		
+		assert_nothing_raised(RLTK::LexingError) { ABFirst.lex('aaabbb') }
+		assert_nothing_raised(RLTK::LexingError) { ABLongest.lex('aaabbb') }
+	end
+	
+	def test_longest_match
+		expected =
+			[
+				RLTK::Token.new(:APLUSBPLUS),
+				RLTK::Token.new(:EOS)
+			]
+		
+		actual = ABLongest.lex('aaabbb')
+		
+		assert_equal(expected, actual)
+	end
+	
+	def test_state
+		expected =
+			[
+				RLTK::Token.new(:A),
+				RLTK::Token.new(:A),
+				RLTK::Token.new(:EOS)
+			]
+		
+		actual = StateLexer.lex('a (* bbb *) a')
+		assert_equal(expected, actual)
+		
+		actual = StateLexer.lex('a (* b (* ccc *) b *) a')
+		assert_equal(expected, actual)
+	end
+end