eden 0.1.1

data/test/instance_var_token_test.rb

@@ -0,0 +1,48 @@
+require File.dirname(__FILE__) + "/test_helper.rb"
+
+class IdentifierTokenTest < Test::Unit::TestCase
+  def setup
+    @sf = Eden::SourceFile.new( "dummy.rb" )
+  end
+
+  def test_instance_var_tokenization
+    @sf.stubs(:source).returns("@token @_token @token2")
+    @sf.tokenize!
+    line = @sf.lines[0]
+    assert_equal 5, line.tokens.size
+    assert_equal "@token", line.tokens[0].content
+    assert_equal :instancevar, line.tokens[0].type
+    assert_equal "@_token", line.tokens[2].content
+    assert_equal :instancevar, line.tokens[2].type
+    assert_equal "@token2", line.tokens[4].content
+    assert_equal :instancevar, line.tokens[4].type
+  end
+
+  # $' used as a global by Hprictot 0.6.164:lib/elements.rb[274]
+  def test_global_var_tokenization
+    @sf.stubs(:source).returns("$: $? $foo $1 $'")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 9, tokens.size
+    assert_equal "$:", tokens[0].content
+    assert_equal :globalvar, tokens[0].type
+    assert_equal "$?", tokens[2].content
+    assert_equal :globalvar, tokens[2].type
+    assert_equal "$foo", tokens[4].content
+    assert_equal :globalvar, tokens[4].type
+    assert_equal "$1", tokens[6].content
+    assert_equal :globalvar, tokens[6].type
+    assert_equal "$'", tokens[8].content
+    assert_equal :globalvar, tokens[8].type
+  end
+
+  # $` used as a global by Hprictot 0.6.164:lib/builder.rb[199]
+  def test_global_var_tokenization_2
+    @sf.stubs(:source).returns("$`")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 1, tokens.size
+    assert_equal "$`", tokens[0].content
+    assert_equal :globalvar, tokens[0].type
+  end
+end

data/test/number_tokenization_test.rb

@@ -0,0 +1,83 @@
+require File.dirname(__FILE__) + "/test_helper.rb"
+
+class NumberTokenizationTest < Test::Unit::TestCase
+  def setup
+    @sf = Eden::SourceFile.new( "dummy.rb" )
+  end
+
+  def test_binary_tokenization
+    @sf.stubs(:source).returns("b12345 0b01010 0b11056")
+    @sf.tokenize!
+    line = @sf.lines[0]
+    assert_equal 6, line.tokens.size
+    assert_equal "b12345", line.tokens[0].content
+    assert_equal :identifier, line.tokens[0].type
+    assert_equal "0b01010", line.tokens[2].content
+    assert_equal :bin_literal, line.tokens[2].type
+    assert_equal "0b110", line.tokens[4].content
+    assert_equal :bin_literal, line.tokens[4].type
+  end
+
+  def test_hexadecimal_tokenisation
+    @sf.stubs(:source).returns("0x0123 0x0F 0xDEADBEEF 0x0FRR")
+    @sf.tokenize!
+    line = @sf.lines[0]
+    assert_equal 8, line.tokens.size
+    assert_equal "0x0123", line.tokens[0].content
+    assert_equal :hex_literal, line.tokens[0].type
+    assert_equal "0x0F", line.tokens[2].content
+    assert_equal :hex_literal, line.tokens[2].type
+    assert_equal "0xDEADBEEF", line.tokens[4].content
+    assert_equal :hex_literal, line.tokens[4].type
+    assert_equal "0x0F", line.tokens[6].content
+    assert_equal :hex_literal, line.tokens[6].type
+  end
+
+  def test_signed_tokenization
+    @sf.stubs(:source).returns("-10 +4")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 3, tokens.size
+    assert_equal "-10", tokens[0].content
+    assert_equal :dec_literal, tokens[0].type
+    assert_equal "+4", tokens[2].content
+    assert_equal :dec_literal, tokens[2].type
+  end
+
+  def test_decimal_tokenisation
+    @sf.stubs(:source).returns("123 9123 0d1987")
+    @sf.tokenize!
+    line = @sf.lines[0]
+    assert_equal 5, line.tokens.size
+    assert_equal "123", line.tokens[0].content
+    assert_equal :dec_literal, line.tokens[0].type
+    assert_equal "9123", line.tokens[2].content
+    assert_equal :dec_literal, line.tokens[2].type
+    assert_equal "0d1987", line.tokens[4].content
+    assert_equal :dec_literal, line.tokens[4].type
+  end
+
+  def test_float_tokenisation
+    # TODO: Add test that 123.method doens't tokenize as float
+    @sf.stubs(:source).returns("123.0")
+    @sf.tokenize!
+    line = @sf.lines[0]
+    assert_equal 1, line.tokens.size
+    assert_equal "123.0", line.tokens[0].content
+    assert_equal :float_literal, line.tokens[0].type
+  end
+
+
+  def test_exponent_tokenisation
+    @sf.stubs(:source).returns("123e24 1.032e-12 1.32e+12")
+    @sf.tokenize!
+    line = @sf.lines[0]
+    assert_equal 5, line.tokens.size
+    assert_equal "123e24", line.tokens[0].content
+    assert_equal :exp_literal, line.tokens[0].type
+    assert_equal "1.032e-12", line.tokens[2].content
+    assert_equal :exp_literal, line.tokens[2].type
+    assert_equal "1.32e+12", line.tokens[4].content
+    assert_equal :exp_literal, line.tokens[4].type
+  end
+end

data/test/operator_tokenization_test.rb

@@ -0,0 +1,180 @@
+require File.dirname(__FILE__) + "/test_helper.rb"
+
+class OperatorTokenizationTest < Test::Unit::TestCase
+ def setup
+    @sf = Eden::SourceFile.new( "dummy.rb" )
+  end
+
+  def test_equals_tokenization
+    @sf.stubs(:source).returns("= == => === =~")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 9, tokens.size
+    assert_equal :equals, tokens[0].type
+    assert_equal "=", tokens[0].content
+    assert_equal :equality, tokens[2].type
+    assert_equal "==", tokens[2].content
+    assert_equal :hash_rocket, tokens[4].type
+    assert_equal "=>", tokens[4].content
+    assert_equal :identity_equality, tokens[6].type
+    assert_equal "===", tokens[6].content
+    assert_equal :matches, tokens[8].type
+    assert_equal "=~", tokens[8].content
+  end
+
+  def test_plus_tokenization
+    @sf.stubs(:source).returns("+ += +@")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 5, tokens.size
+    assert_equal :plus, tokens[0].type
+    assert_equal "+", tokens[0].content
+    assert_equal :plus_equals, tokens[2].type
+    assert_equal "+=", tokens[2].content
+    assert_equal :plus_at, tokens[4].type
+    assert_equal "+@", tokens[4].content
+  end
+
+  def test_minus_tokenization
+    @sf.stubs(:source).returns("- -= -@")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 5, tokens.size
+    assert_equal :minus, tokens[0].type
+    assert_equal "-", tokens[0].content
+    assert_equal :minus_equals, tokens[2].type
+    assert_equal "-=", tokens[2].content
+    assert_equal :minus_at, tokens[4].type
+    assert_equal "-@", tokens[4].content
+  end
+
+  def test_multiply_tokenization
+    @sf.stubs(:source).returns("* *= ** **=")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 7, tokens.size
+    assert_equal :multiply, tokens[0].type
+    assert_equal "*", tokens[0].content
+    assert_equal :multiply_equals, tokens[2].type
+    assert_equal "*=", tokens[2].content
+    assert_equal :exponent, tokens[4].type
+    assert_equal "**", tokens[4].content
+    assert_equal :exponent_equals, tokens[6].type
+    assert_equal "**=", tokens[6].content
+  end
+
+  def test_divide_tokenization
+    @sf.stubs(:source).returns("rah / gah")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 5, tokens.size
+    assert_equal :divide, tokens[2].type
+    assert_equal "/", tokens[2].content
+  end
+  
+  def test_divide_equals_tokenization
+    @sf.stubs(:source).returns("rah /= gah")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal tokens.size, 5
+    assert_equal :divide_equals, tokens[2].type
+    assert_equal "/=", tokens[2].content
+  end
+
+  def test_lt_tokenization
+    @sf.stubs(:source).returns("< <= << <<= <=>")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 9, tokens.size
+    assert_equal :lt, tokens[0].type
+    assert_equal "<", tokens[0].content
+    assert_equal :lte, tokens[2].type
+    assert_equal "<=", tokens[2].content
+    assert_equal :left_shift, tokens[4].type
+    assert_equal "<<", tokens[4].content
+    assert_equal :left_shift_equals, tokens[6].type
+    assert_equal "<<=", tokens[6].content
+    assert_equal :sort_operator, tokens[8].type
+    assert_equal "<=>", tokens[8].content
+  end
+
+  def test_gt_tokenization
+    @sf.stubs(:source).returns("> >= >> >>=")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 7, tokens.size
+    assert_equal :gt, tokens[0].type
+    assert_equal ">", tokens[0].content
+    assert_equal :gte, tokens[2].type
+    assert_equal ">=", tokens[2].content
+    assert_equal :right_shift, tokens[4].type
+    assert_equal ">>", tokens[4].content
+    assert_equal :right_shift_equals, tokens[6].type
+    assert_equal ">>=", tokens[6].content
+  end
+
+  def test_pipe_tokenization
+    @sf.stubs(:source).returns("| |= || ||=")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 7, tokens.size
+    assert_equal :bitwise_or, tokens[0].type
+    assert_equal "|", tokens[0].content
+    assert_equal :bitwise_or_equals, tokens[2].type
+    assert_equal "|=", tokens[2].content
+    assert_equal :logical_or, tokens[4].type
+    assert_equal "||", tokens[4].content
+    assert_equal :logical_or_equals, tokens[6].type
+    assert_equal "||=", tokens[6].content
+  end
+
+  def test_pipe_tokenization
+    @sf.stubs(:source).returns("& &= && &&=")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 7, tokens.size
+    assert_equal :bitwise_and, tokens[0].type
+    assert_equal "&", tokens[0].content
+    assert_equal :bitwise_and_equals, tokens[2].type
+    assert_equal "&=", tokens[2].content
+    assert_equal :logical_and, tokens[4].type
+    assert_equal "&&", tokens[4].content
+    assert_equal :logical_and_equals, tokens[6].type
+    assert_equal "&&=", tokens[6].content
+  end
+
+  def test_caret_tokenization
+    @sf.stubs(:source).returns("^ ^=")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 3, tokens.size
+    assert_equal :caret, tokens[0].type
+    assert_equal "^", tokens[0].content
+    assert_equal :caret_equals, tokens[2].type
+    assert_equal "^=", tokens[2].content
+  end
+
+  def test_modulo_tokenization
+    @sf.stubs(:source).returns("% a %=")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 5, tokens.size
+    assert_equal :modulo, tokens[0].type
+    assert_equal "%", tokens[0].content
+    assert_equal :modulo_equals, tokens[4].type
+    assert_equal "%=", tokens[4].content
+  end
+
+  def test_bang_tokenization
+    @sf.stubs(:source).returns("!= !~ !")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 5, tokens.size
+    assert_equal :not_equals, tokens[0].type
+    assert_equal "!=", tokens[0].content
+    assert_equal :not_matches, tokens[2].type
+    assert_equal "!~", tokens[2].content
+    assert_equal :logical_not, tokens[4].type
+    assert_equal "!", tokens[4].content
+  end
+end

data/test/regex_tokenization_test.rb

@@ -0,0 +1,68 @@
+require File.dirname(__FILE__) + "/test_helper.rb"
+
+class RegexTokenizationTest < Test::Unit::TestCase
+  def setup
+    @sf = Eden::SourceFile.new( "dummy.rb" )
+  end
+
+  def test_delimited_regex_tokenization
+    @sf.stubs(:source).returns("%r{[a-z]}")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 1, tokens.size
+    assert_equal :regex, tokens[0].type
+    assert_equal "%r{[a-z]}", tokens[0].content
+  end
+
+  def test_delimited_regex_tokenization2
+    @sf.stubs(:source).returns("%r{[a-z]}i")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 1, tokens.size
+    assert_equal :regex, tokens[0].type
+    assert_equal "%r{[a-z]}i", tokens[0].content
+  end
+
+  def test_regex_tokenization_at_line_start
+    @sf.stubs(:source).returns("/[asdf]/")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 1, tokens.size
+    assert_equal "/[asdf]/", tokens[0].content
+    assert_equal :regex, tokens[0].type
+  end
+
+  def test_ambiguous_regex_tokenization
+    @sf.stubs(:source).returns("a = b / c\nd = f / g")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens + @sf.lines[1].tokens
+    tokens.each {|t| assert t.type != :regex }
+  end
+
+  def test_regex_tokenization_with_escape_characters
+    @sf.stubs(:source).returns("/test\\/test/")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 1, tokens.size
+    assert_equal "/test\\/test/", tokens[0].content
+    assert_equal :regex, tokens[0].type
+  end
+
+  def test_regex_tokenization_with_modifiers
+    @sf.stubs(:source).returns("/test/i")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 1, tokens.size
+    assert_equal "/test/i", tokens[0].content
+    assert_equal :regex, tokens[0].type
+  end
+
+  def test_regex_tokenization_with_funky_regexes
+    @sf.stubs(:source).returns(%q{%r!\[ *(?:(@)([\w\(\)-]+)|([\w\(\)-]+\(\))) *([~\!\|\*$\^=]*) *'?"?([^'"]*)'?"? *\]!i})
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 1, tokens.size
+    assert_equal :regex, tokens[0].type
+    assert_equal %q{%r!\[ *(?:(@)([\w\(\)-]+)|([\w\(\)-]+\(\))) *([~\!\|\*$\^=]*) *'?"?([^'"]*)'?"? *\]!i}, tokens[0].content
+  end
+end

data/test/single_character_tokenization_test.rb

@@ -0,0 +1,87 @@
+require File.dirname(__FILE__) + "/test_helper.rb"
+
+class SingleCharacterTokenizationTest < Test::Unit::TestCase
+  def setup
+    @sf = Eden::SourceFile.new( "dummy.rb" )
+  end
+
+  def test_single_character_tokenisation
+    @sf.stubs(:source).returns("<>~!@% ^&*()[]{}|.: ;=? +-")
+    @sf.tokenize!
+    line = @sf.lines[0]
+    assert_equal 26, line.tokens.size
+    assert_equal "<", line.tokens[0].content
+    assert_equal :lt, line.tokens[0].type
+    assert_equal ">", line.tokens[1].content
+    assert_equal :gt, line.tokens[1].type
+    assert_equal "~", line.tokens[2].content
+    assert_equal :tilde, line.tokens[2].type
+    assert_equal "!", line.tokens[3].content
+    assert_equal :logical_not, line.tokens[3].type
+    assert_equal "@", line.tokens[4].content
+    assert_equal :at, line.tokens[4].type
+    assert_equal "%", line.tokens[5].content
+    assert_equal :modulo, line.tokens[5].type
+    assert_equal "^", line.tokens[7].content
+    assert_equal :caret, line.tokens[7].type
+    assert_equal "&", line.tokens[8].content
+    assert_equal :bitwise_and, line.tokens[8].type
+    assert_equal "*", line.tokens[9].content
+    assert_equal :multiply, line.tokens[9].type
+    assert_equal "(", line.tokens[10].content
+    assert_equal :lparen, line.tokens[10].type
+    assert_equal ")", line.tokens[11].content
+    assert_equal :rparen, line.tokens[11].type
+    assert_equal "[", line.tokens[12].content
+    assert_equal :lsquare, line.tokens[12].type
+    assert_equal "]", line.tokens[13].content
+    assert_equal :rsquare, line.tokens[13].type
+    assert_equal "{", line.tokens[14].content
+    assert_equal :lcurly, line.tokens[14].type
+    assert_equal "}", line.tokens[15].content
+    assert_equal :rcurly, line.tokens[15].type
+    assert_equal "|", line.tokens[16].content
+    assert_equal :bitwise_or, line.tokens[16].type
+    assert_equal ".", line.tokens[17].content
+    assert_equal :period, line.tokens[17].type
+    assert_equal ":", line.tokens[18].content
+    assert_equal :colon, line.tokens[18].type
+    assert_equal ";", line.tokens[20].content
+    assert_equal :semicolon, line.tokens[20].type
+    assert_equal "=", line.tokens[21].content
+    assert_equal :equals, line.tokens[21].type
+    assert_equal "?", line.tokens[22].content
+    assert_equal :question_mark, line.tokens[22].type    
+    assert_equal "+", line.tokens[24].content
+    assert_equal :plus, line.tokens[24].type    
+    assert_equal "-", line.tokens[25].content
+    assert_equal :minus, line.tokens[25].type    
+  end
+
+  def test_period_tokenization
+    @sf.stubs(:source).returns("... .. .")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 5, tokens.size
+    assert_equal "...", tokens[0].content
+    assert_equal :range_inc, tokens[0].type    
+    assert_equal "..", tokens[2].content
+    assert_equal :range_exc, tokens[2].type    
+    assert_equal ".", tokens[4].content
+    assert_equal :period, tokens[4].type    
+  end
+
+  def test_colon_tokenization
+   @sf.stubs(:source).returns(":test :: : ")
+    @sf.tokenize!
+    tokens = @sf.lines[0].tokens
+    assert_equal 6, tokens.size
+    assert_equal ":test", tokens[0].content
+    assert_equal :symbol, tokens[0].type    
+    assert_equal "::", tokens[2].content
+    assert_equal :scope_res, tokens[2].type    
+    assert_equal ":", tokens[4].content
+    assert_equal :colon, tokens[4].type    
+  end
+
+end