parser 1.4.2 → 2.0.0.beta1

data/test/test_lexer.rb

@@ -6,6 +6,7 @@ class TestLexer < MiniTest::Unit::TestCase
   def setup_lexer(version)
     @lex = Parser::Lexer.new(version)
 
+    @lex.comments = []
     @lex.diagnostics = Parser::Diagnostic::Engine.new
     @lex.diagnostics.all_errors_are_fatal = true
     # @lex.diagnostics.consumer = lambda { |diag| $stderr.puts "", diag.render }
@@ -376,7 +377,10 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tINTEGER, 1,
                    :tNL, nil,
                    :tINTEGER, 2)
-    assert_equal "# one\n# two\n", @lex.comments
+
+    assert_equal 2, @lex.comments.length
+    assert_equal '# one', @lex.comments[0].text
+    assert_equal '# two', @lex.comments[1].text
   end
 
   def test_comment_expr_beg
@@ -388,7 +392,8 @@ class TestLexer < MiniTest::Unit::TestCase
   def test_comment_begin
     util_lex_token("=begin\nblah\nblah\n=end\n42",
                    :tINTEGER, 42)
-    assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
+    assert_equal 1, @lex.comments.length
+    assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments[0].text
   end
 
   def test_comment_begin_bad
@@ -409,12 +414,16 @@ class TestLexer < MiniTest::Unit::TestCase
 
   def test_comment_begin_space
     util_lex_token("=begin blah\nblah\n=end\n")
-    assert_equal "=begin blah\nblah\n=end\n", @lex.comments
+
+    assert_equal 1, @lex.comments.length
+    assert_equal "=begin blah\nblah\n=end\n", @lex.comments[0].text
   end
 
   def test_comment_end_space_and_text
     util_lex_token("=begin blah\nblah\n=end blab\n")
-    assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
+
+    assert_equal 1, @lex.comments.length
+    assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments[0].text
   end
 
   def test_comment_eos
@@ -481,16 +490,15 @@ class TestLexer < MiniTest::Unit::TestCase
                    :kEND, "end")
   end
 
-  # TODO
-  # def test_do_cond
-  #   @lex.cond.push true
+  def test_do_cond
+    @lex.cond.push true
 
-  #   util_lex_token("x do 42 end",
-  #                  :tIDENTIFIER, "x",
-  #                  :kDO_COND, "do",
-  #                  :tINTEGER, 42,
-  #                  :kEND, "end")
-  # end
+    util_lex_token("x do 42 end",
+                   :tIDENTIFIER, "x",
+                   :kDO_COND, "do",
+                   :tINTEGER, 42,
+                   :kEND, "end")
+  end
 
   def test_dot
     util_lex_token ".", :tDOT, "."
@@ -1219,7 +1227,7 @@ class TestLexer < MiniTest::Unit::TestCase
   end
 
   def test_plus
-    util_lex_token("1 + 1", # TODO state?
+    util_lex_token("1 + 1",
                    :tINTEGER, 1,
                    :tPLUS, "+",
                    :tINTEGER, 1)
@@ -2103,6 +2111,239 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tRBRACK,     "]")
   end
 
+  #
+  # Tests for whitespace.
+  #
+
+  def test_whitespace_fname
+    @lex.state = :expr_fname
+    util_lex_token('class',
+                   :kCLASS, 'class')
+
+    @lex.state = :expr_fname
+    util_lex_token(' class',
+                   :kCLASS, 'class')
+
+    @lex.state = :expr_fname
+    util_lex_token("\nclass",
+                   :kCLASS, 'class')
+
+    @lex.state = :expr_fname
+    util_lex_token("\\\nclass",
+                   :kCLASS, 'class')
+
+    @lex.state = :expr_fname
+    util_lex_token("#foo\nclass",
+                   :kCLASS, 'class')
+  end
+
+  def test_whitespace_endfn
+    setup_lexer(21)
+
+    @lex.state = :expr_endfn
+    util_lex_token('foo:',
+                   :tLABEL, 'foo')
+
+    @lex.state = :expr_endfn
+    util_lex_token(' foo:',
+                   :tLABEL, 'foo')
+
+    @lex.state = :expr_endfn
+    util_lex_token("\nfoo:",
+                   :tNL,         nil,
+                   :tIDENTIFIER, 'foo',
+                   :tCOLON,      ':')
+
+    @lex.state = :expr_endfn
+    util_lex_token("\\\nfoo:",
+                   :tLABEL, 'foo')
+
+    @lex.state = :expr_endfn
+    util_lex_token("#foo\nfoo:",
+                   :tNL,         nil,
+                   :tIDENTIFIER, 'foo',
+                   :tCOLON,      ':')
+  end
+
+  def test_whitespace_dot
+    @lex.state = :expr_dot
+    util_lex_token('class',
+                   :tIDENTIFIER, 'class')
+
+    @lex.state = :expr_dot
+    util_lex_token(' class',
+                   :tIDENTIFIER, 'class')
+
+    @lex.state = :expr_dot
+    util_lex_token("\nclass",
+                   :tIDENTIFIER, 'class')
+
+    @lex.state = :expr_dot
+    util_lex_token("\\\nclass",
+                   :tIDENTIFIER, 'class')
+
+    @lex.state = :expr_dot
+    util_lex_token("#foo\nclass",
+                   :tIDENTIFIER, 'class')
+  end
+
+  def test_whitespace_arg
+    @lex.state = :expr_arg
+    util_lex_token('+',
+                   :tPLUS,  '+')
+
+    @lex.state = :expr_arg
+    util_lex_token(' +',
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_arg
+    util_lex_token("\n+",
+                   :tNL,    nil,
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_arg
+    util_lex_token("\\\n+",
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_arg
+    util_lex_token("\\\n +",
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_arg
+    util_lex_token("#foo\n+",
+                   :tNL,    nil,
+                   :tUPLUS, '+')
+  end
+
+  def test_whitespace_endarg
+    @lex.state = :expr_endarg
+    util_lex_token('{',
+                   :tLBRACE_ARG, '{')
+
+    @lex.state = :expr_endarg
+    util_lex_token(' {',
+                   :tLBRACE_ARG, '{')
+
+    @lex.state = :expr_endarg
+    util_lex_token("\n{",
+                   :tNL,         nil,
+                   :tLBRACE,     '{')
+
+    @lex.state = :expr_endarg
+    util_lex_token("\\\n{",
+                   :tLBRACE_ARG, '{')
+
+    @lex.state = :expr_endarg
+    util_lex_token("#foo\n{",
+                   :tNL,         nil,
+                   :tLBRACE,     '{')
+  end
+
+  def test_whitespace_mid
+    @lex.state = :expr_mid
+    util_lex_token('+',
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_mid
+    util_lex_token(' +',
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_mid
+    util_lex_token("\n+",
+                   :tNL,    nil,
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_mid
+    util_lex_token("\\\n+",
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_mid
+    util_lex_token("#foo\n+",
+                   :tNL,    nil,
+                   :tUPLUS, '+')
+  end
+
+  def test_whitespace_beg
+    @lex.state = :expr_beg
+    util_lex_token('+',
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_beg
+    util_lex_token(' +',
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_beg
+    util_lex_token("\n+",
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_beg
+    util_lex_token("\\\n+",
+                   :tUPLUS, '+')
+
+    @lex.state = :expr_beg
+    util_lex_token("#foo\n+",
+                   :tUPLUS, '+')
+  end
+
+  def test_whitespace_value
+    setup_lexer(20)
+
+    @lex.state = :expr_value
+    util_lex_token('a:b',
+                   :tIDENTIFIER, 'a',
+                   :tSYMBOL,     'b')
+
+    @lex.state = :expr_value
+    util_lex_token(' a:b',
+                   :tIDENTIFIER, 'a',
+                   :tSYMBOL,     'b')
+
+    @lex.state = :expr_value
+    util_lex_token("\na:b",
+                   :tLABEL,      'a',
+                   :tIDENTIFIER, 'b')
+
+    @lex.state = :expr_value
+    util_lex_token("\\\na:b",
+                   :tIDENTIFIER, 'a',
+                   :tSYMBOL,     'b')
+
+    @lex.state = :expr_value
+    util_lex_token("#foo\na:b",
+                   :tLABEL,      'a',
+                   :tIDENTIFIER, 'b')
+  end
+
+  def test_whitespace_end
+    @lex.state = :expr_end
+    util_lex_token('+ 1',
+                   :tPLUS,    '+',
+                   :tINTEGER, 1)
+
+    @lex.state = :expr_end
+    util_lex_token(' + 1',
+                   :tPLUS,    '+',
+                   :tINTEGER, 1)
+
+    @lex.state = :expr_end
+    util_lex_token("\n+ 1",
+                   :tNL,      nil,
+                   :tUPLUS,   '+',
+                   :tINTEGER, 1)
+
+    @lex.state = :expr_end
+    util_lex_token("\\\n+ 1",
+                   :tPLUS,    '+',
+                   :tINTEGER, 1)
+
+    @lex.state = :expr_end
+    util_lex_token("#foo\n+ 1",
+                   :tNL,      nil,
+                   :tUPLUS,   '+',
+                   :tINTEGER, 1)
+  end
+
+
   #
   # Tests for bugs.
   #
@@ -2307,7 +2548,7 @@ class TestLexer < MiniTest::Unit::TestCase
   def test_bug_expr_value_document
     util_lex_token("1;\n=begin\n=end",
                    :tINTEGER, 1,
-                   :tSEMI,    ";")
+                   :tSEMI,    ';')
   end
 
   def test_bug_expr_end_colon
@@ -2335,6 +2576,13 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tCONSTANT,  'Exception')
   end
 
+  def test_bug_line_begin_label
+    setup_lexer(19)
+    util_lex_token("foo:bar",
+                   :tIDENTIFIER, 'foo',
+                   :tSYMBOL,     'bar')
+  end
+
   def test_bug_ragel_stack
     util_lex_token("\"\#{$2 ? $2 : 1}\"",
                    :tSTRING_BEG,      "\"",

data/test/test_parser.rb

@@ -434,7 +434,8 @@ class TestParser < MiniTest::Unit::TestCase
       %q[{ foo: 2 }],
       %q{^ begin
         |         ^ end
-        |  ~~~~ expression (pair.sym)
+        |     ^ operator (pair)
+        |  ~~~ expression (pair.sym)
         |  ~~~~~~ expression (pair)
         |~~~~~~~~~~ expression},
         ALL_VERSIONS - %w(1.8))
@@ -637,9 +638,9 @@ class TestParser < MiniTest::Unit::TestCase
         })
   end
 
-  def test_cvdecl
+  def test_cvasgn
     assert_parses(
-      s(:cvdecl, :@@var, s(:int, 10)),
+      s(:cvasgn, :@@var, s(:int, 10)),
       %q{@@var = 10},
       %q{~~~~~ name
         |      ^ operator
@@ -647,17 +648,6 @@ class TestParser < MiniTest::Unit::TestCase
         })
   end
 
-  def test_cvasgn
-    assert_parses(
-      s(:def, :a, s(:args),
-        s(:cvasgn, :@@var, s(:int, 10))),
-      %q{def a; @@var = 10; end},
-      %q{       ~~~~~ name (cvasgn)
-        |             ^ operator (cvasgn)
-        |       ~~~~~~~~~~ expression (cvasgn)
-        })
-  end
-
   def test_gvasgn
     assert_parses(
       s(:gvasgn, :$var, s(:int, 10)),
@@ -723,9 +713,9 @@ class TestParser < MiniTest::Unit::TestCase
 
   # Constants
 
-  def test_cdecl_toplevel
+  def test_casgn_toplevel
     assert_parses(
-      s(:cdecl, s(:cbase), :Foo, s(:int, 10)),
+      s(:casgn, s(:cbase), :Foo, s(:int, 10)),
       %q{::Foo = 10},
       %q{  ~~~ name
         |      ^ operator
@@ -734,9 +724,9 @@ class TestParser < MiniTest::Unit::TestCase
         })
   end
 
-  def test_cdecl_scoped
+  def test_casgn_scoped
     assert_parses(
-      s(:cdecl, s(:const, nil, :Bar), :Foo, s(:int, 10)),
+      s(:casgn, s(:const, nil, :Bar), :Foo, s(:int, 10)),
       %q{Bar::Foo = 10},
       %q{     ~~~ name
         |         ^ operator
@@ -745,9 +735,9 @@ class TestParser < MiniTest::Unit::TestCase
         })
   end
 
-  def test_cdecl_unscoped
+  def test_casgn_unscoped
     assert_parses(
-      s(:cdecl, nil, :Foo, s(:int, 10)),
+      s(:casgn, nil, :Foo, s(:int, 10)),
       %q{Foo = 10},
       %q{~~~ name
         |    ^ operator
@@ -755,7 +745,7 @@ class TestParser < MiniTest::Unit::TestCase
         })
   end
 
-  def test_cdecl_invalid
+  def test_casgn_invalid
     assert_diagnoses(
       [:error, :dynamic_const],
       %q{def f; Foo = 1; end},
@@ -809,7 +799,7 @@ class TestParser < MiniTest::Unit::TestCase
   def test_masgn_splat
     assert_parses(
       s(:masgn,
-        s(:mlhs, s(:ivasgn, :@foo), s(:cvdecl, :@@bar)),
+        s(:mlhs, s(:ivasgn, :@foo), s(:cvasgn, :@@bar)),
         s(:array, s(:splat, s(:lvar, :foo)))),
       %q{@foo, @@bar = *foo},
       %q{              ^ operator (array.splat)
@@ -951,7 +941,7 @@ class TestParser < MiniTest::Unit::TestCase
     assert_parses(
       s(:masgn,
         s(:mlhs,
-          s(:cdecl, s(:self), :A),
+          s(:casgn, s(:self), :A),
           s(:lvasgn, :foo)),
         s(:lvar, :foo)),
       %q{self::A, foo = foo})
@@ -959,7 +949,7 @@ class TestParser < MiniTest::Unit::TestCase
     assert_parses(
       s(:masgn,
         s(:mlhs,
-          s(:cdecl, s(:cbase), :A),
+          s(:casgn, s(:cbase), :A),
           s(:lvasgn, :foo)),
         s(:lvar, :foo)),
       %q{::A, foo = foo})
@@ -1038,7 +1028,7 @@ class TestParser < MiniTest::Unit::TestCase
         |~~~~~~~ expression})
 
     assert_parses(
-      s(:op_asgn, s(:cvdecl, :@@var), :|, s(:int, 10)),
+      s(:op_asgn, s(:cvasgn, :@@var), :|, s(:int, 10)),
       %q{@@var |= 10})
 
     assert_parses(
@@ -1065,13 +1055,13 @@ class TestParser < MiniTest::Unit::TestCase
   def test_const_op_asgn
     assert_parses(
       s(:op_asgn,
-        s(:cdecl, nil, :A), :+,
+        s(:casgn, nil, :A), :+,
         s(:int, 1)),
       %q{A += 1})
 
     assert_parses(
       s(:op_asgn,
-        s(:cdecl, s(:cbase), :A), :+,
+        s(:casgn, s(:cbase), :A), :+,
         s(:int, 1)),
       %q{::A += 1},
       %q{},
@@ -1079,7 +1069,7 @@ class TestParser < MiniTest::Unit::TestCase
 
     assert_parses(
       s(:op_asgn,
-        s(:cdecl, s(:const, nil, :B), :A), :+,
+        s(:casgn, s(:const, nil, :B), :A), :+,
         s(:int, 1)),
       %q{B::A += 1},
       %q{},
@@ -1800,6 +1790,12 @@ class TestParser < MiniTest::Unit::TestCase
         s(:kwarg, :foo)),
       %Q{foo:\n},
       ALL_VERSIONS - %w(1.8 1.9 2.0))
+
+    assert_parses_args(
+      s(:args,
+        s(:kwoptarg, :foo, s(:int, -1))),
+      %Q{foo: -1\n},
+      ALL_VERSIONS - %w(1.8 1.9 2.0))
   end
 
   def assert_parses_margs(ast, code, versions=ALL_VERSIONS - %w(1.8))
@@ -2770,7 +2766,7 @@ class TestParser < MiniTest::Unit::TestCase
         |~~~~~~~~~ expression})
 
     assert_parses(
-      s(:cdecl, s(:lvar, :foo), :A, s(:int, 1)),
+      s(:casgn, s(:lvar, :foo), :A, s(:int, 1)),
       %q{foo::A = 1},
       %q{     ~ name
         |   ^^ double_colon
@@ -4139,25 +4135,6 @@ class TestParser < MiniTest::Unit::TestCase
   # Miscellanea
   #
 
-  def test_crlf_line_endings
-    with_versions(nil, ALL_VERSIONS) do |_ver, parser|
-      source_file = Parser::Source::Buffer.new('(comments)')
-      source_file.source = "\r\nfoo"
-
-      range = lambda do |from, to|
-        Parser::Source::Range.new(source_file, from, to)
-      end
-
-      ast = parser.parse(source_file)
-
-      assert_equal s(:lvar, :foo),
-                   ast
-
-      assert_equal range.call(2, 5),
-                   ast.src.expression
-    end
-  end
-
   def test_begin_cmdarg
     assert_parses(
       s(:send, nil, :p,
@@ -4197,6 +4174,26 @@ class TestParser < MiniTest::Unit::TestCase
       %q{~~ location})
   end
 
+  def test_unterminated_embedded_doc
+    assert_diagnoses(
+      [:fatal, :embedded_document],
+      %Q{=begin\nfoo\nend},
+      %q{~~~~~~ location})
+
+    assert_diagnoses(
+      [:fatal, :embedded_document],
+      %Q{=begin\nfoo\nend\n},
+      %q{~~~~~~ location})
+  end
+
+  def test_codepoint_too_large
+    assert_diagnoses(
+      [:error, :unicode_point_too_large],
+      %q{"\u{120 120000}"},
+      %q{        ~~~~~~ location},
+      ALL_VERSIONS - %w(1.8))
+  end
+
   def test_on_error
     assert_diagnoses(
       [:error, :unexpected_token, { :token => 'tIDENTIFIER' }],
@@ -4204,6 +4201,47 @@ class TestParser < MiniTest::Unit::TestCase
       %q{            ~~~ location})
   end
 
+  #
+  # Token and comment extraction
+  #
+
+  def test_comments
+    with_versions(ALL_VERSIONS) do |_ver, parser|
+      source_file = Parser::Source::Buffer.new('(comments)')
+      source_file.source = "1 + # foo\n 2"
+
+      range = lambda do |from, to|
+        Parser::Source::Range.new(source_file, from, to)
+      end
+
+      _ast, comments = parser.parse_with_comments(source_file)
+
+      assert_equal [
+                     Parser::Source::Comment.new(range.call(4, 9))
+                   ], comments
+    end
+  end
+
+  def test_tokenize
+    with_versions(ALL_VERSIONS) do |_ver, parser|
+      source_file = Parser::Source::Buffer.new('(tokenize)')
+      source_file.source = "1 + # foo\n 2"
+
+      range = lambda do |from, to|
+        Parser::Source::Range.new(source_file, from, to)
+      end
+
+      _ast, _comments, tokens = parser.tokenize(source_file)
+
+      assert_equal [
+                     [:tINTEGER, [ 1,       range.call(0, 1) ]],
+                     [:tPLUS,    [ '+',     range.call(2, 3) ]],
+                     [:tCOMMENT, [ '# foo', range.call(4, 9) ]],
+                     [:tINTEGER, [ 2,       range.call(11, 12) ]],
+                   ], tokens
+    end
+  end
+
   #
   # Bug-specific tests
   #