parser 1.0.1 → 1.1.0

data/lib/parser/source/buffer.rb

@@ -1,9 +1,53 @@
+# encoding:ascii-8bit
+
 module Parser
   module Source
 
     class Buffer
       attr_reader :name, :first_line
 
+      def self.recognize_encoding(string)
+        if string.empty?
+          return Encoding::UTF_8
+        end
+
+        # TODO: Make this more efficient.
+        first_line, second_line = string.lines.first(2)
+        first_line.force_encoding(Encoding::ASCII_8BIT)
+
+        if first_line =~ /\A\xef\xbb\xbf/ # BOM
+          return Encoding::UTF_8
+        elsif first_line[0, 2] == '#!'
+          encoding_line = second_line
+        else
+          encoding_line = first_line
+        end
+
+        encoding_line.force_encoding(Encoding::ASCII_8BIT)
+
+        if encoding_line =~ /coding[:=]?\s*([a-z0-9_-]+)/
+          Encoding.find($1)
+        else
+          string.encoding
+        end
+      end
+
+      # Lexer expects UTF-8 input. This method processes the input
+      # in an arbitrary valid Ruby encoding and returns an UTF-8 encoded
+      # string.
+      #
+      def self.reencode_string(string)
+        encoding = recognize_encoding(string)
+
+        unless encoding.ascii_compatible?
+          raise RuntimeError, "Encoding #{encoding} is not ASCII-compatible"
+        end
+
+        string.
+          force_encoding(encoding).
+          encode(Encoding::UTF_8)
+      end
+
       def initialize(name, first_line = 1)
         @name       = name
         @first_line = first_line
@@ -11,7 +55,9 @@ module Parser
       end
 
       def read
-        self.source = File.read(@name)
+        File.open(@name, 'rb') do |io|
+          self.source = io.read
+        end
 
         self
       end
@@ -25,7 +71,11 @@ module Parser
       end
 
       def source=(source)
-        @source = source.freeze
+        if source.respond_to? :encoding
+          source = self.class.reencode_string(source)
+        end
+
+        @source  = source.freeze
 
         freeze
       end

data/lib/parser.rb

@@ -50,8 +50,6 @@ module Parser
     :regexp_options          => "unknown regexp options: %{options}",
     :cvar_name               => "`%{name}' is not allowed as a class variable name",
     :ivar_name               => "`%{name}' is not allowed as an instance variable name",
-    :ambiguous_literal       => "ambiguous first argument; parenthesize arguments or add whitespace to the right",
-    :ambiguous_prefix        => "`%{prefix}' interpreted as argument prefix",
     :trailing_underscore     => "trailing `_' in number",
     :empty_numeric           => "numeric literal without digits",
     :invalid_octal           => "invalid octal digit",
@@ -60,10 +58,13 @@ module Parser
     :unexpected              => "unexpected %{character}",
     :embedded_document       => "embedded document meats end of file (and they embark on a romantic journey)",
 
+    # Lexer warnings
+    :ambiguous_literal       => "ambiguous first argument; parenthesize arguments or add whitespace to the right",
+    :ambiguous_prefix        => "`%{prefix}' interpreted as argument prefix",
+
     # Parser errors
     :nth_ref_alias           => "cannot define an alias for a back-reference variable",
     :begin_in_method         => "BEGIN in method",
-    :end_in_method           => "END in method; use at_exit",
     :backref_assignment      => "cannot assign to a back-reference variable",
     :invalid_assignment      => "cannot assign to a keyword",
     :module_name_const       => "class or module name must be a constant literal",
@@ -72,16 +73,26 @@ module Parser
     :argument_ivar           => "formal argument cannot be an instance variable",
     :argument_gvar           => "formal argument cannot be a global variable",
     :argument_cvar           => "formal argument cannot be a class variable",
+    :duplicate_argument      => "duplicate argument name",
     :empty_symbol            => "empty symbol literal",
     :odd_hash                => "odd number of entries for a hash",
     :singleton_literal       => "cannot define a singleton method for a literal",
     :dynamic_const           => "dynamic constant assignment",
     :module_in_def           => "module definition in method body",
     :class_in_def            => "class definition in method body",
-    :space_before_lparen     => "don't put space before argument parentheses",
     :unexpected_percent_str  => "%{type}: unknown type of percent-literal",
-    :useless_else            => "else without rescue is useless",
     :block_and_blockarg      => "both block argument and literal block are passed",
     :masgn_as_condition      => "multiple assignment in conditional context",
+
+    # Parser warnings
+    :end_in_method           => "END in method; use at_exit",
+    :space_before_lparen     => "don't put space before argument parentheses",
+    :useless_else            => "else without rescue is useless",
   }.freeze
+
+  def self.check_for_encoding_support
+    unless defined?(Encoding)
+      raise RuntimeError, "Parsing 1.9 and later versions of Ruby is not supported on 1.8 due to the lack of Encoding support"
+    end
+  end
 end

data/parser.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = 'parser'
-  spec.version       = '1.0.1'
+  spec.version       = '1.1.0'
   spec.authors       = ['Peter Zotov']
   spec.email         = ['whitequark@whitequark.org']
   spec.description   = %q{A Ruby parser written in pure Ruby.}
@@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
                           lib/parser/ruby18.rb
                           lib/parser/ruby19.rb
                           lib/parser/ruby20.rb
+                          lib/parser/ruby21.rb
                        )
   spec.executables   = %w()
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})

data/test/helper.rb

@@ -7,7 +7,7 @@ if SimpleCov.usable?
   if defined?(TracePoint)
     require_relative 'racc_coverage_helper'
 
-    RaccCoverage.start(%w(ruby18.y ruby19.y ruby20.y),
+    RaccCoverage.start(%w(ruby18.y ruby19.y ruby20.y ruby21.y),
                        File.expand_path('../../lib/parser', __FILE__))
 
     # Report results faster.

data/test/parse_helper.rb

@@ -1,9 +1,16 @@
-require 'parser/all'
-
 module ParseHelper
   include AST::Sexp
 
-  ALL_VERSIONS = %w(1.8 1.9 2.0)
+  if RUBY_VERSION == '1.8.7'
+    require 'parser/ruby18'
+
+    ALL_VERSIONS = %w(1.8)
+  else
+    require 'parser/all'
+    require 'parser/ruby21'
+
+    ALL_VERSIONS = %w(1.8 1.9 2.0 2.1)
+  end
 
   def setup
     @diagnostics = []
@@ -16,6 +23,7 @@ module ParseHelper
     when '1.8'; parser = Parser::Ruby18.new
     when '1.9'; parser = Parser::Ruby19.new
     when '2.0'; parser = Parser::Ruby20.new
+    when '2.1'; parser = Parser::Ruby21.new
     else raise "Unrecognized Ruby version #{version}"
     end
 
@@ -27,7 +35,7 @@ module ParseHelper
   end
 
   def with_versions(code, versions)
-    versions.each do |version|
+    (versions & ALL_VERSIONS).each do |version|
       @diagnostics.clear
 
       parser = parser_for_ruby_version(version)

data/test/test_current.rb

@@ -0,0 +1,17 @@
+require 'helper'
+require 'parser/current'
+
+class TestCurrent < MiniTest::Unit::TestCase
+  def test_current
+    case RUBY_VERSION
+    when '1.8.7'
+      assert_equal Parser::Ruby18, Parser::CurrentRuby
+    when '1.9.2', '1.9.3'
+      assert_equal Parser::Ruby19, Parser::CurrentRuby
+    when '2.0.0'
+      assert_equal Parser::Ruby20, Parser::CurrentRuby
+    else
+      flunk "Update test_parser_current for #{RUBY_VERSION}"
+    end
+  end
+end

data/test/test_encoding.rb

@@ -0,0 +1,31 @@
+# encoding: binary
+
+require 'helper'
+
+class TestEncoding < MiniTest::Unit::TestCase
+  def recognize(string)
+    Parser::Source::Buffer.recognize_encoding(string)
+  end
+
+  if defined?(Encoding)
+    def test_default
+      assert_equal Encoding::BINARY, recognize("foobar")
+    end
+
+    def test_bom
+      assert_equal Encoding::UTF_8, recognize("\xef\xbb\xbffoobar")
+    end
+
+    def test_magic_comment
+      assert_equal Encoding::KOI8_R, recognize("# coding:koi8-r\nfoobar")
+    end
+
+    def test_shebang
+      assert_equal Encoding::KOI8_R, recognize("#!/bin/foo\n# coding:koi8-r\nfoobar")
+    end
+
+    def test_empty
+      assert_equal Encoding::UTF_8, recognize("")
+    end
+  end
+end

data/test/test_lexer.rb

@@ -50,10 +50,10 @@ class TestLexer < MiniTest::Unit::TestCase
     end
   end
 
-  def util_lex_fname(name, type, end_state = :expr_end)
+  def util_lex_fname(name, type)
     util_lex_token("def #{name} ", :kDEF, 'def', type, name)
 
-    assert_equal end_state, @lex.state
+    assert_equal :expr_endfn, @lex.state
   end
 
   def util_lex_token(input, *args)
@@ -149,14 +149,12 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tIDENTIFIER, "m",
                    :tUMINUS_NUM, "-",
                    :tINTEGER, 3)
-    # TODO: verify warning
   end
 
   def test_ambiguous_uplus
     util_lex_token("m +3",
                    :tIDENTIFIER, "m",
                    :tINTEGER, 3)
-    # TODO: verify warning
   end
 
   def test_and
@@ -319,7 +317,7 @@ class TestLexer < MiniTest::Unit::TestCase
   def test_backtick_method
     @lex.state = :expr_fname
     util_lex_token("`", :tBACK_REF2, "`")
-    assert_equal :expr_end, @lex.state
+    assert_equal :expr_endfn, @lex.state
   end
 
   def test_bad_char
@@ -822,7 +820,7 @@ class TestLexer < MiniTest::Unit::TestCase
   end
 
   def test_identifier_def
-    util_lex_fname "identifier", :tIDENTIFIER, :expr_end
+    util_lex_fname "identifier", :tIDENTIFIER
   end
 
   def test_identifier_eh
@@ -854,7 +852,7 @@ class TestLexer < MiniTest::Unit::TestCase
   end
 
   def test_identifier_equals_def
-    util_lex_fname "identifier=", :tIDENTIFIER, :expr_end
+    util_lex_fname "identifier=", :tIDENTIFIER
   end
 
   def test_identifier_equals_def2

data/test/test_parse_helper.rb

@@ -8,11 +8,16 @@ class TestParseHelper < MiniTest::Unit::TestCase
     assert_instance_of Parser::Ruby18,
                        parser_for_ruby_version('1.8')
 
-    assert_instance_of Parser::Ruby19,
-                       parser_for_ruby_version('1.9')
+    unless RUBY_VERSION == '1.8.7'
+      assert_instance_of Parser::Ruby19,
+                         parser_for_ruby_version('1.9')
 
-    # assert_instance_of Parser::Ruby20,
-    #                    parser_for_ruby_version('2.0')
+      assert_instance_of Parser::Ruby20,
+                         parser_for_ruby_version('2.0')
+
+      assert_instance_of Parser::Ruby21,
+                         parser_for_ruby_version('2.1')
+    end
   end
 
   def parse_maps(what)

data/test/test_parser.rb

@@ -1,3 +1,5 @@
+# encoding:utf-8
+
 require 'helper'
 require 'parse_helper'
 
@@ -1042,6 +1044,54 @@ class TestParser < MiniTest::Unit::TestCase
       %q{~~~ location})
   end
 
+  def test_const_op_asgn
+    assert_parses(
+      s(:op_asgn,
+        s(:cdecl, nil, :A), :+,
+        s(:int, 1)),
+      %q{A += 1})
+
+    assert_parses(
+      s(:op_asgn,
+        s(:cdecl, s(:cbase), :A), :+,
+        s(:int, 1)),
+      %q{::A += 1},
+      %q{},
+      ALL_VERSIONS - %w(1.8 1.9))
+
+    assert_parses(
+      s(:op_asgn,
+        s(:cdecl, s(:const, nil, :B), :A), :+,
+        s(:int, 1)),
+      %q{B::A += 1},
+      %q{},
+      ALL_VERSIONS - %w(1.8 1.9))
+  end
+
+  def test_const_op_asgn_invalid
+    assert_diagnoses(
+      [:error, :dynamic_const],
+      %q{Foo::Bar += 1},
+      %q{     ~~~ location},
+      %w(1.8 1.9))
+
+    assert_diagnoses(
+      [:error, :dynamic_const],
+      %q{::Bar += 1},
+      %q{  ~~~ location},
+      %w(1.8 1.9))
+
+    assert_diagnoses(
+      [:error, :dynamic_const],
+      %q{def foo; Foo::Bar += 1; end},
+      %q{              ~~~ location})
+
+    assert_diagnoses(
+      [:error, :dynamic_const],
+      %q{def foo; ::Bar += 1; end},
+      %q{           ~~~ location})
+  end
+
   # Method binary operator-assignment
 
   def test_op_asgn
@@ -1133,16 +1183,6 @@ class TestParser < MiniTest::Unit::TestCase
       [:error, :backref_assignment],
       %q{$+ |= m foo},
       %q{~~ location})
-
-    assert_diagnoses(
-      [:error, :dynamic_const],
-      %q{Foo::Bar += 1; end},
-      %q{     ~~~ location})
-
-    assert_diagnoses(
-      [:error, :dynamic_const],
-      %q{::Bar += 1; end},
-      %q{  ~~~ location})
   end
 
   # Variable logical operator-assignment
@@ -1716,6 +1756,14 @@ class TestParser < MiniTest::Unit::TestCase
       ALL_VERSIONS - %w(1.8 1.9))
   end
 
+  def test_kwarg_no_paren
+    assert_parses_args(
+      s(:args,
+        s(:kwarg, :foo)),
+      %Q{foo:\n},
+      ALL_VERSIONS - %w(1.8 1.9 2.0))
+  end
+
   def assert_parses_margs(ast, code, versions=ALL_VERSIONS - %w(1.8))
     assert_parses_args(
       s(:args, ast),
@@ -2085,6 +2133,105 @@ class TestParser < MiniTest::Unit::TestCase
       %q{        ~~~~~ location})
   end
 
+  def test_arg_duplicate
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(aa, aa); end},
+      %q{            ^^ location
+        |        ~~ highlights (0)})
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(aa, aa=1); end},
+      %q{            ^^ location
+        |        ~~ highlights (0)})
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(aa, *aa); end},
+      %q{             ^^ location
+        |        ~~ highlights (0)})
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(aa, &aa); end},
+      %q{             ^^ location
+        |        ~~ highlights (0)})
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(aa, (bb, aa)); end},
+      %q{                 ^^ location
+        |        ~~ highlights (0)},
+      ALL_VERSIONS - %w(1.8))
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(aa, *r, aa); end},
+      %q{                ^^ location
+        |        ~~ highlights (0)},
+      ALL_VERSIONS - %w(1.8))
+
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{lambda do |aa; aa| end},
+      %q{               ^^ location
+        |           ~~ highlights (0)},
+      ALL_VERSIONS - %w(1.8))
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(aa, aa: 1); end},
+      %q{            ^^ location
+        |        ~~ highlights (0)},
+      ALL_VERSIONS - %w(1.8 1.9))
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(aa, **aa); end},
+      %q{              ^^ location
+        |        ~~ highlights (0)},
+      ALL_VERSIONS - %w(1.8 1.9))
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(aa, aa:); end},
+      %q{            ^^ location
+        |        ~~ highlights (0)},
+      ALL_VERSIONS - %w(1.8 1.9 2.0))
+  end
+
+  def test_arg_duplicate_ignored
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(_, _); end},
+      %q{},
+      %w(1.8))
+
+    assert_parses(
+      s(:def, :foo,
+        s(:args, s(:arg, :_), s(:arg, :_)),
+        s(:nil)),
+      %q{def foo(_, _); end},
+      %q{},
+      ALL_VERSIONS - %w(1.8))
+
+    assert_diagnoses(
+      [:error, :duplicate_argument],
+      %q{def foo(_a, _a); end},
+      %q{},
+      %w(1.8 1.9))
+
+    assert_parses(
+      s(:def, :foo,
+        s(:args, s(:arg, :_a), s(:arg, :_a)),
+        s(:nil)),
+      %q{def foo(_a, _a); end},
+      %q{},
+      ALL_VERSIONS - %w(1.8 1.9))
+  end
+
   def test_kwarg_invalid
     assert_diagnoses(
       [:error, :argument_const],
@@ -3922,6 +4069,21 @@ class TestParser < MiniTest::Unit::TestCase
       ALL_VERSIONS - %w(1.8 1.9))
   end
 
+  if defined?(Encoding)
+    def test_magic_encoding_comment
+      assert_parses(
+        s(:begin,
+          s(:lvasgn, :"проверка", s(:int, 42)),
+          s(:send, nil, :puts, s(:lvar, :"проверка"))),
+        %Q{# coding:koi8-r
+           \xd0\xd2\xcf\xd7\xc5\xd2\xcb\xc1 = 42
+           puts \xd0\xd2\xcf\xd7\xc5\xd2\xcb\xc1}.
+          force_encoding(Encoding::BINARY),
+        %q{},
+        %w(1.9 2.0 2.1))
+    end
+  end
+
   #
   # Error recovery
   #