ruby_parser 3.0.4 → 3.1.0

data/History.txt

@@ -1,3 +1,16 @@
+=== 3.1.0 / 2012-12-06
+
+* 2 minor enhancements:
+
+  * Added RubyParser.for_current_ruby to provide a parser that matches your runtime. (neilconway)
+  * Duck-typed IDENT_CHAR_RE instead of using RUBY_VERSION
+
+* 3 bug fixes:
+
+  * Cleared out body comments in class/module/defn/defs
+  * Flipped lexer tests to US-ASCII to avoid encoding hell
+  * yyerror is now an alias for syntax_error
+
 === 3.0.4 / 2012-11-26
 
 * 1 bug fix:

data/lib/ruby18_parser.rb

@@ -4617,6 +4617,7 @@ end
 def _reduce_310(val, _values, result)
                       result = new_class val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end
@@ -4645,6 +4646,7 @@ end
 def _reduce_314(val, _values, result)
                       result = new_sclass val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end
@@ -4668,6 +4670,7 @@ end
 def _reduce_317(val, _values, result)
                       result = new_module val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end
@@ -4685,6 +4688,7 @@ def _reduce_319(val, _values, result)
                       result = new_defn val
                       self.env.unextend
                       self.in_def = false
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end
@@ -4709,6 +4713,7 @@ def _reduce_322(val, _values, result)
 
                       self.env.unextend
                       self.in_single -= 1
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end

data/lib/ruby18_parser.y

@@ -537,7 +537,6 @@ rule
                       result = s(:op_asgn, val[0], val[4], val[2], val[3])
                     }
                 | primary_value tCOLON2 tCONSTANT tOP_ASGN arg
-
                     {
                       yyerror "constant re-assignment"
                     }
@@ -1083,6 +1082,7 @@ rule
                     {
                       result = new_class val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kCLASS tLSHFT
                     {
@@ -1103,6 +1103,7 @@ rule
                     {
                       result = new_sclass val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kMODULE
                     {
@@ -1120,6 +1121,7 @@ rule
                     {
                       result = new_module val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kDEF fname
                     {
@@ -1133,6 +1135,7 @@ rule
                       result = new_defn val
                       self.env.unextend
                       self.in_def = false
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kDEF singleton dot_or_colon
                     {
@@ -1151,6 +1154,7 @@ rule
 
                       self.env.unextend
                       self.in_single -= 1
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kBREAK
                     {

data/lib/ruby19_parser.rb

@@ -4681,6 +4681,7 @@ end
 def _reduce_314(val, _values, result)
                       result = new_class val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end
@@ -4709,6 +4710,7 @@ end
 def _reduce_318(val, _values, result)
                       result = new_sclass val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end
@@ -4732,6 +4734,7 @@ end
 def _reduce_321(val, _values, result)
                       result = new_module val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end
@@ -4751,6 +4754,7 @@ def _reduce_323(val, _values, result)
 
                       self.env.unextend
                       self.in_def = false
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end
@@ -4777,6 +4781,7 @@ def _reduce_326(val, _values, result)
 
                       self.env.unextend
                       self.in_single -= 1
+                      self.lexer.comments # we don't care about comments in the body
                     
     result
 end

data/lib/ruby19_parser.y

@@ -608,7 +608,6 @@ rule
                       result = s(:op_asgn, val[0], val[4], val[2], val[3])
                     }
                 | primary_value tCOLON2 tCONSTANT tOP_ASGN arg
-
                     {
                       yyerror "constant re-assignment"
                     }
@@ -1066,6 +1065,7 @@ rule
                     {
                       result = new_class val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kCLASS tLSHFT
                     {
@@ -1086,6 +1086,7 @@ rule
                     {
                       result = new_sclass val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kMODULE
                     {
@@ -1103,6 +1104,7 @@ rule
                     {
                       result = new_module val
                       self.env.unextend
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kDEF fname
                     {
@@ -1118,6 +1120,7 @@ rule
 
                       self.env.unextend
                       self.in_def = false
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kDEF singleton dot_or_colon
                     {
@@ -1138,6 +1141,7 @@ rule
 
                       self.env.unextend
                       self.in_single -= 1
+                      self.lexer.comments # we don't care about comments in the body
                     }
                 | kBREAK
                     {

data/lib/ruby_lexer.rb

@@ -4,13 +4,10 @@ class RubyLexer
 
   RUBY19 = "".respond_to? :encoding
 
-  IDENT_CHAR_RE = case RUBY_VERSION
-                  when /^1\.8/ then
-                    /[\w\x80-\xFF]/
-                  when /^(1\.9|2\.0)/ then # HACK - matching 2.0 for now
+  IDENT_CHAR_RE = if RUBY19 then
                     /[\w\u0080-\uFFFF]/u
                   else
-                    raise "bork"
+                    /[\w\x80-\xFF]/
                   end
 
   IDENT_RE = /^#{IDENT_CHAR_RE}+/
@@ -752,7 +749,6 @@ class RubyLexer
             tok = self.yacc_value = src.matched
             return TOKENS[tok]
           elsif src.scan(/\=begin(?=\s)/) then
-            # @comments << '=' << src.matched
             @comments << src.matched
 
             unless src.scan(/.*?\n=end( |\t|\f)*[^\n]*(\n|\z)/m) then

data/lib/ruby_parser_extras.rb

@@ -108,7 +108,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = '3.0.4' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.1.0' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -1049,10 +1049,7 @@ module RubyParserStuff
     # do nothing for now
   end
 
-  def yyerror msg
-    warn msg if $DEBUG
-    super()
-  end
+  alias yyerror syntax_error
 
   def on_error(et, ev, values)
     super
@@ -1301,6 +1298,17 @@ class RubyParser
     @p18.reset
     @p19.reset
   end
+
+  def self.for_current_ruby
+    case RUBY_VERSION
+    when /^1\.8/ then
+      Ruby18Parser.new
+    when /^1\.9/ then
+      Ruby19Parser.new
+    else
+      raise "unrecognized RUBY_VERSION #{RUBY_VERSION}"
+    end
+  end
 end
 
 ############################################################

data/test/test_ruby_lexer.rb

@@ -1,4 +1,5 @@
 #!/usr/local/bin/ruby
+# encoding: US-ASCII
 
 require 'rubygems'
 gem "minitest"

data/test/test_ruby_parser.rb

@@ -27,6 +27,8 @@ end
 class RubyParserTestCase < ParseTreeTestCase
   attr_accessor :result, :processor
 
+  make_my_diffs_pretty!
+
   def self.previous key
     "Ruby"
   end
@@ -880,8 +882,8 @@ module TestRubyParserShared
   end
 
   def test_i_fucking_hate_line_numbers
-    rb = <<-EOM.gsub(/^ {6}/, '')
-      def a
+    rb = <<-END.gsub(/^ {6}/, '')
+      if true
         p 1
         a.b 2
         c.d 3, 4
@@ -893,53 +895,81 @@ module TestRubyParserShared
         e.f(5)
         g.h(6, 7)
       end
-    EOM
+    END
 
-    pt = s(:defn, :a, s(:args).line(2),
-           s(:call, nil, :p, s(:lit, 1).line(2)).line(2),
-           s(:call, s(:call, nil, :a).line(3), :b,
-             s(:lit, 2).line(3)).line(3),
-           s(:call, s(:call, nil, :c).line(4), :d,
-             s(:lit, 3).line(4), s(:lit, 4).line(4)).line(4),
-           s(:call, s(:call, nil, :e).line(5), :f,
-             s(:lit, 5).line(5)).line(5),
-           s(:call, s(:call, nil, :g).line(6), :h,
-             s(:lit, 6).line(6), s(:lit, 7).line(6)).line(6),
-           s(:call, nil, :p, s(:lit, 1).line(7)).line(7),
-           s(:call, s(:call, nil, :a).line(8), :b,
-             s(:lit, 2).line(8)).line(8),
-           s(:call, s(:call, nil, :c).line(9), :d,
-             s(:lit, 3).line(9), s(:lit, 4).line(9)).line(9),
-           s(:call, s(:call, nil, :e).line(10), :f,
-             s(:lit, 5).line(10)).line(10),
-           s(:call, s(:call, nil, :g).line(11), :h,
-             s(:lit, 6).line(11), s(:lit, 7).line(11)).line(11)
-           ).line(1)
+    pt = s(:if, s(:true).line(1),
+           s(:block,
+             s(:call, nil, :p, s(:lit, 1).line(2)).line(2),
+             s(:call, s(:call, nil, :a).line(3), :b,
+               s(:lit, 2).line(3)).line(3),
+             s(:call, s(:call, nil, :c).line(4), :d,
+               s(:lit, 3).line(4), s(:lit, 4).line(4)).line(4),
+             s(:call, s(:call, nil, :e).line(5), :f,
+               s(:lit, 5).line(5)).line(5),
+             s(:call, s(:call, nil, :g).line(6), :h,
+               s(:lit, 6).line(6), s(:lit, 7).line(6)).line(6),
+             s(:call, nil, :p, s(:lit, 1).line(7)).line(7),
+             s(:call, s(:call, nil, :a).line(8), :b,
+               s(:lit, 2).line(8)).line(8),
+             s(:call, s(:call, nil, :c).line(9), :d,
+               s(:lit, 3).line(9), s(:lit, 4).line(9)).line(9),
+             s(:call, s(:call, nil, :e).line(10), :f,
+               s(:lit, 5).line(10)).line(10),
+             s(:call, s(:call, nil, :g).line(11), :h,
+               s(:lit, 6).line(11), s(:lit, 7).line(11)).line(11)).line(2),
+           nil).line(1)
 
     assert_parse rb, pt
   end
 
   def test_i_fucking_hate_line_numbers2
     rb = <<-EOM.gsub(/^ {6}/, '')
-      def a
-          p('a')
-          b = 1
-          p b
-          c =1
+      if true then
+        p('a')
+        b = 1
+        p b
+        c =1
       end
       a
     EOM
 
-      pt = s(:block,
-             s(:defn, :a, s(:args).line(2),
+    pt = s(:block,
+           s(:if, s(:true).line(1),
+             s(:block,
                s(:call, nil, :p, s(:str, "a").line(2)).line(2),
                s(:lasgn, :b, s(:lit, 1).line(3)).line(3),
                s(:call, nil, :p, s(:lvar, :b).line(4)).line(4),
-               s(:lasgn, :c, s(:lit, 1).line(5)).line(5)).line(1),
-             s(:call, nil, :a).line(7)).line(1)
+               s(:lasgn, :c, s(:lit, 1).line(5)).line(5)).line(2), # TODO line 2?
+             nil).line(1),
+           s(:call, nil, :a).line(7)).line(1)
 
     assert_parse rb, pt
   end
+
+  def test_parse_comments
+    p = RubyParser.new
+    sexp = p.parse <<-CODE
+      # class comment
+      class Inline
+        def show
+          # woot
+        end
+
+        # Returns a list of things
+        def list
+          # woot
+        end
+      end
+      CODE
+
+    assert_equal "# class comment\n", sexp.comments
+    act = sexp.find_nodes(:defn).map(&:comments)
+    exp = ["", "# Returns a list of things\n"]
+
+    assert_equal exp, act
+    assert_equal [], processor.comments
+    assert_equal "", processor.lexer.comments
+  end
 end
 
 class TestRubyParser < MiniTest::Unit::TestCase

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 3
   prerelease: 
   segments: 
   - 3
+  - 1
   - 0
-  - 4
-  version: 3.0.4
+  version: 3.1.0
 platform: ruby
 authors: 
 - Ryan Davis
@@ -36,7 +36,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2012-11-26 00:00:00 Z
+date: 2012-12-07 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sexp_processor

metadata.gz.sig

@@ -1,2 +1 @@
-���a
7'TJ��A����[�W$[�P,��r�S{�C���ͪ���2�S�%˽I��E��%�V��@�얤��l�-���W��q�;�{g�2�Z^�e�mJ���ޡ��C���4�����=Ȍa"�bq���U��E�_է���Z����]z�k��@t���o�.&nUf�2��%��zWl����8H��
-ޜl*�GTq H4�;|���E.#�9�M����2)&���Y� ����o�P]��Q���
+�ۇ�o�����?�^iu�vS�?��bx