ripper_ruby_parser 1.1.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 44bd9ec7c716649fbcf3698e39d9d7951b70a3d0
-  data.tar.gz: 1b54fd14347e28a5c253b8f00be1a641dc7cbda5
+SHA256:
+  metadata.gz: 57d07c4327035328992f94f2e0f49b36362743c84062414a6f09b8738aef709b
+  data.tar.gz: 9e19226ece2a88714fc2e2483e4accb0ee7aca7f31398d25396f72581f3c864f
 SHA512:
-  metadata.gz: 4fd9c3ffee9de0d9948a126ca5e44d3dc46b50b36b4fd7d9537263b84b123ebeac09118f3b0ae12f203f7c3ee1eb438bca92b2b630502b7bc83f6d3ea1a47668
-  data.tar.gz: c24028ea9320a5fdba4eee182d257e9f3c5db7fc57f9b3e6ba15471e90eb96b48c291b7046e8ca52b83af4957a62e6b4321901179d53fe53ea549f431baf7f85
+  metadata.gz: 9f68e5ad8f58a6035ae16be08e5188a4f22f6809ce59705de877527ede90247125525a7be007b86b794ab046a8861ac4bd417ee3974e4d6fe60c877ff92c4230
+  data.tar.gz: 6f1b8cbd085e0249a56bf937d8806cba3dad3a07893f843361c8142321693e6c8e10225782f8fee672eb4ee3c148512778544648731b486f5977c142004f1271

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Change log
 
+## 1.2.0 / 2018-01-12
+
+* Improve code quality
+* Document public API
+* Speed improvements
+  - Process line numbers only once per parse run
+  - Reduce arbitrary conditionals
+  - Use deconstruction to split up block
+* Improve intermediate s-expressions, reducing the number of typeless
+  expressions.
+* Use SexpBuilder base class, giving more low-level access to the structure
+  created by Ripper.
+* Support Ruby 2.5
+* Improve handling of boolean operators with parenthes
+* Improve compatibility for begin..end blocks used as method and operator
+  arguments.
+* Drop support for Ruby 2.0 and 2.1
+* Handle `__ENCODING__` constant.
+
 ## 1.1.2 / 2017-10-07
 
 * Fix support for newer Ruby syntax

data/README.md

@@ -12,7 +12,7 @@ Parse with Ripper, produce sexps that are compatible with RubyParser.
 
 * Drop-in replacement for RubyParser.
 * Should handle 1.9 and later syntax gracefully.
-* Needs MRI 2.0 or higher
+* Requires MRI 2.2 or higher
 
 ## Install
 
@@ -29,7 +29,7 @@ Parse with Ripper, produce sexps that are compatible with RubyParser.
 
 ## Requirements
 
-* Ruby 2.0 or higher
+* Ruby 2.2 or higher
 * sexp_processor
 
 ## Hacking and contributing

data/Rakefile

@@ -5,7 +5,7 @@ require 'rake/testtask'
 namespace :test do
   Rake::TestTask.new(:unit) do |t|
     t.libs = ['lib']
-    t.test_files = FileList['test/unit/*_test.rb']
+    t.test_files = FileList['test/ripper_ruby_parser/**/*_test.rb']
     t.warning = true
   end
 

data/lib/ripper_ruby_parser.rb

@@ -1,9 +1,2 @@
-if RUBY_VERSION < '1.9.3'
-  raise LoadError, 'Only ruby version 1.9.3 and up are supported'
-end
-
 require 'ripper_ruby_parser/version'
 require 'ripper_ruby_parser/parser'
-
-module RipperRubyParser
-end

data/lib/ripper_ruby_parser/commenting_ripper_parser.rb

@@ -2,10 +2,12 @@ require 'ripper'
 require 'ripper_ruby_parser/syntax_error'
 
 module RipperRubyParser
-  # Variant of Ripper's SexpBuilderPP parser class that inserts comments as
+  # Variant of Ripper's SexpBuilder parser class that inserts comments as
   # Sexps into the built parse tree.
-  class CommentingRipperParser < Ripper::SexpBuilderPP
-    def initialize *args
+  #
+  # @api private
+  class CommentingRipperParser < Ripper::SexpBuilder
+    def initialize(*args)
       super
       @comment = nil
       @comment_stack = []
@@ -19,13 +21,13 @@ module RipperRubyParser
       Sexp.from_array(result)
     end
 
-    def on_comment tok
+    def on_comment(tok)
       @comment ||= ''
       @comment += tok
       super
     end
 
-    def on_kw tok
+    def on_kw(tok)
       case tok
       when 'class', 'def', 'module'
         unless @in_symbol
@@ -36,42 +38,122 @@ module RipperRubyParser
       super
     end
 
-    def on_module *args
+    def on_module(*args)
       commentize(:module, super)
     end
 
-    def on_class *args
+    def on_class(*args)
       commentize(:class, super)
     end
 
-    def on_sclass *args
+    def on_sclass(*args)
       commentize(:class, super)
     end
 
-    def on_def *args
+    def on_def(*args)
       commentize(:def, super)
     end
 
-    def on_defs *args
+    def on_defs(*args)
       commentize(:def, super)
     end
 
-    def on_qsymbols_add list, elem
-      super list, [:dyna_symbol, [elem]]
+    def on_args_new
+      [:args]
     end
 
-    def on_symbols_add list, elem
-      super list, [:dyna_symbol, elem]
+    def on_args_add(list, elem)
+      list << elem
     end
 
-    def on_words_add list, elem
-      if elem.count == 1
-        super
+    def on_mlhs_new
+      [:mlhs]
+    end
+
+    def on_mlhs_add(list, elem)
+      if list.first == :mlhs
+        list << elem
       else
-        super list, [:string_content, *elem]
+        [:mlhs_add_post, list, elem]
       end
     end
 
+    def on_mrhs_new
+      [:mrhs]
+    end
+
+    def on_mrhs_add(list, elem)
+      list << elem
+    end
+
+    def on_qsymbols_new
+      [:qsymbols]
+    end
+
+    def on_qsymbols_add(list, elem)
+      list << elem
+    end
+
+    def on_qwords_new
+      [:qwords]
+    end
+
+    def on_qwords_add(list, elem)
+      list << elem
+    end
+
+    def on_regexp_new
+      [:regexp]
+    end
+
+    def on_regexp_add(list, elem)
+      list << elem
+    end
+
+    def on_stmts_new
+      [:stmts]
+    end
+
+    def on_stmts_add(list, elem)
+      list << elem
+    end
+
+    def on_string_add(list, elem)
+      list << elem
+    end
+
+    def on_symbols_new
+      [:symbols]
+    end
+
+    def on_symbols_add(list, elem)
+      list << elem
+    end
+
+    def on_word_new
+      [:word]
+    end
+
+    def on_word_add(list, elem)
+      list << elem
+    end
+
+    def on_words_new
+      [:words]
+    end
+
+    def on_words_add(list, elem)
+      list << elem
+    end
+
+    def on_xstring_new
+      [:xstring]
+    end
+
+    def on_xstring_add(list, elem)
+      list << elem
+    end
+
     def on_op(token)
       @seen_space = false
       super
@@ -92,7 +174,7 @@ module RipperRubyParser
       super
     end
 
-    NUMBER_LITERAL_TYPES = [:@int, :@float]
+    NUMBER_LITERAL_TYPES = [:@int, :@float].freeze
 
     def on_unary(op, value)
       if !@space_before && op == :-@ && NUMBER_LITERAL_TYPES.include?(value.first)
@@ -107,55 +189,51 @@ module RipperRubyParser
       end
     end
 
-    def on_symbeg *args
+    def on_symbeg(*args)
       @in_symbol = true
       super
     end
 
-    def on_symbol *args
+    def on_symbol(*args)
       @in_symbol = false
       super
     end
 
-    def on_embexpr_beg *args
+    def on_embexpr_beg(*args)
       @in_symbol = false
       super
     end
 
-    def on_dyna_symbol *args
+    def on_dyna_symbol(*args)
       @in_symbol = false
       super
     end
 
-    def on_parse_error *args
+    def on_parse_error(*args)
       raise SyntaxError, *args
     end
 
-    def on_class_name_error *args
+    def on_class_name_error(*args)
       raise SyntaxError, *args
     end
 
-    def on_alias_error *args
+    def on_alias_error(*args)
       raise SyntaxError, *args
     end
 
-    def on_assign_error *args
+    def on_assign_error(*args)
       raise SyntaxError, *args
     end
 
-    def on_param_error *args
+    def on_param_error(*args)
       raise SyntaxError, *args
     end
 
     private
 
-    def commentize name, exp
-      raise "Comment stack empty in #{name} event" if @comment_stack.empty?
-      tok, comment = @comment_stack.pop
+    def commentize(_name, exp)
+      _tok, comment = @comment_stack.pop
       @comment = nil
-      unless tok == name
-        raise "Expected on_#{tok} event, got on_#{name}"
-      end
       [:comment, comment || '', exp]
     end
 

data/lib/ripper_ruby_parser/parser.rb

@@ -5,26 +5,40 @@ module RipperRubyParser
   # Main parser class. Brings together Ripper and our
   # RipperRubyParser::SexpProcessor.
   class Parser
-    attr_accessor :extra_compatible
-
-    def initialize processor = SexpProcessor.new
-      @processor = processor
-      @extra_compatible = false
-    end
-
-    def parse source, filename = '(string)', lineno = 1
+    def parse(source, filename = '(string)', lineno = 1)
       parser = CommentingRipperParser.new(source, filename, lineno)
       exp = parser.parse
 
-      @processor.filename = filename
-      @processor.extra_compatible = extra_compatible
-      result = @processor.process exp
+      processor = SexpProcessor.new(filename: filename)
+      result = processor.process exp
 
-      if result == s(:void_stmt)
+      if result.sexp_type == :void_stmt
         nil
       else
+        trickle_up_line_numbers result
+        trickle_down_line_numbers result
         result
       end
     end
+
+    private
+
+    def trickle_up_line_numbers(exp)
+      exp.each do |sub_exp|
+        if sub_exp.is_a? Sexp
+          trickle_up_line_numbers sub_exp
+          exp.line ||= sub_exp.line
+        end
+      end
+    end
+
+    def trickle_down_line_numbers(exp)
+      exp.each do |sub_exp|
+        if sub_exp.is_a? Sexp
+          sub_exp.line ||= exp.line
+          trickle_down_line_numbers sub_exp
+        end
+      end
+    end
   end
 end

data/lib/ripper_ruby_parser/sexp_handlers.rb

@@ -13,8 +13,11 @@ require 'ripper_ruby_parser/sexp_handlers/methods'
 require 'ripper_ruby_parser/sexp_handlers/operators'
 
 module RipperRubyParser
+  # Umbrella module for handlers of particular sexp types
+  #
+  # @api private
   module SexpHandlers
-    def self.included base
+    def self.included(base)
       base.class_eval do
         include HelperMethods
 

data/lib/ripper_ruby_parser/sexp_handlers/arguments.rb

@@ -1,25 +1,26 @@
 module RipperRubyParser
   module SexpHandlers
+    # Sexp handlers for argument lists
     module Arguments
-      def process_args_add_block exp
+      def process_args_add_block(exp)
         _, regular, block = exp.shift 3
-        args = handle_potentially_typeless_sexp(regular)
+        args = process(regular)
         args << s(:block_pass, process(block)) if block
-        s(:arglist, *args)
+        s(:arglist, *args.sexp_body)
       end
 
-      def process_args_add_star exp
+      def process_args_add_star(exp)
         generic_add_star exp
       end
 
-      def process_arg_paren exp
+      def process_arg_paren(exp)
         _, args = exp.shift 2
         args = s() if args.nil?
         args.unshift :arglist unless args.first.is_a? Symbol
         process(args)
       end
 
-      def process_rest_param exp
+      def process_rest_param(exp)
         _, ident = exp.shift 2
         s(:splat, process(ident))
       end

data/lib/ripper_ruby_parser/sexp_handlers/arrays.rb

@@ -1,12 +1,14 @@
 module RipperRubyParser
   module SexpHandlers
+    # Sexp handlers for array literals
     module Arrays
-      def process_array exp
+      def process_array(exp)
         _, elems = exp.shift 2
+        return s(:array) if elems.nil?
         s(:array, *handle_array_elements(elems))
       end
 
-      def process_aref exp
+      def process_aref(exp)
         _, coll, idx = exp.shift 3
 
         coll = process(coll)

data/lib/ripper_ruby_parser/sexp_handlers/assignment.rb

@@ -1,14 +1,15 @@
 module RipperRubyParser
   module SexpHandlers
+    # Sexp handlers for assignments
     module Assignment
-      def process_assign exp
+      def process_assign(exp)
         _, lvalue, value = exp.shift 3
         lvalue = process(lvalue)
         value = process(value)
 
         case value.sexp_type
-        when :splat
-          value = s(:svalue, value)
+        when :mrhs
+          value.sexp_type = :svalue
         when :fake_array
           value = s(:svalue, s(:array, *value.sexp_body))
         end
@@ -17,25 +18,21 @@ module RipperRubyParser
                          create_regular_assignment_sub_type(lvalue, value))
       end
 
-      def process_massign exp
+      def process_massign(exp)
         _, left, right = exp.shift 3
 
-        left = handle_potentially_typeless_sexp left
+        left = process left
 
-        if left.first == :masgn
-          left = left[1]
-          left.shift
-        end
-
-        left = create_multiple_assignment_sub_types left
+        left = left[1] if left.sexp_type == :masgn
+        left = create_multiple_assignment_sub_types left.sexp_body
 
         right = process(right)
 
         case right.sexp_type
         when :fake_array
           right[0] = :array
-        when :splat
-          nil # Do nothing
+        when :mrhs
+          right = right[1]
         else
           right = s(:to_ary, right)
         end
@@ -43,42 +40,41 @@ module RipperRubyParser
         s(:masgn, s(:array, *left), right)
       end
 
-      def process_mrhs_new_from_args exp
+      def process_mrhs_new_from_args(exp)
         _, inner, last = exp.shift 3
-        inner.map! { |item| process(item) }
-        inner.push process(last) unless last.nil?
+        inner = map_process_sexp_body_compact(inner)
+        inner.push process(last) if last
         s(:fake_array, *inner)
       end
 
-      def process_mrhs_add_star exp
-        exp = generic_add_star exp
-
-        if exp.first.is_a? Symbol
-          exp
-        else
-          exp.first
-        end
+      def process_mrhs_add_star(exp)
+        generic_add_star exp
       end
 
-      def process_mlhs_add_star exp
-        _, args, splatarg, rest = exp.shift 4
-        items = handle_potentially_typeless_sexp args
+      def process_mlhs_add_star(exp)
+        _, args, splatarg = exp.shift 3
+        items = process args
         items << s(:splat, process(splatarg))
-        rest.each { |arg| items << process(arg) } if rest
-        items
       end
 
-      def process_mlhs_paren exp
-        _, contents = exp.shift 2
+      def process_mlhs_add_post(exp)
+        _, base, rest = exp.shift 3
+        process(base).push(*process(rest).sexp_body)
+      end
 
-        items = handle_potentially_typeless_sexp(contents)
+      def process_mlhs_paren(exp)
+        _, contents = exp.shift 2
 
-        return items if items.first.is_a? Symbol
+        items = process(contents)
 
-        s(:masgn, s(:array, *create_multiple_assignment_sub_types(items)))
+        if items.sexp_type == :mlhs
+          s(:masgn, s(:array, *create_multiple_assignment_sub_types(items.sexp_body)))
+        else
+          items
+        end
       end
 
-      def process_opassign exp
+      def process_opassign(exp)
         _, lvalue, operator, value = exp.shift 4
 
         lvalue = process(lvalue)
@@ -90,7 +86,7 @@ module RipperRubyParser
 
       private
 
-      def create_multiple_assignment_sub_types sexp_list
+      def create_multiple_assignment_sub_types(sexp_list)
         sexp_list.map! do |item|
           if item.sexp_type == :splat
             if item[1].nil?
@@ -104,7 +100,7 @@ module RipperRubyParser
         end
       end
 
-      def create_valueless_assignment_sub_type item
+      def create_valueless_assignment_sub_type(item)
         item = with_line_number(item.line,
                                 create_regular_assignment_sub_type(item, nil))
         item.pop
@@ -112,11 +108,11 @@ module RipperRubyParser
       end
 
       OPERATOR_ASSIGNMENT_MAP = {
-        :"||" => :op_asgn_or,
-        :"&&" => :op_asgn_and
+        '||': :op_asgn_or,
+        '&&': :op_asgn_and
       }.freeze
 
-      def create_operator_assignment_sub_type lvalue, value, operator
+      def create_operator_assignment_sub_type(lvalue, value, operator)
         case lvalue.sexp_type
         when :aref_field
           _, arr, arglist = lvalue
@@ -134,7 +130,7 @@ module RipperRubyParser
         end
       end
 
-      def create_regular_assignment_sub_type lvalue, value
+      def create_regular_assignment_sub_type(lvalue, value)
         case lvalue.sexp_type
         when :aref_field
           _, arr, arglist = lvalue
@@ -161,11 +157,11 @@ module RipperRubyParser
         cvar: :cvasgn
       }.freeze
 
-      def create_assignment_sub_type lvalue, value
+      def create_assignment_sub_type(lvalue, value)
         s(map_assignment_lvalue_type(lvalue.sexp_type), lvalue[1], value)
       end
 
-      def map_assignment_lvalue_type type
+      def map_assignment_lvalue_type(type)
         @in_method_body && ASSIGNMENT_IN_METHOD_SUB_TYPE_MAP[type] ||
           ASSIGNMENT_SUB_TYPE_MAP[type] ||
           type