ripper_ruby_parser 0.0.1

data/README.rdoc

@@ -0,0 +1,76 @@
+= RipperRubyParser
+
+by Matijs van Zuijlen
+
+http://www.github.com/mvz/ripper_ruby_parser
+
+== Description
+
+Parse with Ripper, produce sexps that are compatible with RubyParser.
+
+== Features/Notes
+
+* Drop-in replacement for RubyParser.
+* Should handle 1.9 syntax gracefully.
+* Only tested with MRI 1.9.3. Definitely does not work with the ripper gem for
+  MRI 1.8.
+* Many edge cases are surely not yet taken care of (but hey, it's version
+  0.0.1).
+
+== Install
+
+* gem install ripper_ruby_parser
+
+== Synopsis
+
+Basic usage:
+
+  require 'ripper_ruby_parser'
+
+  parser = RipperRubyParser::Parser.new
+  result = parser.parse "puts 'Hello World'"
+  p result
+  # => s(:call, nil, :puts, s(:arglist, s(:str, "Hello World!")))
+
+== Requirements
+
+* Ruby 1.9.3. It may work with lower 1.9 versions. Let me know if it does.
+* sexp_processor
+
+== Hacking and contributing
+
+If you want to send pull requests or patches, please:
+
+* Make sure `rake test` runs without reporting any failures. If your code
+  breaks existing stuff, it won't get merged in.
+* Add tests for your feature. Otherwise, I can't see if it works or if I
+  break it later.
+* Make sure latest master merges cleanly with your branch. Things might
+  have moved around since you forked.
+* Try not to include changes that are irrelevant to your feature in the
+  same commit.
+
+== License
+
+(The MIT License)
+
+Copyright (c) 2012 Matijs van Zuijlen
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rake/testtask'
+
+namespace :test do
+
+  Rake::TestTask.new(:unit) do |t|
+    t.libs = ['lib']
+    t.test_files = FileList['test/unit/*_test.rb']
+    t.ruby_opts += ["-w"]
+  end
+
+  Rake::TestTask.new(:end_to_end) do |t|
+    t.libs = ['lib']
+    t.test_files = FileList['test/end_to_end/*_test.rb']
+    t.ruby_opts += ["-w"]
+  end
+
+  task :run => [:unit, :end_to_end]
+
+end
+
+desc 'Alias to test:run'
+task :test => 'test:run'
+
+task :default => :test

data/lib/ripper_ruby_parser/parser.rb

@@ -0,0 +1,18 @@
+require 'ripper'
+require 'ripper_ruby_parser/sexp_processor'
+
+module RipperRubyParser
+  # Main parser class. Brings together Ripper and our
+  # RipperRubyParser::SexpProcessor.
+  class Parser
+    def initialize processor=SexpProcessor.new
+      @processor = processor
+    end
+
+    def parse source
+      exp = Sexp.from_array(Ripper.sexp source)
+      @processor.process exp
+    end
+  end
+end
+

data/lib/ripper_ruby_parser/sexp_ext.rb

@@ -0,0 +1,14 @@
+require 'sexp_processor'
+
+module RipperRubyParser
+  # Extensions to Sexp
+  module SexpExt
+    def fix_empty_type
+      unless sexp_type.is_a? Symbol
+        unshift :__empty
+      end
+    end
+  end
+end
+
+Sexp.send :include, RipperRubyParser::SexpExt

data/lib/ripper_ruby_parser/sexp_handlers/arguments.rb

@@ -0,0 +1,23 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Arguments
+      def process_args_add_block exp
+        _, content, _ = exp.shift 3
+        s(:arglist, *handle_list_with_optional_splat(content))
+      end
+
+      def process_args_add_star exp
+        generic_add_star exp
+      end
+
+      def process_arg_paren exp
+        _, args = exp.shift 2
+        args = s() if args.nil?
+        unless args.first.is_a? Symbol
+          args.unshift :arglist
+        end
+        process(args)
+      end
+    end
+  end
+end

data/lib/ripper_ruby_parser/sexp_handlers/arrays.rb

@@ -0,0 +1,15 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Arrays
+      def process_array exp
+        _, elems = exp.shift 2
+        s(:array, *handle_list_with_optional_splat(elems))
+      end
+
+      def process_aref exp
+        _, item, idx = exp.shift 3
+        s(:call, process(item), :[], process(idx))
+      end
+    end
+  end
+end

data/lib/ripper_ruby_parser/sexp_handlers/assignment.rb

@@ -0,0 +1,80 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Assignment
+      def process_assign exp
+        _, lvalue, value = exp.shift 3
+
+        lvalue = process(lvalue)
+        value = process(value)
+
+        create_assignment_sub_type lvalue, value
+      end
+
+      def process_massign exp
+        _, left, right = exp.shift 3
+
+        left = handle_list_with_optional_splat left
+
+        left.each do |item|
+          case item.sexp_type
+          when :splat
+            item[1][0] = :lasgn
+          else
+            item[0] = :lasgn
+          end
+        end
+
+        right = process(right)
+
+        unless right.sexp_type == :array
+          right = s(:to_ary, right)
+        end
+
+        s(:masgn, s(:array, *left), right)
+      end
+
+      def process_mrhs_new_from_args exp
+        _, inner, last = exp.shift 3
+        inner.map! {|item| process(item)}
+        inner.push process(last)
+        s(:array, *inner)
+      end
+
+      def process_mlhs_add_star exp
+        generic_add_star exp
+      end
+
+      def process_opassign exp
+        _, lvalue, operator, value = exp.shift 4
+
+        lvalue = process(lvalue)
+        value = process(value)
+        operator = operator[1].gsub(/=/, '').to_sym
+        operator_call = s(:call, lvalue, operator, s(:arglist, value))
+
+        case lvalue.sexp_type
+        when :ivar
+          s(:iasgn, lvalue[1], operator_call)
+        when :aref_field
+          s(:op_asgn1, lvalue[1], s(:arglist, lvalue[2][1]), operator, value)
+        else
+          s(:lasgn, lvalue[1], operator_call)
+        end
+      end
+
+      def create_assignment_sub_type lvalue, value
+        case lvalue.sexp_type
+        when :ivar
+          s(:iasgn, lvalue[1], value)
+        when :aref_field
+          s(:attrasgn, lvalue[1], :[]=, s(:arglist, lvalue[2][1], value))
+        when :const
+          s(:cdecl, lvalue[1], value)
+        else
+          s(:lasgn, lvalue[1], value)
+        end
+      end
+    end
+  end
+end
+

data/lib/ripper_ruby_parser/sexp_handlers/blocks.rb

@@ -0,0 +1,49 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Blocks
+      def process_method_add_block exp
+        _, call, block = exp.shift 3
+        block = process(block)
+        args = convert_block_args(block[1])
+        stmt = block[2].first
+        s(:iter, process(call), args, stmt)
+      end
+
+      def process_brace_block exp
+        handle_generic_block exp
+      end
+
+      def process_do_block exp
+        handle_generic_block exp
+      end
+
+      def process_params exp
+        _, normal, defaults, *_ = exp.shift 6
+
+        args = [*normal].map do |id|
+          identifier_node_to_symbol id
+        end
+
+        assigns = [*defaults].map do |pair|
+          sym = identifier_node_to_symbol pair[0]
+          val = process pair[1]
+          s(:lasgn, sym, val)
+        end
+
+        if assigns.length > 0
+          args += assigns.map {|lasgn| lasgn[1]}
+          args << s(:block, *assigns)
+        end
+
+        s(:args, *args)
+      end
+
+      private
+
+      def handle_generic_block exp
+        _, args, stmts = exp.shift 3
+        s(:block, process(args), s(handle_statement_list(stmts)))
+      end
+    end
+  end
+end

data/lib/ripper_ruby_parser/sexp_handlers/conditionals.rb

@@ -0,0 +1,64 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Conditionals
+      def process_if exp
+        _, cond, truepart, falsepart = exp.shift 4
+        s(:if,
+          process(cond),
+          handle_statement_list(truepart),
+          process(falsepart))
+      end
+
+      def process_elsif exp
+        process_if exp
+      end
+
+      def process_if_mod exp
+        _, cond, truepart = exp.shift 3
+        s(:if, process(cond), process(truepart), nil)
+      end
+
+      def process_unless_mod exp
+        _, cond, truepart = exp.shift 3
+        s(:if, process(cond), nil, process(truepart))
+      end
+
+      def process_unless exp
+        _, cond, truepart, falsepart = exp.shift 4
+        s(:if,
+          process(cond),
+          process(falsepart),
+          handle_statement_list(truepart))
+      end
+
+      def process_case exp
+        _, expr, clauses = exp.shift 3
+        s(:case, process(expr), *process(clauses))
+      end
+
+      def process_when exp
+        _, values, truepart, falsepart = exp.shift 4
+
+        if falsepart.nil?
+          falsepart = [nil]
+        else
+          falsepart = process(falsepart)
+          if falsepart.first.is_a? Symbol
+            falsepart = s(falsepart)
+          end
+        end
+
+        s(s(:when,
+            process(s(:array, values)),
+            handle_statement_list(truepart)),
+          *falsepart)
+      end
+
+      def process_else exp
+        _, body = exp.shift 2
+        handle_statement_list body
+      end
+    end
+  end
+end
+

data/lib/ripper_ruby_parser/sexp_handlers/hashes.rb

@@ -0,0 +1,27 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Hashes
+      def process_hash exp
+        _, elems = exp.shift 2
+        s(:hash, *process(elems))
+      end
+
+      def process_assoclist_from_args exp
+        _, elems = exp.shift 2
+        result = s()
+        elems.each {|sub_exp|
+          process(sub_exp).each {|elm|
+            result << elm
+          }
+        }
+        result
+      end
+
+      def process_assoc_new exp
+        _, left, right = exp.shift 3
+        s(process(left), process(right))
+      end
+    end
+  end
+end
+

data/lib/ripper_ruby_parser/sexp_handlers/helper_methods.rb

@@ -0,0 +1,49 @@
+module RipperRubyParser
+  module SexpHandlers
+    module HelperMethods
+      def handle_list_with_optional_splat exp
+        if exp.nil?
+          []
+        elsif exp.first.is_a? Symbol
+          process(exp)
+        else
+          exp.map { |sub_exp| process(sub_exp) }
+        end
+      end
+
+      def convert_block_args(args)
+        args && s(:lasgn, args[1][1])
+      end
+
+      def handle_statement_list exp
+        statements = exp.
+          map { |sub_exp| process(sub_exp) }.
+          reject { |sub_exp| sub_exp.sexp_type == :void_stmt }
+
+        if statements.length == 1
+          statements.first
+        else
+          s(:block, *statements)
+        end
+      end
+
+      def identifier_node_to_symbol exp
+        assert_type exp, :@ident
+        _, ident, _ = exp.shift 3
+
+        ident.to_sym
+      end
+
+      def generic_add_star exp
+        _, args, splatarg = exp.shift 3
+        items = args.map { |sub| process(sub) }
+        items << s(:splat, process(splatarg))
+        s(*items)
+      end
+
+      def is_literal? exp
+        exp.sexp_type == :lit
+      end
+    end
+  end
+end

data/lib/ripper_ruby_parser/sexp_handlers/literals.rb

@@ -0,0 +1,83 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Literals
+      def process_string_literal exp
+        _, content = exp.shift 2
+        process(content)
+      end
+
+      def process_string_content exp
+        _, inner = exp.shift 2
+
+        string = extract_inner_string(inner)
+        rest = []
+
+        if string.sexp_type == :str
+          string = string[1]
+        else
+          rest << string
+          string = ""
+        end
+
+        string = unescape(string)
+
+        until exp.empty? do
+          result = process(exp.shift)
+          if result.sexp_type == :str
+            result[1] = unescape(result[1])
+          end
+          rest << result
+        end
+
+        if rest.empty?
+          s(:str, string)
+        else
+          s(:dstr, string, *rest)
+        end
+      end
+
+      def process_string_embexpr exp
+        _, list = exp.shift 2
+        s(:evstr, process(list.first))
+      end
+
+      def process_regexp_literal exp
+        _, content, _ = exp.shift 3
+
+        string = extract_inner_string content[0]
+
+        s(:lit, Regexp.new(string[1]))
+      end
+
+      def process_symbol_literal exp
+        _, symbol = exp.shift 2
+        sym = symbol[1]
+        s(:lit, extract_node_symbol(sym))
+      end
+
+      def process_dyna_symbol exp
+        _, list = exp.shift 2
+
+        string = process list[0]
+        s(:lit, string[1].to_sym)
+      end
+
+      def process_at_tstring_content exp
+        _, string, _ = exp.shift 3
+        s(:str, string)
+      end
+
+      private
+
+      def extract_inner_string exp
+        process(exp) || s(:str, "")
+      end
+
+      def unescape string
+        string.gsub /(\\[n\\"])/ do
+          eval "\"#{$1}\""
+        end
+      end
+    end
+  end
+end

data/lib/ripper_ruby_parser/sexp_handlers/loops.rb

@@ -0,0 +1,11 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Loops
+      def process_until exp
+        _, cond, block = exp.shift 3
+        s(:until, process(cond), handle_statement_list(block), true)
+      end
+    end
+  end
+end
+

data/lib/ripper_ruby_parser/sexp_handlers/method_calls.rb

@@ -0,0 +1,47 @@
+module RipperRubyParser
+  module SexpHandlers
+    module MethodCalls
+      def process_method_add_arg exp
+        _, call, parens = exp.shift 3
+        call = process call
+        s(:call, call[1], call[2], process(parens))
+      end
+
+      def process_call exp
+        _, receiver, _, method = exp.shift 4
+        s(:call, process(receiver), identifier_node_to_symbol(method), s(:arglist))
+      end
+
+      def process_command exp
+        _, ident, arglist = exp.shift 3
+
+        ident = identifier_node_to_symbol ident
+        arglist = process arglist
+
+        s(:call, nil, ident, arglist)
+      end
+
+      def process_command_call exp
+        _, receiver, _, method, arguments = exp.shift 5
+        s(:call,
+          process(receiver),
+          identifier_node_to_symbol(method),
+          process(arguments))
+      end
+
+      def process_vcall exp
+        _, ident = exp.shift 3
+
+        ident = identifier_node_to_symbol ident
+
+        s(:call, nil, ident, s(:arglist))
+      end
+
+      def process_fcall exp
+        _, method = exp.shift 2
+        s(:call, nil, identifier_node_to_symbol(method), s(:arglist))
+      end
+    end
+  end
+end
+

data/lib/ripper_ruby_parser/sexp_handlers/methods.rb

@@ -0,0 +1,38 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Methods
+      def process_def exp
+        _, ident, params, body = exp.shift 4
+        ident = identifier_node_to_symbol ident
+        s(:defn, ident, process(params), method_body(body))
+      end
+
+      def process_defs exp
+        _, receiver, _, method, args, body = exp.shift 6
+        s(:defs, process(receiver),
+          identifier_node_to_symbol(method),
+          process(args), process(body))
+      end
+
+      def process_return exp
+        _, arglist = exp.shift 2
+        arglist = process(arglist)
+        s(:return, arglist[1])
+      end
+
+      def process_return0 exp
+        _ = exp.shift
+        s(:return)
+      end
+
+      def method_body exp
+        scope = process exp
+        block = scope[1]
+        if block.length == 1
+          block.push s(:nil)
+        end
+        scope
+      end
+    end
+  end
+end

data/lib/ripper_ruby_parser/sexp_handlers/operators.rb

@@ -0,0 +1,41 @@
+module RipperRubyParser
+  module SexpHandlers
+    module Operators
+      OPERATOR_MAP = {
+        "&&".to_sym => :and,
+        "||".to_sym => :or
+      }
+
+      def process_binary exp
+        _, left, op, right = exp.shift 4
+        mapped = OPERATOR_MAP[op]
+        if mapped
+          s(mapped, process(left), process(right))
+        else
+          s(:call, process(left), op, s(:arglist, process(right)))
+        end
+      end
+
+      def process_unary exp
+        _, _, arg = exp.shift 3
+        arg = process(arg)
+        if is_literal? arg
+          s(:lit, -arg[1])
+        else
+          s(:call, arg, :-@, s(:arglist))
+        end
+      end
+
+      def process_dot2 exp
+        _, left, right = exp.shift 3
+        left = process(left)
+        right = process(right)
+        if is_literal?(left) && is_literal?(right)
+          s(:lit, Range.new(left[1], right[1]))
+        else
+          s(:dot2, left, right)
+        end
+      end
+    end
+  end
+end