ripper2ruby 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Sven Fuchs <svenfuchs@artweb-design.de>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,10 @@
+Ripper2Ruby
+===========
+
+Similar to ruby2ruby this library allows to parse Ruby code, modify and 
+recompile it back to Ruby.
+
+Differences:
+
+* uses Ripper for parsing (shipped with Ruby 1.9)
+* produces a full object-oriented representation of the Ruby code

data/lib/core_ext/array/flush.rb

@@ -0,0 +1,5 @@
+class Array
+  def flush
+    self.dup.tap { self.clear }
+  end
+end

data/lib/core_ext/hash/delete_at.rb

@@ -0,0 +1,5 @@
+class Hash
+  def delete_at(*keys)
+    keys.map { |key| delete(key) }
+  end
+end

data/lib/core_ext/object/meta_class.rb

@@ -0,0 +1,5 @@
+class Object
+  def meta_class
+    (class << self; self; end)
+  end
+end

data/lib/core_ext/object/try.rb

@@ -0,0 +1,12 @@
+class Object
+  # alias_method :try, :__send__
+  def try(method, *args, &block)
+    send(method, *args, &block) if respond_to?(method)
+  end
+end
+
+class NilClass
+  def try(*args)
+    nil
+  end
+end

data/lib/erb/stripper.rb

@@ -0,0 +1,48 @@
+# replaces html and erb tags with whitespace so that we can parse the result
+# as pure ruby preserving the exact positions of tokens in the original erb
+# source code
+
+require 'erb'
+$KCODE = 'u' if RUBY_VERSION < '1.9'
+
+module Erb
+  class Scanner < ERB::Compiler::Scanner
+    def scan
+      stag_reg = /(.*?)(^[ \t]*<%%|<%%=|<%=|<%#|<%-|<%|\z)/m
+      etag_reg = /(.*?)(%%>|\-%>|%>|\z)/m
+      scanner = StringScanner.new(@src)
+      while !scanner.eos?
+        scanner.scan(@stag ? etag_reg : stag_reg)
+        yield(scanner[1]) unless scanner[1].nil?
+        yield(scanner[2]) unless scanner[2].nil?
+      end
+    end
+  end
+  ERB::Compiler::Scanner.regist_scanner(Scanner, nil, false)
+
+  class Stripper
+    def to_ruby(source)
+      result = ''
+      comment = false
+      scanner = ERB::Compiler.new(nil).make_scanner(source)
+      scanner.scan do |token|
+        comment = true if token == '<%#'
+        if scanner.stag.nil?
+          result << to_whitespace(token)
+          scanner.stag = token if ['<%', '<%%', '<%-', '<%=', '<%%=', '<%#'].include?(token.strip)
+        elsif ['%>', '%%>', '-%>'].include?(token.strip)
+          result << to_whitespace(token.gsub(/>/, ';'))
+          scanner.stag = nil
+        else
+          result << (comment ? to_whitespace(token) : token)
+          comment = false
+        end
+      end
+      result
+    end
+  
+    def to_whitespace(str)
+      str.gsub(/[^\s;]/, ' ')
+    end
+  end
+end

data/lib/highlighters/ansi.rb

@@ -0,0 +1,29 @@
+module Highlighters
+  class Ansi
+    COLORS = { :red =>";31", :yellow => ";33", :green => ";32" }
+    STYLES = { :bold => ";1", :underline => ";4" }
+    
+    def initialize(*formats)
+      @formats = formats
+    end
+    
+    def highlight(text)
+      ansi_format(text, @formats)
+    end
+  
+    def ansi_format(text, formats)
+      res = "\e[0"
+      if formats.is_a?(Array) || formats.is_a?(Hash)
+        COLORS.each { |k,v| res += v if formats.include?(k) }
+        STYLES.each { |k,v| res += v if formats.include?(k) }
+      elsif formats.is_a?(Symbol)
+        COLORS.each { |k,v| res += v if formats == k }
+        STYLES.each { |k,v| res += v if formats == k }
+      elsif formats.respond_to?(:to_sym)
+        COLORS.each { |k,v| res += v if formats.to_sym == k }
+        STYLES.each { |k,v| res += v if formats.to_sym == k }
+      end
+      res += "m" + text.to_s + "\e[0m"
+    end
+  end
+end

data/lib/ripper/event_log.rb

@@ -0,0 +1,45 @@
+require 'ripper'
+require 'highlighters/ansi'
+
+class Ripper
+  class EventLog < Ripper::SexpBuilder
+    class << self
+      def out(src)
+        parser = new(src)
+        parser.parse
+        parser.out
+      end
+    end
+  
+    attr_reader :log
+  
+    def initialize(src)
+      @log = []
+      super
+    end
+    
+    def out
+      log.each do |type, sexp|
+        arg = sexp[1] =~ /\s/ ? sexp[1].inspect : sexp[1]
+        line = (sexp[0].to_s).ljust(20)
+        if type == :scanner
+          puts line + arg[0..30] 
+        else
+          puts highlight(line)
+        end
+      end
+    end
+  
+    def highlight(str)
+      Highlighters::Ansi.new(:bold, :green).highlight(str)
+    end
+  
+    { :scanner => SCANNER_EVENTS, :parser => PARSER_EVENTS }.each do |type, events|
+      events.each do |event|
+        define_method :"on_#{event}" do |*args|
+          log << [type, super(*args)]
+        end
+      end
+    end
+  end
+end

data/lib/ripper/ruby_builder.rb

@@ -0,0 +1,168 @@
+require 'ripper'
+require 'ruby'
+
+require 'core_ext/hash/delete_at'
+require 'core_ext/array/flush'
+
+require 'ripper/ruby_builder/token'
+require 'ripper/ruby_builder/stack'
+
+Dir[File.dirname(__FILE__) + '/ruby_builder/events/*.rb'].each { |file| require file }
+
+# Ripper::RubyBuilder extends Ripper's SexpBuilder and builds a rich, object
+# oriented representation of Ruby code.
+#
+#   code = Ripper::RubyBuilder.build("foo(1, :bar, %w(baz)"), filename)
+#   code.to_ruby # => "foo(1, :bar, %w(baz)"
+#
+# RubyBuilder uses SexpBuilder's lexing and parsing event callbacks (see
+# ruby_builder/events) and builds up Ruby::Nodes which can then be used.
+# See RubyBuilder::Stack, RubyBuilder::Queue, RubyBuilder::Buffer and
+# RubyBuilder::Token for more details about the parsing process.
+
+class Ripper
+  class RubyBuilder < Ripper::SexpBuilder
+    class ParseError < RuntimeError
+    end
+
+    class << self
+      def build(src, filename = nil)
+        new(src, filename).parse
+      end
+    end
+
+    NEWLINE           = [:@nl, :@ignored_nl]
+    WHITESPACE        = [:@sp, :@comment] + NEWLINE
+    OPENERS           = [:@lparen, :@lbracket, :@lbrace, :@class, :@module, :@def, :@begin, :@while, :@until,
+                         :@for, :@if, :@elsif, :@else, :@unless, :@case, :@when, :@embexpr_beg, :@do, :@rescue,
+                         :'@=', :'@::']
+    KEYWORDS          = [:@alias, :@and, :@BEGIN, :@begin, :@break, :@case, :@class, :@def, :@defined,
+                         :@do, :@else, :@elsif, :@END, :@end, :@ensure, :@false, :@for, :@if, :@in,
+                         :@module, :@next, :@nil, :@not, :@or, :@redo, :@rescue, :@retry, :@return,
+                         :@self, :@super, :@then, :@true, :@undef, :@unless, :@until, :@when, :@while,
+                         :@yield]
+
+    SEPARATORS        = [:@semicolon, :@comma]
+
+    UNARY_OPERATORS   = [:'@+', :'@-', :'@!', :'@~', :@not, :'@+@', :'@-@']
+    BINARY_OPERATORS  = [:'@**', :'@*', :'@/', :'@%', :'@+', :'@-', :'@<<', :'@>>', :'@&', :'@|', :'@^',
+                         :'@>', :'@>=', :'@<', :'@<=', :'@<=>', :'@==', :'@===', :'@!=', :'@=~', :'@!~',
+                         :'@&&', :'@||', :@and, :@or]
+    TERNARY_OPERATORS = [:'@?', :'@:']
+    ASSIGN_OPERATORS  = [:'@=', :'@+=', :'@-=', :'@*=', :'@**=', :'@%=', :'@/=', :'@|=', :'@&=', :'@^=',
+                         :'@[]=', :'@||=', :'@&&=', :'@<<=', :'@>>=']
+    ACCESS_OPERATORS  = [:'@[]']
+
+    OPERATORS = UNARY_OPERATORS + BINARY_OPERATORS + TERNARY_OPERATORS + ASSIGN_OPERATORS + ACCESS_OPERATORS
+
+
+    include Lexer, Statements, Const, Method, Call, Block, Args, Assignment, Operator,
+            If, Case, For, While, Identifier, Literal, String, Symbol, Array, Hash
+
+    attr_reader :src, :filename, :stack, :string_stack, :trailing_whitespace
+
+    def initialize(src, filename = nil, lineno = nil)
+      @src = src ||= filename && File.read(filename) || ''
+      @src.gsub!(/([\s\n]*)\Z/) { |s| @trailing_whitespace = Ruby::Whitespace.new(s) and nil }
+
+      @filename = filename
+      @stack = []
+      @stack = Stack.new
+      @string_stack = []
+
+      super
+    end
+
+    protected
+      def position
+        Ruby::Node::Position.new(lineno.to_i - 1, column)
+      end
+
+      def prolog
+        Ruby::Prolog.new(stack.buffer.flush)
+      end
+
+      def push(sexp = nil)
+        token = Token.new(sexp[0], sexp[1], position) if sexp.is_a?(::Array)
+        stack.push(token)
+        token
+      end
+
+      def pop(*args)
+        stack.pop(*args)
+      end
+
+      def pop_token(*types)
+        options = types.last.is_a?(::Hash) ? types.pop : {}
+        options[:max] = 1
+        pop_tokens(*types << options).first
+      end
+
+      def pop_tokens(*types)
+        pop(*types).map { |token| build_token(token) }.flatten.compact
+      end
+
+      def pop_identifier(type, options = {})
+        pop_token(type, options).to_identifier
+      end
+
+      def pop_string_content
+        pop_token(:@tstring_content).to_string_content
+      end
+
+      def pop_operator(options = {})
+        pop_token(*OPERATORS, options)
+      end
+
+      def pop_unary_operator(options = {})
+        pop_token(*UNARY_OPERATORS, options)
+      end
+
+      def pop_binary_operator(options = {})
+        pop_token(*BINARY_OPERATORS, options)
+      end
+
+      def pop_ternary_operator(options = {})
+        pop_token(*TERNARY_OPERATORS, options)
+      end
+
+      def pop_assignment_operator(options = {})
+        pop_token(*ASSIGN_OPERATORS, options)
+      end
+      
+      def pop_end_data
+        token = pop_token(:@__end__)
+        token.token = Ruby::Node::Text::Clip.new(src, token.position).to_s if token # TODO clean up
+        token
+      end
+
+      def build_token(token)
+        Ruby::Token.new(token.token, token.position, token.prolog) if token
+      end
+
+      def build_keyword(token)
+        klass = Ruby.const_get(token.token[0].upcase + token.token[1..-1])
+        klass.new(token, token.position, token.prolog)
+      rescue NameError
+        Ruby::Keyword.new(token, token.position, token.prolog)
+      end
+
+      def build_xstring(token)
+        case token.type
+        when :@symbeg
+          Ruby::DynaSymbol.new(nil, build_token(token))
+        when :@regexp_beg
+          Ruby::Regexp.new(nil, build_token(token))
+        else
+          Ruby::ExecutableString.new(nil, build_token(token))
+        end
+      end
+
+      # def extract_src(from, to)
+      #   lines = Ruby::Node::Text.split(src)
+      #   lines[from.row] = lines[from.row][from.col..-1] # || ''
+      #   lines[to.row] = lines[to.row][0, to.col]
+      #   lines[from.row..to.row].join
+      # end
+  end
+end

data/lib/ripper/ruby_builder/buffer.rb

@@ -0,0 +1,34 @@
+# When tokens are pushed to the stack they may first be buffered when they
+# belong to the Prolog part of a Ruby::Node. Buffered tokens will then be
+# aggregated to the Prolog of the next token that is not buffered. Tokens
+# belonging to the Prolog part of a Ruby::Node are whitespace, separator and
+# heredoc tokens.
+#
+# E.g. when a whitespace char (" ") is pushed to the stack it will be buffered.
+# Then when an :@ident token is pushed to the stack the contents of the buffer
+# will be assigned to the Prolog of the :@ident token. Thus the whitespace char
+# ends up in the Prolog of the :@ident token.
+
+class Ripper
+  class RubyBuilder < Ripper::SexpBuilder
+    class Buffer < Array
+      def aggregate(token)
+        if token.nil?
+          false
+        elsif token.whitespace?
+          self << Ruby::Whitespace.new(token.token, token.position)
+          true
+        elsif token.separator?
+          self << Ruby::Token.new(token.token, token.position)
+          true
+        elsif token.heredoc?
+          self << token.token
+          true
+        else
+          token.prolog = Ruby::Prolog.new(flush) unless empty?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/ripper/ruby_builder/events/args.rb

@@ -0,0 +1,40 @@
+class Ripper
+  class RubyBuilder < Ripper::SexpBuilder
+    module Args
+      def on_method_add_arg(call, args)
+        call.arguments = args
+        call
+      end
+
+      def on_arg_paren(args)
+        args ||= Ruby::ArgsList.new
+        args.rdelim ||= pop_token(:@rparen)
+        args.ldelim ||= pop_token(:@lparen)
+        args
+      end
+      
+      def on_args_add_block(args, block)
+        args << Ruby::Arg.new(block, pop_token(:'@&')) if block
+        args
+      end
+      
+      def on_args_add_star(args, arg)
+        args << Ruby::Arg.new(arg, pop_token(:'@*', :pass => true))
+        args
+      end
+
+      def on_args_add(args, arg)
+        args << arg
+        args
+      end
+      
+      def on_blockarg(identifier)
+        Ruby::Arg.new(identifier, pop_token(:'@&'))
+      end
+
+      def on_args_new
+        Ruby::ArgsList.new
+      end
+    end
+  end
+end

data/lib/ripper/ruby_builder/events/array.rb

@@ -0,0 +1,71 @@
+class Ripper
+  class RubyBuilder < Ripper::SexpBuilder
+    module Array
+      def on_array(args)
+        rdelim = pop_token(:@rbracket)
+        ldelim = pop_token(:@lbracket)
+        args ? args.to_array(ldelim, rdelim) : Ruby::Array.new(nil, ldelim, rdelim)
+      end
+
+      def on_words_new(*args)
+        rdelim = pop_token(:@words_end)
+        ldelim = pop_token(:@words_beg)
+        words = Ruby::Array.new(nil, ldelim, rdelim)
+        string_stack << words
+        words
+      end
+
+      def on_words_add(array, word)
+        array.rdelim ||= pop_token(:@words_end)
+        array << word
+        array
+      end
+
+      def on_qwords_new(*args)
+        rdelim = pop_token(:@words_end)
+        ldelim = pop_token(:@qwords_beg)
+        words = Ruby::Array.new(nil, ldelim, rdelim)
+        string_stack << words unless rdelim
+        words
+      end
+
+      def on_qwords_add(array, word)
+        word = on_word_add(on_word_new, word) # simulating missing on_word_new and on_word_add events
+        array.rdelim ||= pop_token(:@words_end)
+        array << word
+        array
+      end
+
+      def on_words_end(rdelim = nil)
+        words = string_stack.pop
+        words.rdelim ||= pop_token(:@tstring_end, :@words_sep)
+        words
+      end
+
+      def on_aref(target, args)
+        args ||= Ruby::ArgsList.new
+        args.ldelim ||= pop_token(:@lbracket, :left => target)
+        args.rdelim ||= pop_token(:@rbracket, :reverse => true, :pass => true, :left => args.ldelim)
+        Ruby::Call.new(target, nil, nil, args)
+      end
+
+      def on_aref_field(target, args)
+        args ||= Ruby::ArgsList.new
+        args.ldelim ||= pop_token(:@lbracket, :left => target)
+        args.rdelim ||= pop_token(:@rbracket, :reverse => true, :pass => true, :left => args.ldelim)
+        Ruby::Call.new(target, nil, nil, args)
+      end
+      
+      protected
+      
+        WORD_DELIMITER_MAP = { '(' => ')', '[' => ']', '{' => '}' }
+      
+        def closes_words?(token)
+          return false unless string_stack.last.is_a?(Ruby::Array)
+          return false unless string_stack.last.ldelim.token =~ /^%w\s*([^\s]*)/i
+          (WORD_DELIMITER_MAP[$1] || $1) == token.gsub(/[%w\s]/i, '')
+        end
+    end
+  end
+end
+