ripper2ruby 0.0.1

data/lib/ripper/ruby_builder/events/while.rb

@@ -0,0 +1,27 @@
+class Ripper
+  class RubyBuilder < Ripper::SexpBuilder
+    module While
+      def on_while(expression, statements)
+        rdelim = pop_token(:@end)
+        ldelim = pop_token(:@do)
+        identifier = pop_token(:@while)
+        Ruby::While.new(identifier, expression, statements, ldelim, rdelim)
+      end
+      
+      def on_while_mod(expression, statement)
+        Ruby::WhileMod.new(pop_token(:@while), expression, statement)
+      end
+      
+      def on_until(expression, statements)
+        rdelim = pop_token(:@end)
+        ldelim = pop_token(:@do)
+        identifier = pop_token(:@until)
+        Ruby::Until.new(identifier, expression, statements, ldelim, rdelim)
+      end
+
+      def on_until_mod(expression, statement)
+        Ruby::UntilMod.new(pop_token(:@until), expression, statement)
+      end
+    end
+  end
+end

data/lib/ripper/ruby_builder/queue.rb

@@ -0,0 +1,33 @@
+# When tokens are pushed to the stack they will be pushed to a queue. Tokens
+# that open new constructs in Ruby (parentheses, semicolons, keywords like
+# class, do, if, etc.) will be held in the queue until the next token is
+# pushed. The queue will then empty itself and return the previously queued
+# token together with the currently pushed token.
+#
+# The reason for this is the way Ripper parses Ruby code. The lexer will fire
+# events every time a known token is found. The parser will fire events when
+# known Ruby constructs are completed. Thus often times when a parser event
+# fires the lexer has already pushed the next (opening) token to the stack.
+#
+# Otoh, event handlers responding to a parser event will want to check for
+# expected tokens (e.g. an opening parentheses for a method call). Thus, when
+# the opening tokens (added by lexer events) are held in a queue while the
+# parser event is fired it will be easier to pop the right tokens belonging
+# to the parser event from the stack.
+
+class Ripper
+  class RubyBuilder < Ripper::SexpBuilder
+    class Queue < ::Array
+      def <<(token)
+        result = [shift]
+        if token.nil?
+        elsif token.opener?
+          push(token)
+        else
+          result << token
+        end
+        result.compact
+      end
+    end
+  end
+end

data/lib/ripper/ruby_builder/stack.rb

@@ -0,0 +1,125 @@
+require 'ripper/ruby_builder/queue'
+require 'ripper/ruby_builder/buffer'
+
+# The stack holds the current "state" of the parser and facilitates communication
+# between lexer events (which fire on known Ruby tokens) and parser events (which
+# fire on known Ruby constructs).
+#
+# E.g. when the Ruby code foo(:bar) is parsed the lexer will fire events for
+# the identifiers 'foo' and 'bar', for the opening and closing parentheses and
+# for the colon . Tt fires the events in the order of the occurence of the
+# tokens from left to right. RubyBuilder pushes all these tokens to the stack.
+#
+# See RubyBuilder::Queue and RubyBuilder::Buffer for what else happens when a
+# token is pushed to the stack.
+#
+# The parser on the other hand will fire events for known Ruby constructs such
+# as the argument list, arguments being added to the argument list, the method
+# call etc. The parser fires these events in the order of Ruby constructs being
+# recognized - i.e. when they are completed. RubyBuilder responds to these
+# events and will pop tokens off from the stack as required (e.g. for
+# constructing an argument list it will try to pop off the corresponding
+# left and right parentheses.)
+#
+# When RubyBuilder pops tokens off from the stack it wants to be careful not to
+# pop off tokens that belong to higher level constructs that haven't yet fired.
+# E.g. for a nested method call foo(bar(1)) the inner call fires first because
+# it completes first. Thus, when RubyBuilder constructs this call it must not
+# pop off the opening parentheses belonging to the outer call (which of course)
+# is already on the stack.
+#
+# For that reason when popping off tokens the stack by default stops searching
+# for the token when an opening token is found. RubyBuilder can force it to
+# search past opening tokens by setting the :pass option to true. Similarly
+# RubyBuilder can set constraints to what tokens it wants to be popped off:
+#
+# :pass    => true   # search past opening tokens
+# :max     => count  # number of tokens
+# :value   => 'foo'  # value of the token
+# :pos     => pos    # position of the token
+# :left    => token  # token must be located right of the given token
+# :right   => token  # token must be located left of the given token
+# :reverse => true   # searches the stack in reverse order (i.e. tokens are shifted)
+
+class Ripper
+  class RubyBuilder < Ripper::SexpBuilder
+    class Stack < ::Array
+      attr_reader :queue, :buffer
+
+      def initialize
+        @queue = Queue.new
+        @buffer = Buffer.new
+      end
+
+      def push(token)
+        return token if buffer.aggregate(token)
+        tokens = queue << token
+        tokens.each do |token|
+          self << token
+        end
+      end
+
+      alias :_pop :pop
+      def pop(*types)
+        options = types.last.is_a?(::Hash) ? types.pop : {}
+        max, pass, revert = options.delete_at(:max, :pass, :reverse)
+        ignored, tokens = [], []
+        reverse! if revert
+
+        while !empty? && !(max && tokens.length >= max)
+          if types.include?(last.type) && matches?(options)
+            tokens << _pop()
+          elsif last.opener? && !pass
+            break
+          else
+            ignored.unshift(_pop())
+          end
+        end
+
+        replace(self + ignored)
+        reverse! if revert
+        tokens
+      end
+
+      protected
+
+        def matches?(conditions)
+          conditions.inject(true) do |result, (type, value)|
+            result && case type
+            when :value
+              has_value?(value)
+            when :pos
+              at?(value)
+            when :right
+              left_of?(value)
+            when :left
+              right_of?(value)
+            end
+          end
+        end
+
+        def at?(pos)
+          pos.nil? || last.position == pos
+        end
+
+        def left_of?(right)
+          right.nil? || last.nil? || last < right
+        end
+
+        def right_of?(left)
+          left.nil? || last.nil? || left < last
+        end
+
+        def has_value?( value)
+          case value
+          when nil
+            true
+          when ::Array
+            value.include?(last.value)
+          else
+            last.value == value
+          end
+        end
+    end
+  end
+end

data/lib/ripper/ruby_builder/token.rb

@@ -0,0 +1,91 @@
+require 'ruby/node/position'
+
+# Tokens are simple value objects that hold the token type, value and position.
+# There are a bunch of helper methods to check the token type and convert the
+# token to Ruby nodes.
+#
+# We mostly operate with Ripper's token types (such as :@ident etc.). For Ripper's
+# sexp types :@kw (keyword) and :@op we use more specific token types based on
+# the sexp's value. E.g. Ripper's sexp [:@op, '+', [0, 0]] would become a token
+# with the type :@+.
+
+class Ripper
+  class RubyBuilder < Ripper::SexpBuilder
+    class Token
+      include Comparable
+
+      attr_accessor :type, :token, :position, :prolog
+
+      def initialize(type = nil, token = nil, position = nil)
+        @type = token_type(type, token)
+        @token = token
+        @position = position if position
+      end
+
+      def newline?
+        NEWLINE.include?(type)
+      end
+
+      def whitespace?
+        WHITESPACE.include?(type)
+      end
+
+      def opener?
+        OPENERS.include?(type)
+      end
+
+      def keyword?
+        KEYWORDS.include?(type)
+      end
+
+      def operator?
+        OPERATORS.include?(type)
+      end
+
+      def separator?
+        SEPARATORS.include?(type)
+      end
+
+      def prolog?
+        whitespace? or separator? or heredoc?
+      end
+
+      def known?
+        keyword? || operator? || opener? || whitespace? || [:@backtick].include?(type)
+      end
+
+      def comment?
+        type == :@comment
+      end
+
+      def heredoc?
+        type == :@heredoc
+      end
+
+      def to_sexp
+        [type, token, [row + 1, column]]
+      end
+
+      def to_identifier
+        Ruby::Identifier.new(token, position, prolog)
+      end
+
+      def <=>(other)
+        position <=> (other.respond_to?(:position) ? other.position : other)
+      end
+
+      protected
+
+        def token_type(type, token)
+          case type
+          when :@kw
+            :"@#{token.gsub(/\W/, '')}"
+          when :@op
+            :"@#{token}"
+          else
+            type
+          end
+        end
+    end
+  end
+end

data/lib/ripper2ruby.rb

@@ -0,0 +1 @@
+require 'ripper/ruby_builder'

data/lib/ruby.rb

@@ -0,0 +1,28 @@
+Dir[File.dirname(__FILE__) + '/ruby/*.rb'].each do |file|
+  require "ruby/#{File.basename(file)}"
+end
+
+# Object oriented representation of Ruby code.
+#
+# The base class is Ruby::Node. It facilitates
+#
+#   * a composite pattern (see Ruby::Node::Composite)
+#   * means for extracting from the original source (see Ruby::Node::Source)
+#
+# There are two main concrete classes derived from Node: Token and Aggregate.
+#
+# Tokens are "atomic" node types that represent non-composite Ruby constructs
+# such as Keyword, Identifier, StringContent and literal types such as integers,
+# floats, true, false, nil etc. Aggregates are composed node types that hold
+# one or many tokens, such as Class, Module, Block, If, For, Case, While etc.
+#
+# Each node type supports the to_ruby method which will return an exact copy
+# of the orginal code it was parsed from.
+#
+# There are also a few helper methods for converting a node to another type
+# (see Ruby::Node::Conversions) and very few helper methods for altering
+# existing code structures (see Ruby::Alternation).
+
+module Ruby
+  include Conversions
+end

data/lib/ruby/aggregate.rb

@@ -0,0 +1,71 @@
+require 'ruby/node'
+
+module Ruby
+  class Aggregate < Node
+    def position(prolog = false)
+      nodes = self.nodes
+      nodes.unshift(self.prolog) if prolog
+      nodes.compact.each { |n| return n.position.dup if n } && nil
+    end
+
+    def position=(position)
+      nodes.each { |n| return n.position = position if n }
+    end
+    
+    def prolog
+      nodes.each { |n| return n.prolog if n } && nil
+    end
+    
+    def prolog=(prolog)
+      nodes.each { |n| return n.prolog = prolog if n }
+    end
+
+    def to_ruby(prolog = false)
+      nodes = self.nodes.compact
+      (nodes.shift.try(:to_ruby, prolog) || '') + nodes.map { |node| node.to_ruby(true) }.join
+    end
+  end
+  
+  class DelimitedAggregate < Aggregate
+    child_accessor :ldelim, :rdelim
+    
+    def initialize(ldelim = nil, rdelim = nil)
+      self.ldelim = ldelim
+      self.rdelim = rdelim
+    end
+  end
+  
+  class NamedAggregate < DelimitedAggregate
+    child_accessor :identifier
+    
+    def initialize(identifier, ldelim = nil, rdelim = nil)
+      self.identifier = identifier
+      super(ldelim, rdelim)
+    end
+  end
+
+  require 'ruby/token'
+  class Variable < Token # TODO join with DelimitedVariable
+  end
+  
+  class DelimitedVariable < DelimitedAggregate
+    child_accessor :identifier
+    
+    def initialize(identifier, ldelim = nil)
+      self.identifier = identifier
+      super(ldelim)
+    end
+    
+    def value
+      identifier.token.to_sym
+    end
+    
+    def nodes
+      [ldelim, identifier].compact
+    end
+
+    def method_missing(method, *args, &block)
+      identifier.respond_to?(method) ? identifier.send(method, *args, &block) : super
+    end
+  end
+end

data/lib/ruby/alternation/args.rb

@@ -0,0 +1,25 @@
+module Ruby
+  module Alternation
+    module ArgsList
+      def options
+        last.arg.is_a?(Ruby::Hash) ? last : nil
+      end
+    
+      def set_option(key, value)
+        if options.nil?
+          # TODO gotta add a separator as well, so maybe better replace the whole options hash
+          self << to_node({key => value}, position.tap { |p| p[1] += length })
+        else
+          options[key] = to_node(value, options[key].position, options[key].prolog)
+        end
+      end
+    
+      protected
+    
+        def to_node(arg, position = nil, prolog = nil)
+          arg = super
+          arg.is_a?(Arg) ? arg : Arg.new(arg)
+        end
+    end
+  end
+end

data/lib/ruby/alternation/hash.rb

@@ -0,0 +1,25 @@
+module Ruby
+  module Alternation
+    module Hash
+      def [](key)
+        each { |assoc| return assoc.value if assoc.key.value == key } or nil
+      end
+    
+      def []=(key, value)
+        value = Ruby.from_native(value, nil, ' ') unless value.is_a?(Node)
+        if assoc = detect { |assoc| assoc.key.value == key }
+          assoc.value = value
+        else
+          # # TODO never happens, fix positions
+          # separators << Token.new(',')
+          # elements << Assoc.new(key, value)
+          # self[key]
+        end
+      end
+    
+      def delete(key)
+        delete_if { |assoc| assoc.key.value == key }
+      end
+    end
+  end
+end

data/lib/ruby/alternation/list.rb

@@ -0,0 +1,19 @@
+module Ruby
+  module Alternation
+    module List
+      def <<(element)
+        elements << element
+        self
+      end
+    
+      def pop
+        [elements.pop]
+      end
+    
+      def []=(ix, element)
+        element = to_node(element, self[ix].position, self[ix].prolog)
+        super
+      end
+    end
+  end
+end