sourcify 0.1.0

data/lib/sourcify/proc/lexer18.rb

@@ -0,0 +1,224 @@
+require 'irb/ruby-lex'
+require 'irb/ruby-token'
+
+module Sourcify
+  module Proc
+
+    class Lexer18
+
+      # Implementation of this class has been inspired by the discussion at
+      # http://www.justskins.com/forums/breaking-ruby-code-into-117453.html
+
+      include Lexer::Commons
+
+      def initialize(io, file, line)
+        @file, @line, @io, @pos = file, line, io, io.pos
+        @lex = RubyLex.new
+        @lex.set_input(@io)
+        @lex.get_readed
+      end
+
+      def lex
+        (@tokens = []).extend(Extensions)
+        @magic_lines = []
+
+        while @tk = @lex.token
+          tkc = @tk.class.to_s.sub(/\ARubyToken::/, '').downcase.to_sym
+          @tokens << [@tk.line_no, @tk.char_no, tkc]
+          post_qstring if @qstring_data
+          send(:"on_#{tkc}") rescue NoMethodError
+          @lex.get_readed if tkc == :tknl
+        end
+      end
+
+      def on_tknl
+        raise EndOfLine unless @results.empty?
+      end
+
+      def on_tk__line__
+        @magic_lines << [@tk.seek, @tk.line_no + @line]
+      end
+
+      def on_tkdstring
+        @qstring_data = [@tk.seek, @tk.line_no + @line]
+      end
+
+      alias_method :on_tkstring, :on_tkdstring   # heredoc
+      alias_method :on_tkdregexp, :on_tkdstring  # regexp
+      alias_method :on_tkdxstring, :on_tkdstring # ` command
+
+      def post_qstring
+        seek, line = @qstring_data
+        @magic_lines << [(seek .. @tk.seek), line]
+        @qstring_data = nil
+      end
+
+      def on_tkdo
+        if !@do_end_counter.started?
+          @do_end_counter.marker = @tk.seek
+          @do_end_counter.increment_start
+        elsif @tokens.same_as_curr_line.keywords(:tkfor, :tkwhile, :tkuntil).empty?
+          # It is possible for a 'for', 'while' or 'until' to have an attached 'do',
+          # for such a case, we want to skip it
+          @do_end_counter.increment_start
+        end
+      end
+
+      def on_tkend
+        if @do_end_counter.started? && @do_end_counter.increment_end.telly?
+          @result = grab_result_and_reset_lex(@do_end_counter.marker, 3)
+          @is_multiline_block = @tokens.multiline?
+          raise EndOfBlock
+        end
+      end
+
+      def on_tkclass
+        # Pretty straightforward for these, each of them will consume an 'end' close it
+        @do_end_counter.increment_start if @do_end_counter.started?
+      end
+
+      # These work the same as 'class', the exception is 'for', which can have an optional
+      # 'do' attached:
+      # * for a in [1,2] do ... end
+      # * for a in [1,2] \n ... end
+      %w{def module begin case for}.each{|kw| alias_method :"on_tk#{kw}", :on_tkclass }
+
+      def on_tkwhile
+        # This has optional trailing 'do', and can work as a modifier as well, eg:
+        # * while true do ... end # => 'do' must be on the same line as 'while'
+        # * while true \n ... end
+        # * ... while true # => 'while' is pre-pended with non-spaces
+        if @do_end_counter.started? && @tokens.start_of_line?
+          @do_end_counter.increment_start
+        end
+      end
+
+      # These work exactly the same as 'while'.
+      %w{until if unless}.each{|kw| alias_method :"on_tk#{kw}", :on_tkwhile }
+
+      def on_tklbrace
+        unless @do_end_counter.started?
+          @braced_counter.marker = @tk.seek unless @braced_counter.started?
+          @braced_counter.increment_start
+        end
+      end
+
+      def on_tkrbrace
+        if @braced_counter.started? && @braced_counter.increment_end.telly?
+          @result = grab_result_and_reset_lex(@braced_counter.marker, 1)
+          @is_multiline_block = @tokens.multiline?
+          raise EndOfBlock
+        end
+      end
+
+      alias_method :on_tkflbrace, :on_tklbrace
+      alias_method :on_tkfrbrace, :on_tkrbrace
+
+      def on_tkassoc
+        if @braced_counter.started? && @braced_counter[:start] == 1
+          @braced_counter.decrement_start
+        end
+      end
+
+      def on_tkgt
+        on_tkassoc if @tokens[-2 .. -1].map(&:last) == [:tkassign, :tkgt]
+      end
+
+      def grab_result_and_reset_lex(marker, offset)
+        @io.seek(@pos+marker)
+        diff = @tk.seek - marker + offset
+        result = replace_magic_lines(@io.read(diff), marker)
+        @io.seek(@pos + diff)
+        @lex.set_input(@io)
+        @lex.get_readed
+        result
+      end
+
+      def replace_magic_lines(result, marker, offset = 0)
+        @magic_lines.inject(result) do |rs, (pos,val)|
+          meth = :"replace_magic_line_by_#{pos.class.to_s.downcase}"
+          n_rs = send(meth, rs, marker + offset, pos, val)
+          offset = result.length - n_rs.length
+          n_rs
+        end
+      end
+
+      def replace_magic_line_by_fixnum(rs, offset, pos, val)
+        m = rs.match(/^(.{#{pos - offset}})__LINE__(.*)$/m)
+        m[1] + val.pred.to_s + m[2]
+      end
+
+      def replace_magic_line_by_range(rs, offset, pos, lineno)
+        @io.seek(@pos + pos.begin)
+        subject = @io.read(pos.end - pos.begin)
+        return rs if %w{' %q %w}.any?{|q| subject.start_with?(q) }
+        prepend, append = rs.match(/^(.*?)#{Regexp.quote(subject)}(.*)$/m)[1..2]
+        middle = subject.split("\n").each_with_index do |line, i|
+          line.gsub!(pattern = /(.*?\#\{)__LINE__(\})/) do |s|
+            (m = s.match(pattern)[1..2])[0].end_with?('\\#{') ?
+              s : m.insert(1, (lineno + i).pred.to_s).join
+          end
+        end.join("\n")
+        prepend + middle + append
+      end
+
+      # Ease working with the hybrid token set collected from RubyLex
+      module Extensions
+
+        ROW, COL, TYP = 0, 1, 2
+
+        def same_as_curr_line
+          same_line(curr_line)
+        end
+
+        def multiline?
+          self[0][ROW] != self[-1][ROW]
+        end
+
+        def curr_line
+          curr[ROW]
+        end
+
+        def curr
+          self[-1]
+        end
+
+        def same_line(line)
+          (
+            # ignore the current node
+            self[0..-2].reverse.take_while do |e|
+              if e[TYP] == :tsemi
+                false
+              elsif e[ROW] == line
+                true
+              elsif e[ROW] == line.pred && e[TYP] != :tknl
+                line -= 1
+                true
+              end
+            end.reverse
+          ).extend(Extensions)
+        end
+
+        def keywords(*types)
+          (
+            types = [types].flatten
+            select{|e| types.include?(e[TYP]) }
+          ).extend(Extensions)
+        end
+
+        def non_spaces(*types)
+          (
+            types = [types].flatten
+            reject{|e| types.empty? or types.include?(e[TYP]) }
+          ).extend(Extensions)
+        end
+
+        def start_of_line?
+          same_as_curr_line.non_spaces.empty?
+        end
+
+      end
+
+    end
+  end
+end

data/lib/sourcify/proc/lexer19.rb

@@ -0,0 +1,195 @@
+require 'ripper'
+
+module Sourcify
+  module Proc
+    class Lexer19 < Ripper::Lexer
+
+      include Lexer::Commons
+
+      def on_nl(token)
+        super.tap do |rs|
+          raise EndOfLine unless @results.empty?
+        end
+      end
+
+      alias_method :on_ignored_nl, :on_nl
+
+      def on_kw(token)
+        super.tap do |rs|
+          send(:"on_kw_#{token}", rs) rescue NoMethodError
+        end
+      end
+
+      def on_kw_class(rs)
+        # Pretty straightforward for these, each of them will consume an 'end' close it
+        @do_end_counter.increment_start if @do_end_counter.started?
+      end
+
+      # These work the same as 'class', the exception is 'for', which can have an optional
+      # 'do' attached:
+      # * for a in [1,2] do ... end
+      # * for a in [1,2] \n ... end
+      %w{def module begin case for}.each{|kw| alias_method :"on_kw_#{kw}", :on_kw_class }
+
+      def on_kw_while(rs)
+        # This has optional trailing 'do', and can work as a modifier as well, eg:
+        # * while true do ... end # => 'do' must be on the same line as 'while'
+        # * while true \n ... end
+        # * ... while true # => 'while' is pre-pended with non-spaces
+        if @do_end_counter.started? && (rs.start_of_line? or rs.within_block?)
+          @do_end_counter.increment_start
+        end
+      end
+
+      # These work exactly the same as 'while'.
+      %w{until if unless}.each{|kw| alias_method :"on_kw_#{kw}", :on_kw_while }
+
+      def on_kw_do(rs)
+        if !@do_end_counter.started?
+          rs.extend(Extensions) unless rs.respond_to?(:curr)
+          @do_end_counter.marker = rs.curr
+          @do_end_counter.increment_start
+        elsif rs.same_as_curr_line.keywords(%w{for while until}).empty?
+          # It is possible for a 'for', 'while' or 'until' to have an attached 'do',
+          # for such a case, we want to skip it
+          @do_end_counter.increment_start
+        end
+      end
+
+      def on_kw_end(rs)
+        if @do_end_counter.started? && @do_end_counter.increment_end.telly?
+          @result = rs.to_code(@do_end_counter.marker)
+          @is_multiline_block = rs.multiline?
+          raise EndOfBlock
+        end
+      end
+
+      def on_lbrace(token)
+        super.tap do |rs|
+          unless @do_end_counter.started?
+            rs.extend(Extensions) unless rs.respond_to?(:curr)
+            @braced_counter.marker = rs.curr unless @braced_counter.started?
+            @braced_counter.increment_start
+          end
+        end
+      end
+
+      def on_rbrace(token)
+        super.tap do |rs|
+          if @braced_counter.started? && @braced_counter.increment_end.telly?
+            @result = rs.to_code(@braced_counter.marker)
+            @is_multiline_block = rs.multiline?
+            raise EndOfBlock
+          end
+        end
+      end
+
+      def on_embexpr_beg(token)
+        super.tap do |rs|
+          @braced_counter.increment_start if @braced_counter.started?
+        end
+      end
+
+      def on_op(token)
+        super.tap do |rs|
+          if @braced_counter.started? && token == '=>' && @braced_counter[:start] == 1
+            @braced_counter.decrement_start
+          end
+        end
+      end
+
+      def on_label(token)
+        super.tap do |rs|
+          if @braced_counter.started? && @braced_counter[:start] == 1
+            @braced_counter.decrement_start
+          end
+        end
+      end
+
+      # Ease working with the result set generated by Ripper
+      module Extensions
+
+        POS, TYP, VAL = 0, 1, 2
+        ROW, COL= 0, 1
+
+        def same_as_curr_line
+          same_line(curr_line)
+        end
+
+        def multiline?
+          self[0][POS][ROW] != self[-1][POS][ROW]
+        end
+
+        def curr_line
+          curr[POS][ROW]
+        end
+
+        def curr
+          self[-1]
+        end
+
+        def same_line(line)
+          (
+            # ignore the current node
+            self[0..-2].reverse.take_while do |e|
+              if e[TYP] == :on_semicolon && e[VAL] == ';'
+                false
+              elsif e[POS][ROW] == line
+                true
+              elsif e[TYP] == :on_sp && e[VAL] == "\\\n"
+                line -= 1
+                true
+              end
+            end.reverse
+          ).extend(Extensions)
+        end
+
+        def keywords(*types)
+          (
+            types = [types].flatten.map(&:to_s)
+            select{|e| e[TYP] == :on_kw && (types.empty? or types.include?(e[VAL])) }
+          ).extend(Extensions)
+        end
+
+        def non_spaces(*types)
+          (
+            types = [types].flatten
+            reject{|e| e[TYP] == :on_sp && (types.empty? or types.include?(e[VAL])) }
+          ).extend(Extensions)
+        end
+
+        def start_of_line?
+          same_as_curr_line.non_spaces.empty?
+        end
+
+        def within_block?
+          same_as_curr_line.non_spaces[-1][TYP] == :on_lparen
+        end
+
+        def to_code(marker)
+          heredoc_beg = false # fixing mysteriously missing newline after :on_heredoc_begin
+          self[index(marker) .. -1].map do |e|
+            if e[TYP] == :on_heredoc_beg
+              heredoc_beg = true
+              e[VAL]
+            elsif heredoc_beg && e[TYP] != :on_nl
+              heredoc_beg = false
+              "\n" + e[VAL]
+            else
+              heredoc_beg = false
+              if e[TYP] == :on_label
+                ':%s => ' % e[VAL][0..-2]
+              elsif e[TYP] == :on_kw && e[VAL] == '__LINE__'
+                e[POS][ROW]
+              else
+                e[VAL]
+              end
+            end
+          end.join
+        end
+
+      end
+
+    end
+  end
+end

data/lib/sourcify/proc/parser.rb

@@ -0,0 +1,74 @@
+require 'sourcify/proc/lexer'
+require 'sourcify/proc/counter'
+
+module Sourcify
+  module Proc
+    class Parser
+
+      RUBY_PARSER = RubyParser.new
+      RUBY_2_RUBY = Ruby2Ruby.new
+
+      def initialize(_proc)
+        @binding, @arity = _proc.binding, _proc.arity
+        @file, @line = _proc.source_location
+      end
+
+      def source
+        RUBY_2_RUBY.process(sexp)
+      end
+
+      def sexp
+        @sexp ||= (
+          raw_sexp = RUBY_PARSER.parse(raw_source, @file)
+          Sexp.from_array(replace_with_lvars(raw_sexp.to_a))
+        )
+      end
+
+      private
+
+        def raw_source
+          @raw_source ||= (
+            frags = Sourcify::Proc::Lexer.new(raw_source_io, @file, @line).work.
+              select{|frag| eval('proc ' + frag).arity == @arity }
+            raise MultipleMatchingProcsPerLineError if frags.size > 1
+            'proc %s' % frags[0]
+          )
+        end
+
+        def raw_source_io
+          File.open(@file, 'r') do |fh|
+            fh.extend(File::Tail).forward(@line.pred)
+            StringIO.new(fh.readlines.join, 'r')
+          end
+        end
+
+        def replace_with_lvars(array)
+          return array if [:class, :sclass, :defn, :module].include?(array[0])
+          array.map do |e|
+            if e.is_a?(Array)
+              no_arg_method_call_or_lvar(e) or replace_with_lvars(e)
+            else
+              e
+            end
+          end
+        end
+
+        def no_arg_method_call_or_lvar(e)
+          if represents_no_arg_call?(e)
+            has_as_local_var?(var = e[2]) ? [:lvar, var] : e
+          end
+        end
+
+        def represents_no_arg_call?(e)
+          e.size == 4 && e[0..1] == [:call, nil] &&
+            e[3] == [:arglist] && (var = e[2]).is_a?(Symbol)
+        end
+
+        def has_as_local_var?(var)
+          qvar = (@q ||= (RUBY_VERSION.include?('1.9.') ? ":%s" : "'%s'")) % var
+          @binding.eval("local_variables.include?(#{qvar})")
+        end
+
+    end
+  end
+end