parser 1.2.0 → 1.3.0

data/lib/parser.rb

@@ -17,6 +17,9 @@ module Parser
   require 'parser/source/buffer'
   require 'parser/source/range'
 
+  require 'parser/source/rewriter'
+  require 'parser/source/rewriter/action'
+
   require 'parser/source/map'
   require 'parser/source/map/operator'
   require 'parser/source/map/collection'
@@ -47,6 +50,8 @@ module Parser
 
   require 'parser/base'
 
+  require 'parser/rewriter'
+
   ERRORS = {
     # Lexer errors
     :unicode_point_too_large => "invalid Unicode codepoint (too large)",

data/lib/parser/builders/default.rb

@@ -339,9 +339,6 @@ module Parser
       when :back_ref, :nth_ref
         message = ERRORS[:backref_assignment]
         diagnostic :error, message, node.src.expression
-
-      else
-        raise NotImplementedError, "build_assignable #{node.inspect}"
       end
     end
 
@@ -375,9 +372,6 @@ module Parser
       when :back_ref, :nth_ref
         message = ERRORS[:backref_assignment]
         diagnostic :error, message, lhs.src.expression
-
-      else
-        raise NotImplementedError, "build op_assign #{lhs.inspect}"
       end
     end
 
@@ -664,7 +658,7 @@ module Parser
     def condition(cond_t, cond, then_t,
                   if_true, else_t, if_false, end_t)
       n(:if, [ check_condition(cond), if_true, if_false ],
-        condition_map(cond_t, then_t, if_true, else_t, if_false, end_t))
+        condition_map(cond_t, cond, then_t, if_true, else_t, if_false, end_t))
     end
 
     def condition_mod(if_true, if_false, cond_t, cond)
@@ -687,7 +681,7 @@ module Parser
 
     def case(case_t, expr, when_bodies, else_t, else_body, end_t)
       n(:case, [ expr, *(when_bodies << else_body)],
-        condition_map(case_t, nil, nil, else_t, else_body, end_t))
+        condition_map(case_t, expr, nil, nil, else_t, else_body, end_t))
     end
 
     # Loops
@@ -891,7 +885,17 @@ module Parser
     end
 
     def string_part_map(string_t)
-      Source::Map::Collection.new(loc(string_t).begin, loc(string_t).end,
+      str_range = loc(string_t)
+
+      begin_l = Source::Range.new(str_range.source_buffer,
+                                  str_range.begin_pos,
+                                  str_range.begin_pos + 1)
+
+      end_l   = Source::Range.new(str_range.source_buffer,
+                                  str_range.end_pos - 1,
+                                  str_range.end_pos)
+
+      Source::Map::Collection.new(begin_l, end_l,
                                   loc(string_t))
     end
 
@@ -1052,17 +1056,19 @@ module Parser
                                join_exprs(pre_e, post_e))
     end
 
-    def condition_map(keyword_t, begin_t, body_e, else_t, else_e, end_t)
+    def condition_map(keyword_t, cond_e, begin_t, body_e, else_t, else_e, end_t)
       if end_t
         end_l = loc(end_t)
       elsif else_e && else_e.src.expression
         end_l = else_e.src.expression
-      elsif else_t
+      elsif loc(else_t)
         end_l = loc(else_t)
       elsif body_e.src.expression
         end_l = body_e.src.expression
-      else
+      elsif loc(begin_t)
         end_l = loc(begin_t)
+      else
+        end_l = cond_e.src.expression
       end
 
       Source::Map::Condition.new(loc(keyword_t),
@@ -1084,11 +1090,10 @@ module Parser
     def rescue_body_map(keyword_t, exc_list_e, assoc_t,
                         exc_var_e, then_t,
                         compstmt_e)
-      end_l = compstmt_e.src.expression ||
-              loc(then_t) ||
-              exc_var_e.src.expression  ||
-              exc_list_e.src.expression ||
-              loc(keyword_t)
+      end_l = compstmt_e.src.expression || loc(then_t)
+      end_l = exc_var_e.src.expression  if end_l.nil? && exc_var_e
+      end_l = exc_list_e.src.expression if end_l.nil? && exc_list_e
+      end_l = loc(keyword_t)            if end_l.nil?
 
       Source::Map::RescueBody.new(loc(keyword_t), loc(assoc_t), loc(then_t),
                                   loc(keyword_t).join(end_l))
@@ -1097,7 +1102,11 @@ module Parser
     def eh_keyword_map(compstmt_e, keyword_t, body_es,
                        else_t, else_e)
       if synthesized_nil?(compstmt_e)
-        begin_l = loc(keyword_t)
+        if keyword_t.nil?
+          begin_l = body_es.first.src.expression
+        else
+          begin_l = loc(keyword_t)
+        end
       else
         begin_l = compstmt_e.src.expression
       end
@@ -1132,7 +1141,8 @@ module Parser
     end
 
     def loc(token)
-      token[1] if token
+      # Pass through `nil`s and return nil for tNL.
+      token[1] if token && token[0]
     end
 
     def diagnostic(type, message, location, highlights=[])

data/lib/parser/lexer.rl

@@ -240,7 +240,7 @@ class Parser::Lexer
   end
 
   def range(s = @ts, e = @te)
-    Parser::Source::Range.new(@source_buffer, s, e - 1)
+    Parser::Source::Range.new(@source_buffer, s, e)
   end
 
   def emit(type, value = tok, s = @ts, e = @te)
@@ -993,7 +993,8 @@ class Parser::Lexer
            fnext expr_endfn; fbreak; };
 
       constant
-      => { emit(:tCONSTANT) };
+      => { emit(:tCONSTANT)
+           fnext expr_endfn; fbreak; };
 
       bareword [?=!]?
       => { emit(:tIDENTIFIER)

data/lib/parser/lexer/explanation.rb

@@ -20,9 +20,9 @@ module Parser
       from, to = range.begin.column, range.end.column
 
       line = range.source_line
-      line[from..to] = "\e[4m#{line[from..to]}\e[0m"
+      line[from...to] = "\e[4m#{line[from...to]}\e[0m"
 
-      tail_len   = to - from
+      tail_len   = to - from - 1
       tail       = "~" * (tail_len >= 0 ? tail_len : 0)
       decoration =  "#{" " * from}\e[1;31m^#{tail}\e[0m #{token} ".
                         ljust(70) + info

data/lib/parser/rewriter.rb

@@ -0,0 +1,34 @@
+module Parser
+  class Rewriter < Parser::AST::Processor
+    def rewrite(source_buffer, ast)
+      @source_rewriter = Source::Rewriter.new(source_buffer)
+
+      process(ast)
+
+      @source_rewriter.process
+    end
+
+    private
+
+    def assignment?(node)
+      [:lvasgn, :ivasgn, :gvasgn,
+       :cvasgn, :cvdecl, :cdecl].include?(node.type)
+    end
+
+    def remove(range)
+      @source_rewriter.remove(range)
+    end
+
+    def insert_before(range, content)
+      @source_rewriter.insert_before(range, content)
+    end
+
+    def insert_after(range, content)
+      @source_rewriter.insert_after(range, content)
+    end
+
+    def replace(range, content)
+      @source_rewriter.replace(range, content)
+    end
+  end
+end

data/lib/parser/runner.rb

@@ -123,12 +123,11 @@ module Parser
       end
     end
 
-    def process_all_input
-      input_size = @files.size + @fragments.size
-      if input_size > 1
-        puts "Using #{@parser_class} to parse #{input_size} files."
-      end
+    def input_size
+      @files.size + @fragments.size
+    end
 
+    def process_all_input
       parsing_time =
         Benchmark.measure do
           process_fragments

data/lib/parser/runner/ruby_parse.rb

@@ -14,37 +14,79 @@ module Parser
         elsif node.src.expression.nil?
           puts "\e[31m[location info present but empty]\e[0m"
         else
-          puts "\e[32m#{node.src.expression.source_line}\e[0m"
-
-          hilight_line = ""
-
-          print_line = lambda do |line|
-            puts line.
-              gsub(/[a-z_]+/) { |m| "\e[1;33m#{m}\e[0m" }.
-              gsub(/~+/)      { |m| "\e[1;35m#{m}\e[0m" }
+          source_line_no = nil
+          source_line    = ""
+          hilight_line   = ""
+
+          print_line = lambda do
+            unless hilight_line.empty?
+              puts hilight_line.
+                gsub(/[a-z_]+/) { |m| "\e[1;33m#{m}\e[0m" }.
+                gsub(/[~.]+/)   { |m| "\e[1;35m#{m}\e[0m" }
+              hilight_line = ""
+            end
           end
 
-          node.src.to_hash.each do |name, range|
-            next if range.nil?
-
-            length    = range.length + 1 + name.length
-            end_col   = range.begin.column + length
-            col_range = range.begin.column...end_col
+          print_source = lambda do |range|
+            source_line = range.source_line
+            puts "\e[32m#{source_line}\e[0m"
+            source_line
+          end
 
-            if hilight_line.length < end_col
-              hilight_line = hilight_line.ljust(end_col)
+          node.src.to_hash.
+            sort_by do |name, range|
+              [(range ? range.line : 0),
+               (name == :expression ? 1 : 0)]
+            end.
+            each do |name, range|
+              next if range.nil?
+
+              if source_line_no != range.line
+                print_line.call()
+                source_line    = print_source.call(range)
+                source_line_no = range.line
+              end
+
+              beg_col = range.begin.column
+
+              if beg_col + range.length > source_line.length
+                multiline    = true
+                range_length = source_line.length - beg_col + 3
+              else
+                multiline    = false
+                range_length = range.length
+              end
+
+              length  = range_length + 1 + name.length
+              end_col = beg_col + length
+
+              if beg_col > 0
+                col_range = (beg_col - 1)...end_col
+              else
+                col_range = beg_col...end_col
+              end
+
+              if hilight_line.length < end_col
+                hilight_line = hilight_line.ljust(end_col)
+              end
+
+              if hilight_line[col_range] =~ /^\s*$/
+                if multiline
+                  tail = ('~' * (source_line.length - beg_col)) + '...'
+                else
+                  tail = '~' * range_length
+                end
+
+                tail = ' ' + tail if beg_col > 0
+
+                hilight_line[col_range] = tail + " #{name}"
+              else
+                print_line.call
+                redo
+              end
             end
 
-            if hilight_line[col_range] =~ /^\s*$/
-              hilight_line[col_range] = '~' * range.length + " #{name}"
-            else
-              print_line.call(hilight_line)
-              hilight_line = ""
-              redo
-            end
-          end
-
-          print_line.call(hilight_line) unless hilight_line.empty?
+          print_line.call
         end
 
         super
@@ -73,6 +115,14 @@ module Parser
       end
     end
 
+    def process_all_input
+      super
+
+      if input_size > 1
+        puts "Using #{@parser_class} to parse #{input_size} files."
+      end
+    end
+
     def process(buffer)
       ast = @parser.parse(buffer)
 

data/lib/parser/runner/ruby_rewrite.rb

@@ -1,13 +1,93 @@
 require 'parser/runner'
+require 'tempfile'
 
 module Parser
 
   class Runner::RubyRewrite < Runner
+    def initialize
+      super
+
+      @rewriters = []
+    end
+
     private
 
     def runner_name
       'ruby-rewrite'
     end
+
+    def setup_option_parsing
+      super
+
+      @slop.on 'l=', 'load=', 'Load a rewriter' do |file|
+        load_and_discover(file)
+      end
+    end
+
+    def load_and_discover(file)
+      load file
+
+      const_name = file.
+        sub(/\.rb$/, '').
+        gsub(/(^|_)([a-z])/) do |m|
+          "#{$2.upcase}"
+        end
+
+      @rewriters << Object.const_get(const_name)
+    end
+
+    def process(initial_buffer)
+      buffer = initial_buffer
+
+      @rewriters.each do |rewriter_class|
+        @parser.reset
+        ast = @parser.parse(buffer)
+
+        rewriter = rewriter_class.new
+        new_source = rewriter.rewrite(buffer, ast)
+
+        new_buffer = Source::Buffer.new(initial_buffer.name +
+                                    '|after ' + rewriter_class.name)
+        new_buffer.source = new_source
+
+        @parser.reset
+        new_ast = @parser.parse(new_buffer)
+
+        unless ast == new_ast
+          $stderr.puts "ASTs do not match:"
+
+          old = Tempfile.new('old')
+          old.write ast.inspect + "\n"; old.flush
+
+          new = Tempfile.new('new')
+          new.write new_ast.inspect + "\n"; new.flush
+
+          IO.popen("diff -u #{old.path} #{new.path}") do |io|
+            diff = io.read.
+              sub(/^---.*/,    "--- #{buffer.name}").
+              sub(/^\+\+\+.*/, "+++ #{new_buffer.name}")
+
+            $stderr.write diff
+          end
+
+          exit 1
+        end
+
+        buffer = new_buffer
+      end
+
+      if File.exist?(buffer.name)
+        File.open(buffer.name, 'w') do |file|
+          file.write buffer.source
+        end
+      else
+        if input_size > 1
+          puts "Rewritten content of #{buffer.name}:"
+        end
+
+        puts buffer.source
+      end
+    end
   end
 
 end

data/lib/parser/source/map.rb

@@ -15,6 +15,10 @@ module Parser
         freeze
       end
 
+      def line
+        @expression.line
+      end
+
       def with_expression(expression_l)
         with { |map| map.update_expression(expression_l) }
       end

data/lib/parser/source/range.rb

@@ -21,7 +21,7 @@ module Parser
       end
 
       def size
-        @end_pos - @begin_pos + 1
+        @end_pos - @begin_pos
       end
 
       alias length size
@@ -39,13 +39,25 @@ module Parser
       end
 
       def column_range
-        self.begin.column..self.end.column
+        self.begin.column...self.end.column
       end
 
       def source_line
         @source_buffer.source_line(line)
       end
 
+      def to_source
+        source_line[column_range]
+      end
+
+      def is?(*what)
+        what.include?(to_source)
+      end
+
+      def to_a
+        (@begin_pos...@end_pos).to_a
+      end
+
       def to_s
         line, column = @source_buffer.decompose_position(@begin_pos)
 
@@ -55,11 +67,18 @@ module Parser
       def join(other)
         Range.new(@source_buffer,
             [@begin_pos, other.begin_pos].min,
-            [@end_pos, other.end_pos].max)
+            [@end_pos,   other.end_pos].max)
+      end
+
+      def ==(other)
+        other.is_a?(Range) &&
+          @source_buffer == other.source_buffer &&
+          @begin_pos     == other.begin_pos     &&
+          @end_pos       == other.end_pos
       end
 
       def inspect
-        "#<Source::Range #{@source_buffer.name} #{@begin_pos}..#{@end_pos}>"
+        "#<Source::Range #{@source_buffer.name} #{@begin_pos}...#{@end_pos}>"
       end
     end