parser 1.1.0 → 1.2.0

data/lib/parser/ruby21.y

@@ -78,17 +78,17 @@ rule
         bodystmt: compstmt opt_rescue opt_else opt_ensure
                     {
                       rescue_bodies     = val[1]
-                      else_,   t_else   = val[2]
-                      ensure_, t_ensure = val[3]
+                      else_t,   else_   = val[2]
+                      ensure_t, ensure_ = val[3]
 
                       if rescue_bodies.empty? && !else_.nil?
-                        diagnostic :warning, :useless_else, t_else
+                        diagnostic :warning, :useless_else, else_t
                       end
 
                       result = @builder.begin_body(val[0],
                                   rescue_bodies,
-                                  else_,   t_else,
-                                  ensure_, t_ensure)
+                                  else_t,   else_,
+                                  ensure_t, ensure_)
                     }
 
         compstmt: stmts opt_terms
@@ -163,11 +163,11 @@ rule
                     }
                 | stmt kWHILE_MOD expr_value
                     {
-                      result = @builder.loop_mod(val[0], val[1], val[2])
+                      result = @builder.loop_mod(:while, val[0], val[1], val[2])
                     }
                 | stmt kUNTIL_MOD expr_value
                     {
-                      result = @builder.loop_mod(val[0], val[1], val[2])
+                      result = @builder.loop_mod(:until, val[0], val[1], val[2])
                     }
                 | stmt kRESCUE_MOD stmt
                     {
@@ -374,7 +374,7 @@ rule
                     }
                 | tLPAREN mlhs_inner rparen
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
 
       mlhs_inner: mlhs_basic
@@ -436,7 +436,7 @@ rule
        mlhs_item: mlhs_node
                 | tLPAREN mlhs_inner rparen
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
 
        mlhs_head: mlhs_item tCOMMA
@@ -992,7 +992,7 @@ rule
                     }
                     opt_nl tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[4])
+                      result = @builder.begin(val[0], val[1], val[4])
                     }
                 | tLPAREN_ARG
                     {
@@ -1000,11 +1000,11 @@ rule
                     }
                     opt_nl tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], nil, val[3])
+                      result = @builder.begin(val[0], nil, val[3])
                     }
                 | tLPAREN compstmt tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
@@ -1098,7 +1098,7 @@ rule
                     }
                     compstmt kEND
                     {
-                      result = @builder.loop(val[0], val[2], val[3],
+                      result = @builder.loop(:while, val[0], val[2], val[3],
                                              val[5], val[6])
                     }
                 | kUNTIL
@@ -1111,16 +1111,24 @@ rule
                     }
                     compstmt kEND
                     {
-                      result = @builder.loop(val[0], val[2], val[3],
+                      result = @builder.loop(:until, val[0], val[2], val[3],
                                              val[5], val[6])
                     }
                 | kCASE expr_value opt_terms case_body kEND
                     {
-                      result = @builder.case(val[0], val[1], val[3], val[4])
+                      *when_bodies, (else_t, else_body) = *val[3]
+
+                      result = @builder.case(val[0], val[1],
+                                             when_bodies, else_t, else_body,
+                                             val[4])
                     }
                 | kCASE            opt_terms case_body kEND
                     {
-                      result = @builder.case(val[0], nil, val[2], val[3])
+                      *when_bodies, (else_t, else_body) = *val[2]
+
+                      result = @builder.case(val[0], nil,
+                                             when_bodies, else_t, else_body,
+                                             val[3])
                     }
                 | kFOR for_var kIN
                     {
@@ -1642,8 +1650,7 @@ opt_block_args_tail:
 
            cases: opt_else
                     {
-                      else_t, else_ = val[0]
-                      result = [ else_ ]
+                      result = [ val[0] ]
                     }
                 | case_body
 
@@ -1674,13 +1681,13 @@ opt_block_args_tail:
 
          exc_var: tASSOC lhs
                     {
-                      result = val
+                      result = [ val[0], val[1] ]
                     }
                 | none
 
       opt_ensure: kENSURE compstmt
                     {
-                      result = val
+                      result = [ val[0], val[1] ]
                     }
                 | none
 

data/lib/parser/runner.rb

@@ -0,0 +1,198 @@
+require 'benchmark'
+require 'find'
+require 'slop'
+
+require 'parser'
+require 'parser/compatibility/slop'
+
+module Parser
+
+  class Runner
+    def self.go(options)
+      new.execute(options)
+    end
+
+    def initialize
+      @slop         = Slop.new(:strict => true)
+      @parser_class = nil
+      @parser       = nil
+      @files        = []
+      @fragments    = []
+
+      @source_count = 0
+      @source_size  = 0
+
+      setup_option_parsing
+    end
+
+    def execute(options)
+      parse_options(options)
+      prepare_parser
+
+      process_all_input
+    end
+
+    private
+
+    def runner_name
+      raise NotImplementedError, "implement #{self.class}##{__callee__}"
+    end
+
+    def setup_option_parsing
+      @slop.banner "Usage: #{runner_name} [options] FILE|DIRECTORY..."
+
+      @slop.on 'h', 'help', 'Display this help message and exit', :tail => true do
+        puts @slop.help
+        puts <<-HELP
+
+  If you specify a DIRECTORY, then all *.rb files are fetched
+  from it recursively and appended to the file list.
+
+  The default parsing mode is for current Ruby (#{RUBY_VERSION}).
+        HELP
+        exit
+      end
+
+      @slop.on 'V', 'version', 'Output version information and exit', :tail => true do
+        puts "#{runner_name} based on parser version #{Parser::VERSION}"
+        exit
+      end
+
+      @slop.on '18', 'Parse as Ruby 1.8.7 would' do
+        require 'parser/ruby18'
+        @parser_class = Parser::Ruby18
+      end
+
+      @slop.on '19', 'Parse as Ruby 1.9.3 would' do
+        require 'parser/ruby19'
+        @parser_class = Parser::Ruby19
+      end
+
+      @slop.on '20', 'Parse as Ruby 2.0.0 would' do
+        require 'parser/ruby20'
+        @parser_class = Parser::Ruby20
+      end
+
+      @slop.on '21', 'Parse as Ruby trunk would (use with caution)' do
+        require 'parser/ruby21'
+        @parser_class = Parser::Ruby21
+      end
+
+      @slop.on 'w',  'warnings',  'Enable warnings'
+
+      @slop.on 'B',  'benchmark', 'Benchmark the processor'
+
+      @slop.on 'e=', 'Process a fragment of Ruby code' do |fragment|
+        @fragments << fragment
+      end
+    end
+
+    def parse_options(options)
+      @slop.parse!(options)
+
+      # Slop has just removed recognized options from `options`.
+      options.each do |file_or_dir|
+        if File.directory?(file_or_dir)
+          Find.find(file_or_dir) do |path|
+            @files << path if path.end_with? '.rb'
+          end
+        else
+          @files << file_or_dir
+        end
+      end
+
+      if @files.empty? && @fragments.empty?
+        $stderr.puts "Need something to parse!"
+        exit 1
+      end
+
+      if @parser_class.nil?
+        require 'parser/current'
+        @parser_class = Parser::CurrentRuby
+      end
+    end
+
+    def prepare_parser
+      @parser = @parser_class.new
+
+      @parser.diagnostics.all_errors_are_fatal = true
+      @parser.diagnostics.ignore_warnings      = !@slop.warnings?
+
+      @parser.diagnostics.consumer = lambda do |diagnostic|
+        puts(diagnostic.render)
+      end
+    end
+
+    def process_all_input
+      input_size = @files.size + @fragments.size
+      if input_size > 1
+        puts "Using #{@parser_class} to parse #{input_size} files."
+      end
+
+      parsing_time =
+        Benchmark.measure do
+          process_fragments
+          process_files
+        end
+
+      if @slop.benchmark?
+        report_with_time(parsing_time)
+      end
+    end
+
+    def process_fragments
+      @fragments.each_with_index do |fragment, index|
+        buffer = Source::Buffer.new("(fragment:#{index})")
+        buffer.source = fragment
+
+        process_buffer(buffer)
+      end
+    end
+
+    def process_files
+      @files.each do |filename|
+        buffer = Parser::Source::Buffer.new(filename)
+        buffer.read
+
+        process_buffer(buffer)
+      end
+    end
+
+    def process_buffer(buffer)
+      @parser.reset
+
+      process(buffer)
+
+      @source_count += 1
+      @source_size  += buffer.source.size
+
+    rescue Parser::SyntaxError
+      # skip
+
+    rescue StandardError
+      $stderr.puts("Failed on: #{buffer.name}")
+      raise
+    end
+
+    def process(buffer)
+      raise NotImplementedError, "implement #{self.class}##{__callee__}"
+    end
+
+    def report_with_time(parsing_time)
+      cpu_time = parsing_time.utime
+
+      speed = '%.3f' % (@source_size / cpu_time / 1000)
+      puts "Parsed #{@source_count} files (#{@source_size} characters)" \
+           " in #{'%.2f' % cpu_time} seconds (#{speed} kchars/s)."
+
+      if defined?(RUBY_ENGINE)
+        engine = RUBY_ENGINE
+      else
+        engine = 'ruby'
+      end
+
+      puts "Running on #{engine} #{RUBY_VERSION}."
+    end
+  end
+
+end

data/lib/parser/runner/ruby_parse.rb

@@ -0,0 +1,87 @@
+require 'parser/runner'
+require 'parser/lexer/explanation'
+
+module Parser
+
+  class Runner::RubyParse < Parser::Runner
+
+    class LocationProcessor < Parser::AST::Processor
+      def process(node)
+        p node
+
+        if node.src.nil?
+          puts "\e[31m[no location info]\e[0m"
+        elsif node.src.expression.nil?
+          puts "\e[31m[location info present but empty]\e[0m"
+        else
+          puts "\e[32m#{node.src.expression.source_line}\e[0m"
+
+          hilight_line = ""
+
+          print_line = lambda do |line|
+            puts line.
+              gsub(/[a-z_]+/) { |m| "\e[1;33m#{m}\e[0m" }.
+              gsub(/~+/)      { |m| "\e[1;35m#{m}\e[0m" }
+          end
+
+          node.src.to_hash.each do |name, range|
+            next if range.nil?
+
+            length    = range.length + 1 + name.length
+            end_col   = range.begin.column + length
+            col_range = range.begin.column...end_col
+
+            if hilight_line.length < end_col
+              hilight_line = hilight_line.ljust(end_col)
+            end
+
+            if hilight_line[col_range] =~ /^\s*$/
+              hilight_line[col_range] = '~' * range.length + " #{name}"
+            else
+              print_line.call(hilight_line)
+              hilight_line = ""
+              redo
+            end
+          end
+
+          print_line.call(hilight_line) unless hilight_line.empty?
+        end
+
+        super
+      end
+    end
+
+    private
+
+    def runner_name
+      'ruby-parse'
+    end
+
+    def setup_option_parsing
+      super
+
+      @slop.on 'L', 'locate',  'Explain how source maps for AST nodes are laid out'
+
+      @slop.on 'E', 'explain', 'Explain how the source is tokenized' do
+        ENV['RACC_DEBUG'] = '1'
+
+        Parser::Base.class_eval do
+          def next_token
+            @lexer.advance_and_explain
+          end
+        end
+      end
+    end
+
+    def process(buffer)
+      ast = @parser.parse(buffer)
+
+      if @slop.locate?
+        LocationProcessor.new.process(ast)
+      elsif !@slop.benchmark?
+        p ast
+      end
+    end
+  end
+
+end

data/lib/parser/runner/ruby_rewrite.rb

@@ -0,0 +1,13 @@
+require 'parser/runner'
+
+module Parser
+
+  class Runner::RubyRewrite < Runner
+    private
+
+    def runner_name
+      'ruby-rewrite'
+    end
+  end
+
+end

data/lib/parser/source/buffer.rb

@@ -72,6 +72,7 @@ module Parser
 
       def source=(source)
         if source.respond_to? :encoding
+          source = source.dup if source.frozen?
           source = self.class.reencode_string(source)
         end
 

data/lib/parser/source/map.rb

@@ -14,6 +14,26 @@ module Parser
 
         freeze
       end
+
+      def with_expression(expression_l)
+        with { |map| map.update_expression(expression_l) }
+      end
+
+      def to_hash
+        Hash[instance_variables.map do |ivar|
+          [ ivar[1..-1].to_sym, instance_variable_get(ivar) ]
+        end]
+      end
+
+      protected
+
+      def with(&block)
+        dup.tap(&block).freeze
+      end
+
+      def update_expression(expression_l)
+        @expression = expression_l
+      end
     end
 
   end