parser 1.1.0 → 1.2.0

data/lib/parser/source/map/block.rb

@@ -0,0 +1,16 @@
+module Parser
+  module Source
+
+    class Map::Block < Map
+      attr_reader :begin
+      attr_reader :end
+
+      def initialize(begin_l, end_l, expression_l)
+        @begin, @end = begin_l, end_l
+
+        super(expression_l)
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/collection.rb

@@ -0,0 +1,16 @@
+module Parser
+  module Source
+
+    class Map::Collection < Map
+      attr_reader :begin
+      attr_reader :end
+
+      def initialize(begin_l, end_l, expression_l)
+        @begin, @end = begin_l, end_l
+
+        super(expression_l)
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/condition.rb

@@ -0,0 +1,19 @@
+module Parser
+  module Source
+
+    class Map::Condition < Map
+      attr_reader :keyword
+      attr_reader :begin
+      attr_reader :else
+      attr_reader :end
+
+      def initialize(keyword_l, begin_l, else_l, end_l, expression_l)
+        @keyword = keyword_l
+        @begin, @else, @end = begin_l, else_l, end_l
+
+        super(expression_l)
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/constant.rb

@@ -0,0 +1,27 @@
+module Parser
+  module Source
+
+    class Map::Constant < Map
+      attr_reader :double_colon
+      attr_reader :name
+      attr_reader :operator
+
+      def initialize(double_colon, name, expression)
+        @double_colon, @name = double_colon, name
+
+        super(expression)
+      end
+
+      def with_operator(operator_l)
+        with { |map| map.update_operator(operator_l) }
+      end
+
+      protected
+
+      def update_operator(operator_l)
+        @operator = operator_l
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/definition.rb

@@ -0,0 +1,21 @@
+module Parser
+  module Source
+
+    class Map::Definition < Map
+      attr_reader :keyword
+      attr_reader :operator
+      attr_reader :name
+      attr_reader :end
+
+      def initialize(keyword_l, operator_l, name_l, end_l)
+        @keyword  = keyword_l
+        @operator = operator_l
+        @name     = name_l
+        @end      = end_l
+
+        super(@keyword.join(@end))
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/for.rb

@@ -0,0 +1,17 @@
+module Parser
+  module Source
+
+    class Map::For < Map
+      attr_reader :keyword, :in
+      attr_reader :begin, :end
+
+      def initialize(keyword_l, in_l, begin_l, end_l, expression_l)
+        @keyword, @in = keyword_l, in_l
+        @begin, @end  = begin_l, end_l
+
+        super(expression_l)
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/keyword.rb

@@ -0,0 +1,18 @@
+module Parser
+  module Source
+
+    class Map::Keyword < Map
+      attr_reader :keyword
+      attr_reader :begin
+      attr_reader :end
+
+      def initialize(keyword_l, begin_l, end_l, expression_l)
+        @keyword     = keyword_l
+        @begin, @end = begin_l, end_l
+
+        super(expression_l)
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/rescue_body.rb

@@ -0,0 +1,19 @@
+module Parser
+  module Source
+
+    class Map::RescueBody < Map
+      attr_reader :keyword
+      attr_reader :assoc
+      attr_reader :begin
+
+      def initialize(keyword_l, assoc_l, begin_l, expression_l)
+        @keyword = keyword_l
+        @assoc   = assoc_l
+        @begin   = begin_l
+
+        super(expression_l)
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/send.rb

@@ -0,0 +1,29 @@
+module Parser
+  module Source
+
+    class Map::Send < Map
+      attr_reader :selector
+      attr_reader :operator
+      attr_reader :begin
+      attr_reader :end
+
+      def initialize(selector_l, begin_l, end_l, expression_l)
+        @selector    = selector_l
+        @begin, @end = begin_l, end_l
+
+        super(expression_l)
+      end
+
+      def with_operator(operator_l)
+        with { |map| map.update_operator(operator_l) }
+      end
+
+      protected
+
+      def update_operator(operator_l)
+        @operator = operator_l
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/ternary.rb

@@ -0,0 +1,16 @@
+module Parser
+  module Source
+
+    class Map::Ternary < Map
+      attr_reader :question
+      attr_reader :colon
+
+      def initialize(question_l, colon_l, expression_l)
+        @question, @colon = question_l, colon_l
+
+        super(expression_l)
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/variable.rb

@@ -0,0 +1,26 @@
+module Parser
+  module Source
+
+    class Map::Variable < Map
+      attr_reader :name
+      attr_reader :operator
+
+      def initialize(name_l, expression_l=name_l)
+        @name = name_l
+
+        super(expression_l)
+      end
+
+      def with_operator(operator_l)
+        with { |map| map.update_operator(operator_l) }
+      end
+
+      protected
+
+      def update_operator(operator_l)
+        @operator = operator_l
+      end
+    end
+
+  end
+end

data/lib/parser/source/range.rb

@@ -3,39 +3,43 @@ module Parser
 
     class Range
       attr_reader :source_buffer
-      attr_reader :begin, :end
+      attr_reader :begin_pos, :end_pos
 
-      def initialize(source_buffer, begin_, end_)
-        @source_buffer = source_buffer
-        @begin, @end = begin_, end_
+      def initialize(source_buffer, begin_pos, end_pos)
+        @source_buffer       = source_buffer
+        @begin_pos, @end_pos = begin_pos, end_pos
 
         freeze
       end
 
-      def size
-        @end - @begin + 1
+      def begin
+        Range.new(@source_buffer, @begin_pos, @begin_pos)
       end
 
-      def line
-        line, _ = @source_buffer.decompose_position(@begin)
+      def end
+        Range.new(@source_buffer, @end_pos, @end_pos)
+      end
 
-        line
+      def size
+        @end_pos - @begin_pos + 1
       end
 
-      def begin_column
-        _, column = @source_buffer.decompose_position(@begin)
+      alias length size
 
-        column
+      def line
+        line, _ = @source_buffer.decompose_position(@begin_pos)
+
+        line
       end
 
-      def end_column
-        _, column = @source_buffer.decompose_position(@end)
+      def column
+        _, column = @source_buffer.decompose_position(@begin_pos)
 
         column
       end
 
       def column_range
-        begin_column..end_column
+        self.begin.column..self.end.column
       end
 
       def source_line
@@ -43,22 +47,19 @@ module Parser
       end
 
       def to_s
-        line, column = @source_buffer.decompose_position(@begin)
+        line, column = @source_buffer.decompose_position(@begin_pos)
+
         [@source_buffer.name, line, column + 1].join(':')
       end
 
       def join(other)
-        if other.source_buffer == @source_buffer
-          Range.new(@source_buffer,
-              [@begin, other.begin].min,
-              [@end, other.end].max)
-        else
-          raise ArgumentError, "Cannot join SourceRanges for different SourceFiles"
-        end
+        Range.new(@source_buffer,
+            [@begin_pos, other.begin_pos].min,
+            [@end_pos, other.end_pos].max)
       end
 
       def inspect
-        "#<Source::Range #{@source_buffer.name} #{@begin}..#{@end}>"
+        "#<Source::Range #{@source_buffer.name} #{@begin_pos}..#{@end_pos}>"
       end
     end
 

data/lib/parser/version.rb

@@ -0,0 +1,3 @@
+module Parser
+  VERSION = '1.2.0'
+end

data/parser.gemspec

@@ -1,8 +1,9 @@
 # encoding: utf-8
+require File.expand_path('../lib/parser/version', __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = 'parser'
-  spec.version       = '1.1.0'
+  spec.version       = Parser::VERSION
   spec.authors       = ['Peter Zotov']
   spec.email         = ['whitequark@whitequark.org']
   spec.description   = %q{A Ruby parser written in pure Ruby.}
@@ -17,11 +18,12 @@ Gem::Specification.new do |spec|
                           lib/parser/ruby20.rb
                           lib/parser/ruby21.rb
                        )
-  spec.executables   = %w()
+  spec.executables   = %w(ruby-parse ruby-rewrite)
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
   spec.add_dependency             'ast',       '~> 1.0'
+  spec.add_dependency             'slop',      '~> 3.4'
 
   spec.add_development_dependency 'bundler',   '~> 1.2'
   spec.add_development_dependency 'rake',      '~> 0.9'

data/test/parse_helper.rb

@@ -45,12 +45,12 @@ module ParseHelper
 
   def assert_source_range(begin_pos, end_pos, range, version, what)
     assert range.is_a?(Parser::Source::Range),
-           "(#{version}) is_a?(Source::Range) for #{what}"
+           "(#{version}) #{range.inspect}.is_a?(Source::Range) for #{what}"
 
-    assert_equal begin_pos, range.begin,
+    assert_equal begin_pos, range.begin_pos,
                  "(#{version}) begin of #{what}"
 
-    assert_equal end_pos, range.end,
+    assert_equal end_pos, range.end_pos,
                  "(#{version}) end of #{what}"
   end
 
@@ -86,10 +86,8 @@ module ParseHelper
           raise "No entity with AST path #{ast_path} in #{parsed_ast.inspect}"
         end
 
-        next # TODO skip location checking
-
         assert astlet.source_map.respond_to?(map_field),
-               "(#{version}) source_map.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
+               "(#{version}) #{astlet.source_map.inspect}.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
 
         range = astlet.source_map.send(map_field)
 
@@ -135,8 +133,6 @@ module ParseHelper
       parse_source_map_descriptions(source_maps) \
           do |begin_pos, end_pos, map_field, ast_path, line|
 
-        next # TODO skip location checking
-
         case map_field
         when 'location'
           assert_source_range begin_pos, end_pos,
@@ -162,7 +158,7 @@ module ParseHelper
        ^(?# $1 skip)            ^(\s*)
         (?# $2 highlight)        ([~\^]+)
                                  \s+
-        (?# $3 source_map_field) ([a-z]+)
+        (?# $3 source_map_field) ([a-z_]+)
         (?# $5 ast_path)         (\s+\(([a-z_.\/0-9]+)\))?
                                 $/
 
@@ -199,7 +195,14 @@ module ParseHelper
     path.inject(ast) do |astlet, path_component|
       # Split "dstr/2" to :dstr and 1
       type_str, index_str = path_component.split('/')
-      type, index = type_str.to_sym, index_str.to_i - 1
+
+      type  = type_str.to_sym
+
+      if index_str.nil?
+        index = 0
+      else
+        index = index_str.to_i - 1
+      end
 
       matching_children = \
         astlet.children.select do |child|

data/test/test_lexer.rb

@@ -27,7 +27,7 @@ class TestLexer < MiniTest::Unit::TestCase
 
   def util_escape(expected, input)
     source_buffer = Parser::Source::Buffer.new('(util_escape)')
-    source_buffer.source = "%Q[\\#{input}]"
+    source_buffer.source = "\"\\#{input}\""
 
     @lex.reset
     @lex.source_buffer = source_buffer
@@ -1277,6 +1277,9 @@ class TestLexer < MiniTest::Unit::TestCase
     util_bad_token "1_e1"
     util_bad_token "1_.1"
     util_bad_token "1__1"
+
+    util_bad_token "1end"
+    util_bad_token "1.1end"
   end
 
   def test_plus_unary_number
@@ -1802,7 +1805,9 @@ class TestLexer < MiniTest::Unit::TestCase
 
   def test_string_double_nested_curlies
     util_lex_token('%{nest{one{two}one}nest}',
-                   :tSTRING, "nest{one{two}one}nest")
+                   :tSTRING_BEG,     '%{',
+                   :tSTRING_CONTENT, "nest{one{two}one}nest",
+                   :tSTRING_END,     '}')
   end
 
   def test_string_double_no_interp
@@ -1820,7 +1825,9 @@ class TestLexer < MiniTest::Unit::TestCase
 
   def test_string_pct_Q
     util_lex_token("%Q[s1 s2]",
-                   :tSTRING, "s1 s2")
+                   :tSTRING_BEG,     '%Q[',
+                   :tSTRING_CONTENT, "s1 s2",
+                   :tSTRING_END,     ']')
   end
 
   def test_string_pct_W
@@ -1873,12 +1880,16 @@ class TestLexer < MiniTest::Unit::TestCase
 
   def test_string_pct_angle
     util_lex_token("%<blah>",
-                   :tSTRING, "blah")
+                   :tSTRING_BEG,     '%<',
+                   :tSTRING_CONTENT, "blah",
+                   :tSTRING_END,     '>')
   end
 
-  def test_string_pct_other
+  def test_string_pct_pct
     util_lex_token("%%blah%",
-                   :tSTRING, "blah")
+                   :tSTRING_BEG,     '%',
+                   :tSTRING_CONTENT, "blah",
+                   :tSTRING_END,     '%')
   end
 
   def test_string_pct_w
@@ -1968,12 +1979,16 @@ class TestLexer < MiniTest::Unit::TestCase
 
   def test_symbol_double
     util_lex_token(":\"symbol\"",
-                   :tSYMBOL, "symbol")
+                   :tSYMBEG,         ":\"",
+                   :tSTRING_CONTENT, "symbol",
+                   :tSTRING_END,     "\"")
   end
 
   def test_symbol_single
     util_lex_token(":'symbol'",
-                   :tSYMBOL, "symbol")
+                   :tSYMBEG,         ":'",
+                   :tSTRING_CONTENT, "symbol",
+                   :tSTRING_END,     "'")
   end
 
   def test_ternary
@@ -2120,11 +2135,15 @@ class TestLexer < MiniTest::Unit::TestCase
   def test_bug_expr_beg_percent
     @lex.state = :expr_beg
     util_lex_token("%=foo=",
-                   :tSTRING, "foo")
+                   :tSTRING_BEG,     "%=",
+                   :tSTRING_CONTENT, 'foo',
+                   :tSTRING_END,     "=")
 
     @lex.state = :expr_beg
     util_lex_token("% = ",
-                   :tSTRING, "=")
+                   :tSTRING_BEG,     "% ",
+                   :tSTRING_CONTENT, '=',
+                   :tSTRING_END,     ' ')
   end
 
   def test_bug_expr_beg_document
@@ -2162,7 +2181,9 @@ class TestLexer < MiniTest::Unit::TestCase
 
     @lex.state = :expr_arg
     util_lex_token(" %[1]",
-                   :tSTRING,     "1")
+                   :tSTRING_BEG,     "%[",
+                   :tSTRING_CONTENT, '1',
+                   :tSTRING_END,     ']')
 
     @lex.state = :expr_arg
     util_lex_token(" %=1=",