parser 2.0.0.pre3 → 2.0.0.pre4

data/lib/parser/ruby21.y

@@ -16,7 +16,7 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
       tBACK_REF2 tSYMBEG tSTRING_BEG tXSTRING_BEG tREGEXP_BEG tREGEXP_OPT
       tWORDS_BEG tQWORDS_BEG tSYMBOLS_BEG tQSYMBOLS_BEG tSTRING_DBEG
       tSTRING_DVAR tSTRING_END tSTRING_DEND tSTRING tSYMBOL
-      tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAMBDA tLAMBEG
+      tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAMBDA tLAMBEG tCHARACTER
 
 prechigh
   right    tBANG tTILDE tUPLUS
@@ -1675,6 +1675,10 @@ opt_block_args_tail:
                     {
                       result = @builder.string(val[0])
                     }
+                | tCHARACTER
+                    {
+                      result = @builder.character(val[0])
+                    }
 
          xstring: tXSTRING_BEG xstring_contents tSTRING_END
                     {
@@ -1740,7 +1744,7 @@ opt_block_args_tail:
                     }
                 | qword_list tSTRING_CONTENT tSPACE
                     {
-                      result = val[0] << @builder.string(val[1])
+                      result = val[0] << @builder.string_internal(val[1])
                     }
 
        qsym_list: # nothing
@@ -1749,7 +1753,7 @@ opt_block_args_tail:
                     }
                 | qsym_list tSTRING_CONTENT tSPACE
                     {
-                      result = val[0] << @builder.symbol(val[1])
+                      result = val[0] << @builder.symbol_internal(val[1])
                     }
 
  string_contents: # nothing
@@ -1781,7 +1785,7 @@ regexp_contents: # nothing
 
   string_content: tSTRING_CONTENT
                     {
-                      result = @builder.string(val[0])
+                      result = @builder.string_internal(val[0])
                     }
                 | tSTRING_DVAR string_dvar
                     {

data/lib/parser/source/comment/associator.rb

@@ -2,18 +2,32 @@ module Parser
   module Source
 
     ##
+    #
+    # @!attribute skip_directives
+    #  Skip file processing directives disguised as comments,
+    #  namely:
+    #
+    #    * Shebang line,
+    #    * Magic encoding comment.
+    #
     # @api public
     #
     class Comment::Associator
+      attr_accessor :skip_directives
+
       def initialize(comments, ast)
         @comments    = comments
         @ast         = ast
+
+        @skip_directives = true
       end
 
       def associate
         @mapping     = Hash.new { |h, k| h[k] = [] }
         @comment_num = 0
 
+        advance_through_directives if @skip_directives
+
         process(nil, @ast)
 
         @mapping
@@ -52,7 +66,7 @@ module Parser
       def current_comment_between?(prev_node, next_node)
         return false if current_comment.nil?
 
-        comment_loc = current_comment.location
+        comment_loc = current_comment.location.expression
         next_loc    = next_node.location.expression
 
         if prev_node.nil?
@@ -69,6 +83,18 @@ module Parser
         @mapping[node] << current_comment
         advance_comment
       end
+
+      def advance_through_directives
+        # Skip shebang.
+        if current_comment.text =~ /^#!/
+          advance_comment
+        end
+
+        # Skip encoding line.
+        if current_comment.text =~ Buffer::ENCODING_RE
+          advance_comment
+        end
+      end
     end
 
   end

data/lib/parser/source/comment.rb

@@ -8,7 +8,7 @@ module Parser
     #  @return String
     #
     # @!attribute [r] location
-    #  @return Parser::Source::Range
+    #  @return Parser::Source::Map
     class Comment
       attr_reader  :text
 
@@ -23,10 +23,10 @@ module Parser
       end
 
       ##
-      # @param [Parser::Source::Range] location
-      def initialize(location)
-        @location = location
-        @text     = location.source.freeze
+      # @param [Parser::Source::Range] range
+      def initialize(range)
+        @location = Parser::Source::Map.new(range)
+        @text     = range.source.freeze
 
         freeze
       end

data/lib/parser/source/map.rb

@@ -25,6 +25,14 @@ module Parser
         with { |map| map.update_expression(expression_l) }
       end
 
+      def ==(other)
+        other.class == self.class &&
+          instance_variables.map do |ivar|
+            instance_variable_get(ivar) ==
+              other.send(:instance_variable_get, ivar)
+          end.reduce(:&)
+      end
+
       def to_hash
         Hash[instance_variables.map do |ivar|
           [ ivar[1..-1].to_sym, instance_variable_get(ivar) ]

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.0.0.pre3'
+  VERSION = '2.0.0.pre4'
 end

data/test/test_lexer.rb

@@ -1023,7 +1023,7 @@ class TestLexer < Minitest::Test
   def test_question_eh_a__19
     setup_lexer 19
 
-    assert_scanned '?a', :tSTRING, "a"
+    assert_scanned '?a', :tCHARACTER, "a"
   end
 
   def test_question_eh_escape_M_escape_C__18
@@ -1035,7 +1035,7 @@ class TestLexer < Minitest::Test
   def test_question_eh_escape_M_escape_C__19
     setup_lexer 19
 
-    assert_scanned '?\M-\C-a', :tSTRING, "\M-\C-a"
+    assert_scanned '?\M-\C-a', :tCHARACTER, "\M-\C-a"
   end
 
   def test_integer_hex
@@ -1368,7 +1368,7 @@ class TestLexer < Minitest::Test
   def test_question__19
     setup_lexer 19
 
-    assert_scanned "?*", :tSTRING, "*"
+    assert_scanned "?*", :tCHARACTER, "*"
   end
 
   def test_question_bad_eos
@@ -1405,17 +1405,17 @@ class TestLexer < Minitest::Test
     setup_lexer 19
 
     @lex.state = :expr_beg
-    assert_scanned "?\\ ", :tSTRING, " "
+    assert_scanned "?\\ ", :tCHARACTER, " "
     @lex.state = :expr_beg
-    assert_scanned "?\\n", :tSTRING, "\n"
+    assert_scanned "?\\n", :tCHARACTER, "\n"
     @lex.state = :expr_beg
-    assert_scanned "?\\t", :tSTRING, "\t"
+    assert_scanned "?\\t", :tCHARACTER, "\t"
     @lex.state = :expr_beg
-    assert_scanned "?\\v", :tSTRING, "\v"
+    assert_scanned "?\\v", :tCHARACTER, "\v"
     @lex.state = :expr_beg
-    assert_scanned "?\\r", :tSTRING, "\r"
+    assert_scanned "?\\r", :tCHARACTER, "\r"
     @lex.state = :expr_beg
-    assert_scanned "?\\f", :tSTRING, "\f"
+    assert_scanned "?\\f", :tCHARACTER, "\f"
   end
 
   def test_rbracket
@@ -2807,8 +2807,8 @@ class TestLexer < Minitest::Test
   def test_bug_fid_char
     setup_lexer(19)
     assert_scanned(%Q{eof??a},
-                   :tFID,    'eof?',
-                   :tSTRING, 'a')
+                   :tFID,       'eof?',
+                   :tCHARACTER, 'a')
   end
 
   def test_bug_nonlabel_context__18

data/test/test_parser.rb

@@ -180,13 +180,29 @@ class TestParser < Minitest::Test
       %q{~~~~~~~~ expression})
   end
 
+  def test_character
+    assert_parses(
+      s(:str, 'a'),
+      %q{?a},
+      %q{^ begin
+        |~~ expression},
+      ALL_VERSIONS - %w(1.8))
+
+    assert_parses(
+      s(:int, 97),
+      %q{?a},
+      %q{~~ expression},
+      %w(1.8))
+  end
+
   # Symbols
 
   def test_symbol_plain
     assert_parses(
       s(:sym, :foo),
       %q{:foo},
-      %q{~~~~ expression})
+      %q{~ begin
+        |~~~~ expression})
 
     assert_parses(
       s(:sym, :foo),
@@ -4547,7 +4563,7 @@ class TestParser < Minitest::Test
     assert_parses_with_comments(
       s(:send, s(:int, 1), :+, s(:int, 2)),
       %Q{1 + # foo\n 2},
-      [ [4, 9]  ])
+      [ [4, 9] ])
   end
 
   def test_comment_single

data/test/test_source_comment_associator.rb

@@ -16,6 +16,8 @@ class TestSourceCommentAssociator < Minitest::Test
 
   def test_associate
     ast, associations = associate(<<-END)
+#!/usr/bin/env ruby
+# coding: utf-8
 # Class comment
 # another class comment
 class Foo

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre3
+  version: 2.0.0.pre4
 platform: ruby
 authors:
 - Peter Zotov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-26 00:00:00.000000000 Z
+date: 2013-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast
@@ -265,7 +265,6 @@ files:
 - lib/parser/source/comment.rb
 - lib/parser/source/comment/associator.rb
 - lib/parser/source/map.rb
-- lib/parser/source/map/block.rb
 - lib/parser/source/map/collection.rb
 - lib/parser/source/map/condition.rb
 - lib/parser/source/map/constant.rb

data/lib/parser/source/map/block.rb

@@ -1,16 +0,0 @@
-module Parser
-  module Source
-
-    class Map::Block < Map
-      attr_reader :begin
-      attr_reader :end
-
-      def initialize(begin_l, end_l, expression_l)
-        @begin, @end = begin_l, end_l
-
-        super(expression_l)
-      end
-    end
-
-  end
-end