parser 2.3.0.3 → 2.3.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 32b8e79e47ae5063bd6a4451c47b337f43d5a2a2
-  data.tar.gz: c8d08375dca4b4089fa0dade5d77f13284498852
+  metadata.gz: 1cb171255b96bf71809258731c221082841d8af1
+  data.tar.gz: 2ef132309fc78e96f665e60bdbfecb389f4bda6d
 SHA512:
-  metadata.gz: 039b018cd53937074f6a0eb1b366c5d83c0de6ed104a3ca63597b265c695f010e2ae85fdee74bc9c00418510e386e9fc7e2ff44dd9dc9ead2ddef5cc3ef0ec1e
-  data.tar.gz: 4b8b8b3497d1034e4662aa8ad3cf85c57cd65cb73e038bf7737c63db832d719714aa2bafd2fef5dcdb85d2ee4342ef3c4c8ceb5d26f3abc5cfed67e4c6902aff
+  metadata.gz: 9f374c086ac8fff2fb0dfc2d0cfb9badc4e6da09b9612992c51e3207bdeff35491ccd7edb13e07b0f7dfe31bd9129df31bf4a4d550f58d9534bc9fdf55c6b064
+  data.tar.gz: f1821293d14bf5716f5c9e3d8f9af4d6eb7f84f05f3fed3552521770b33add9fd639502c6949c9c04a61775b275d208a199e3261b01c5b1082cf624c37968532

data/.travis.yml

@@ -4,7 +4,6 @@ rvm:
  - 1.9.2
  - 1.9.3
  - 2.0.0
- - 2.1.8
  - 2.2.4
  - 2.3.0
  - ruby-head

data/CHANGELOG.md

@@ -1,6 +1,9 @@
 Changelog
 =========
 
+v2.3.0.4 (2016-02-09)
+---------------------
+
 v2.3.0.3 (2016-02-06)
 ---------------------
 
@@ -14,7 +17,7 @@ Bugs fixed:
  * Add :csend to Parser::Meta::NODE_TYPES (Markus Schirp)
  * lexer/dedenter: "\<\<x\n  y\\n  z\nx": don't dedent after escaped newline. (whitequark)
 
-v2.3.0.3 (2016-01-16)
+v2.3.0.4 (2016-01-16)
 ---------------------
 
 v2.3.0.1 (2016-01-14)

data/lib/parser/lexer.rl

@@ -19,7 +19,7 @@
 #    they're pointing to ("current"), plus one. `@ts` contains the index
 #    of the corresponding character. The code for extracting matched token is:
 #
-#       @source[@ts...@te]
+#       @source_buffer.slice(@ts...@te)
 #
 #  * If your input is `foooooooobar` and the rule is:
 #
@@ -109,8 +109,6 @@ class Parser::Lexer
     @tokens     = nil
     @comments   = nil
 
-    @has_encode = ''.respond_to?(:encode)
-
     reset
   end
 
@@ -128,10 +126,7 @@ class Parser::Lexer
 
     @force_utf32   = false # Set to true by some tests
 
-    @source        = nil # source string
     @source_pts    = nil # @source as a codepoint array
-    @encoding      = nil # target encoding for output strings
-    @need_encode   = nil
 
     @p             = 0   # stream position (saved manually in #advance)
     @ts            = nil # token start
@@ -186,33 +181,12 @@ class Parser::Lexer
     @source_buffer = source_buffer
 
     if @source_buffer
-      @source = @source_buffer.source
-      @need_encode = false
-
-      @encoding = @source.encoding if @has_encode
+      source = @source_buffer.source
 
-      if @has_encode && @source.encoding == Encoding::UTF_8
-        @source_pts = @source.unpack('U*')
+      if source.respond_to?(:encode) && source.encoding == Encoding::UTF_8
+        @source_pts = source.unpack('U*')
       else
-        @source_pts = @source.unpack('C*')
-      end
-
-      if @has_encode &&
-        (@source_pts.size > 1_000_000 || @force_utf32) &&
-        @encoding != Encoding::UTF_32LE
-        # A heuristic: if the buffer is larger than 1M, then
-        # store it in UTF-32 and convert the tokens as they're
-        # going out. If it's smaller, the conversion overhead
-        # dominates runtime and this stops being beneficial.
-        #
-        # This is not really a good heuristic, as the result
-        # heavily depends on token/character ratio. If it's low,
-        # say the gem consists mostly of long identifiers and
-        # symbols, then storing the source in UTF-8 would be faster.
-        #
-        # Patches accepted.
-        @source = @source.encode(Encoding::UTF_32LE)
-        @need_encode = true
+        @source_pts = source.unpack('C*')
       end
 
       if @source_pts[0] == 0xfeff
@@ -220,7 +194,6 @@ class Parser::Lexer
         @p = 1
       end
     else
-      @source     = nil
       @source_pts = nil
     end
   end
@@ -327,24 +300,24 @@ class Parser::Lexer
     versions.include?(@version)
   end
 
+  def stack_pop
+    @top -= 1
+    @stack[@top]
+  end
+
   if "".respond_to?(:encode)
     def encode_escape(ord)
-      ord.chr.force_encoding(@encoding)
+      ord.chr.force_encoding(source_buffer.source.encoding)
     end
 
-    def tok(s = @ts, e = @te)
-      source = @source[s...e]
-      return source unless @need_encode
-      source.encode(@encoding)
-    end
   else
     def encode_escape(ord)
       ord.chr
     end
+  end
 
-    def tok(s = @ts, e = @te)
-      @source[s...e]
-    end
+  def tok(s = @ts, e = @te)
+    @source_buffer.slice(s...e)
   end
 
   def range(s = @ts, e = @te)
@@ -707,13 +680,13 @@ class Parser::Lexer
 
   maybe_escaped_char = (
         '\\' c_any      %unescape_char
-    | ( c_any - [\\] )  % { @escape = @source[p - 1].chr }
+    | ( c_any - [\\] )  % { @escape = @source_buffer.slice(p - 1).chr }
   );
 
   maybe_escaped_ctrl_char = ( # why?!
         '\\' c_any      %unescape_char %slash_c_char
     |   '?'             % { @escape = "\x7f" }
-    | ( c_any - [\\?] ) % { @escape = @source[p - 1].chr } %slash_c_char
+    | ( c_any - [\\?] ) % { @escape = @source_buffer.slice(p - 1).chr } %slash_c_char
   );
 
   escape = (
@@ -840,8 +813,7 @@ class Parser::Lexer
 
     # tLABEL_END is only possible in non-cond context on >= 2.2
     if @version >= 22 && !@cond.active?
-      lookahead = @source[@te...@te+2]
-      lookahead = lookahead.encode(@encoding) if @need_encode
+      lookahead = @source_buffer.slice(@te...@te+2)
     end
 
     current_literal = literal
@@ -863,7 +835,7 @@ class Parser::Lexer
   action extend_string_escaped {
     current_literal = literal
     # Get the first character after the backslash.
-    escaped_char = @source[@escape_s].chr
+    escaped_char = @source_buffer.slice(@escape_s).chr
 
     if current_literal.munge_escape? escaped_char
       # If this particular literal uses this character as an opening
@@ -1022,7 +994,7 @@ class Parser::Lexer
         end
 
         fhold;
-        fret;
+        fnext *stack_pop;
         fbreak;
       end
     end
@@ -1241,7 +1213,7 @@ class Parser::Lexer
           emit(:tGVAR)
         end
 
-        fret; fbreak;
+        fnext *stack_pop; fbreak;
       };
 
       class_var_v
@@ -1251,7 +1223,7 @@ class Parser::Lexer
         end
 
         emit(:tCVAR)
-        fret; fbreak;
+        fnext *stack_pop; fbreak;
       };
 
       instance_var_v
@@ -1261,7 +1233,7 @@ class Parser::Lexer
         end
 
         emit(:tIVAR)
-        fret; fbreak;
+        fnext *stack_pop; fbreak;
       };
   *|;
 
@@ -1637,7 +1609,7 @@ class Parser::Lexer
       # %<string>
       '%' ( any - [A-Za-z] )
       => {
-        type, delimiter = @source[@ts].chr, tok[-1].chr
+        type, delimiter = @source_buffer.slice(@ts).chr, tok[-1].chr
         fgoto *push_literal(type, delimiter, @ts);
       };
 
@@ -1732,7 +1704,7 @@ class Parser::Lexer
       '?' c_space_nl
       => {
         escape = { " "  => '\s', "\r" => '\r', "\n" => '\n', "\t" => '\t',
-                   "\v" => '\v', "\f" => '\f' }[@source[@ts + 1]]
+                   "\v" => '\v', "\f" => '\f' }[@source_buffer.slice(@ts + 1)]
         diagnostic :warning, :invalid_escape_use, { :escape => escape }, range
 
         p = @ts - 1
@@ -1805,7 +1777,7 @@ class Parser::Lexer
         if version?(18)
           ident = tok(@ts, @te - 2)
 
-          emit((@source[@ts] =~ /[A-Z]/) ? :tCONSTANT : :tIDENTIFIER,
+          emit((@source_buffer.slice(@ts) =~ /[A-Z]/) ? :tCONSTANT : :tIDENTIFIER,
                ident, @ts, @te - 2)
           fhold; # continue as a symbol
 

data/lib/parser/lexer/literal.rb

@@ -242,7 +242,7 @@ module Parser
       # Prime the buffer with lexer encoding; otherwise,
       # concatenation will produce varying results.
       if defined?(Encoding)
-        @buffer.force_encoding(@lexer.encoding)
+        @buffer.force_encoding(@lexer.source_buffer.source.encoding)
       end
 
       @buffer_s = nil

data/lib/parser/source/buffer.rb

@@ -107,9 +107,12 @@ module Parser
         @lines       = nil
         @line_begins = nil
 
+        # UTF-32-reencoded source for O(1) slicing
+        @slice_source = nil
+
         # Cache for fast lookup
-        @line_for_position = {}
-        @col_for_position  = {}
+        @line_for_position   = {}
+        @column_for_position = {}
       end
 
       ##
@@ -178,6 +181,21 @@ module Parser
         end
 
         @source = input.gsub("\r\n".freeze, "\n".freeze).freeze
+
+        if defined?(Encoding) &&
+           !@source.ascii_only? &&
+           @source.encoding != Encoding::UTF_32LE &&
+           @source.encoding != Encoding::BINARY
+          @slice_source = @source.encode(Encoding::UTF_32LE)
+        end
+      end
+
+      def slice(range)
+        if @slice_source.nil?
+          @source[range]
+        else
+          @slice_source[range].encode(@source.encoding)
+        end
       end
 
       ##
@@ -214,7 +232,7 @@ module Parser
       # @api private
       #
       def column_for_position(position)
-        @col_for_position[position] ||= begin
+        @column_for_position[position] ||= begin
           _, line_begin = line_for(position)
           position - line_begin
         end

data/lib/parser/source/range.rb

@@ -126,7 +126,7 @@ module Parser
       # @return [String] all source code covered by this range.
       #
       def source
-        @source_buffer.source[self.begin_pos...self.end_pos]
+        @source_buffer.slice(self.begin_pos...self.end_pos)
       end
 
       ##

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.3.0.3'
+  VERSION = '2.3.0.4'
 end

data/test/test_parser.rb

@@ -5330,6 +5330,16 @@ class TestParser < Minitest::Test
       ALL_VERSIONS - %w(1.8))
   end
 
+  def test_parser_bug_272
+    assert_parses(
+      s(:block,
+        s(:send, nil, :a,
+          s(:ivar, :@b)),
+        s(:args,
+          s(:arg, :c)), nil),
+      %q{a @b do |c|;end})
+  end
+
   def test_bug_lambda_leakage
     assert_parses(
       s(:begin,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.3.0.3
+  version: 2.3.0.4
 platform: ruby
 authors:
 - whitequark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-06 00:00:00.000000000 Z
+date: 2016-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast