parser 2.3.0.1 → 2.3.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 452a2098780caf4b2b9d00870b2791ae4e1db7e4
-  data.tar.gz: a3ec510fb0160c8bfa9f29b0f80edcd4705db85d
+  metadata.gz: 3cb2802760d71d558e9ab344e63deb18525be110
+  data.tar.gz: 32edc10c308de0d00030d4b9fbac6bd40bfff36f
 SHA512:
-  metadata.gz: b3322d4861c4c33a1381791f977928cba01428ee3153fb489cc1aed0ec88bb5c03570cfa448b073b1f6dc5776eb4aed6ef796b81aa5a607dae18439fec36b6c4
-  data.tar.gz: b490d684a3ee85df2ae7b9700195bee2792fe44273104535080bc22ab5ffbcd7ef351ec8d143e11829579005288606e4af2b5cbd2ff11a3bb824bbb6583d4908
+  metadata.gz: 365c674e6cf6b3fa43f81903a7f46ce787075eb927e4aecce9b59ff05d81b5c47d4304f063f2f31c528f8846faec1983736f7bbd169697e19950e03346b0e52a
+  data.tar.gz: 1504d14896c1cb0dde7a8bc46aa60b04036e03310b1f097e84e5f14587699f18b086184292fbb5fb1aaff4d201d172cd9c424d1b8bc11de725d2c64567893928

data/CHANGELOG.md

@@ -1,6 +1,22 @@
 Changelog
 =========
 
+v2.3.0.2 (2016-01-24)
+---------------------
+
+Bugs fixed:
+ * Add :csend to Parser::Meta::NODE_TYPES (Markus Schirp)
+ * lexer/dedenter: "\<\<x\n  y\\n  z\nx": don't dedent after escaped newline. (whitequark)
+
+v2.3.0.2 (2016-01-16)
+---------------------
+
+v2.3.0.1 (2016-01-14)
+---------------------
+
+Features implemented:
+ * ruby23.y: implement optional superclass (cremno)
+
 v2.3.0.0 (2016-01-14)
 ---------------------
 

data/README.md

@@ -1,8 +1,8 @@
 # Parser
 
-[![Gem Version](https://badge.fury.io/rb/parser.png)](https://badge.fury.io/rb/parser)
-[![Build Status](https://travis-ci.org/whitequark/parser.png?branch=master)](https://travis-ci.org/whitequark/parser)
-[![Coverage Status](https://coveralls.io/repos/whitequark/parser/badge.png?branch=master)](https://coveralls.io/r/whitequark/parser)
+[![Gem Version](https://badge.fury.io/rb/parser.svg)](https://badge.fury.io/rb/parser)
+[![Build Status](https://travis-ci.org/whitequark/parser.svg?branch=master)](https://travis-ci.org/whitequark/parser)
+[![Coverage Status](https://coveralls.io/repos/whitequark/parser/badge.svg?branch=master)](https://coveralls.io/r/whitequark/parser)
 
 _Parser_ is a production-ready Ruby parser written in pure Ruby. It recognizes as
 much or more code than Ripper, Melbourne, JRubyParser or ruby\_parser, and

data/lib/parser/lexer.rl

@@ -92,7 +92,7 @@ class Parser::Lexer
   REGEXP_META_CHARACTERS = Regexp.union(*"\\$()*+.<>?[]^{|}".chars).freeze
   UNDERSCORE_STRING = '_'.freeze
 
-  RBRACE_OR_RPAREN = %w"} ]".freeze
+  RBRACE_OR_RBRACK = %w"} ]".freeze
 
   attr_reader   :source_buffer
   attr_reader   :encoding
@@ -426,6 +426,12 @@ class Parser::Lexer
       else
         self.class.lex_en_plain_words
       end
+    elsif new_literal.backslash_delimited?
+      if new_literal.interpolate?
+        self.class.lex_en_interp_backslash_delimited
+      else
+        self.class.lex_en_plain_backslash_delimited
+      end
     else
       if new_literal.interpolate?
         self.class.lex_en_interp_string
@@ -645,10 +651,10 @@ class Parser::Lexer
   flo_pow  = [eE] [+\-]? ( digit+ '_' )* digit+;
 
   int_suffix =
-    ''   % { @num_xfrm = lambda { |chars|  emit(:tINTEGER,   chars) } }
-  | 'r'  % { @num_xfrm = lambda { |chars|  emit(:tRATIONAL,  Rational(chars)) } }
-  | 'i'  % { @num_xfrm = lambda { |chars|  emit(:tIMAGINARY, Complex(0, chars)) } }
-  | 'ri' % { @num_xfrm = lambda { |chars|  emit(:tIMAGINARY, Complex(0, Rational(chars))) } };
+    ''   % { @num_xfrm = lambda { |chars| emit(:tINTEGER,   chars) } }
+  | 'r'  % { @num_xfrm = lambda { |chars| emit(:tRATIONAL,  Rational(chars)) } }
+  | 'i'  % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, chars)) } }
+  | 'ri' % { @num_xfrm = lambda { |chars| emit(:tIMAGINARY, Complex(0, Rational(chars))) } };
 
   flo_pow_suffix =
     ''   % { @num_xfrm = lambda { |chars| emit(:tFLOAT,     Float(chars)) } }
@@ -682,11 +688,8 @@ class Parser::Lexer
       codepoint = codepoint_str.to_i(16)
 
       if codepoint >= 0x110000
-        @escape = lambda do
-          diagnostic :error, :unicode_point_too_large, nil,
-                     range(codepoint_s, codepoint_s + codepoint_str.length)
-        end
-
+        diagnostic :error, :unicode_point_too_large, nil,
+                   range(codepoint_s, codepoint_s + codepoint_str.length)
         break
       end
 
@@ -701,9 +704,7 @@ class Parser::Lexer
   }
 
   action invalid_complex_escape {
-    @escape = lambda do
-      diagnostic :fatal, :invalid_escape
-    end
+    diagnostic :fatal, :invalid_escape
   }
 
   action slash_c_char {
@@ -731,20 +732,17 @@ class Parser::Lexer
       % { @escape = encode_escape(tok(@escape_s, p).to_i(8) % 0x100) }
 
       # \xff
-    | ( 'x' xdigit{1,2}
+    | 'x' xdigit{1,2}
         % { @escape = encode_escape(tok(@escape_s + 1, p).to_i(16)) }
+
       # \u263a
-      | 'u' xdigit{4}
-        % { @escape = tok(@escape_s + 1, p).to_i(16).chr(Encoding::UTF_8) }
-      )
+    | 'u' xdigit{4}
+      % { @escape = tok(@escape_s + 1, p).to_i(16).chr(Encoding::UTF_8) }
 
       # %q[\x]
     | 'x' ( c_any - xdigit )
       % {
-        @escape = lambda do
-          diagnostic :fatal, :invalid_hex_escape, nil,
-                     range(@escape_s - 1, p + 2)
-        end
+        diagnostic :fatal, :invalid_hex_escape, nil, range(@escape_s - 1, p + 2)
       }
 
       # %q[\u123] %q[\u{12]
@@ -756,10 +754,7 @@ class Parser::Lexer
             )
           )
       % {
-        @escape = lambda do
-          diagnostic :fatal, :invalid_unicode_escape, nil,
-                     range(@escape_s - 1, p)
-        end
+        diagnostic :fatal, :invalid_unicode_escape, nil, range(@escape_s - 1, p)
       }
 
       # \u{123 456}
@@ -770,10 +765,7 @@ class Parser::Lexer
         | ( c_any - '}' )* c_eof
         | xdigit{7,}
         ) % {
-          @escape = lambda do
-            diagnostic :fatal, :unterminated_unicode, nil,
-                       range(p - 1, p)
-          end
+          diagnostic :fatal, :unterminated_unicode, nil, range(p - 1, p)
         }
       )
 
@@ -881,53 +873,35 @@ class Parser::Lexer
 
   action extend_string_escaped {
     current_literal = literal
-    if current_literal.nest_and_try_closing('\\', @ts, @ts + 1)
-      # If the literal is actually closed by the backslash,
-      # rewind the input prior to consuming the escape sequence.
-      p = @escape_s - 1
-      fnext *pop_literal; fbreak;
+    # Get the first character after the backslash.
+    escaped_char = @source[@escape_s].chr
+
+    if current_literal.munge_escape? escaped_char
+      # If this particular literal uses this character as an opening
+      # or closing delimiter, it is an escape sequence for that
+      # particular character. Write it without the backslash.
+
+      if current_literal.regexp? && REGEXP_META_CHARACTERS.match(escaped_char)
+        # Regular expressions should include escaped delimiters in their
+        # escaped form, except when the escaped character is
+        # a closing delimiter but not a regexp metacharacter.
+        #
+        # The backslash itself cannot be used as a closing delimiter
+        # at the same time as an escape symbol, but it is always munged,
+        # so this branch also executes for the non-closing-delimiter case
+        # for the backslash.
+        current_literal.extend_string(tok, @ts, @te)
+      else
+        current_literal.extend_string(escaped_char, @ts, @te)
+      end
     else
-      # Get the first character after the backslash.
-      escaped_char = @source[@escape_s].chr
-
-      if current_literal.munge_escape? escaped_char
-        # If this particular literal uses this character as an opening
-        # or closing delimiter, it is an escape sequence for that
-        # particular character. Write it without the backslash.
-
-        if current_literal.regexp? && REGEXP_META_CHARACTERS.match(escaped_char)
-          # Regular expressions should include escaped delimiters in their
-          # escaped form, except when the escaped character is
-          # a closing delimiter but not a regexp metacharacter.
-          #
-          # The backslash itself cannot be used as a closing delimiter
-          # at the same time as an escape symbol, but it is always munged,
-          # so this branch also executes for the non-closing-delimiter case
-          # for the backslash.
-          current_literal.extend_string(tok, @ts, @te)
-        else
-          current_literal.extend_string(escaped_char, @ts, @te)
-        end
+      # It does not. So this is an actual escape sequence, yay!
+      if current_literal.regexp?
+        # Regular expressions should include escape sequences in their
+        # escaped form. On the other hand, escaped newlines are removed.
+        current_literal.extend_string(tok.gsub(ESCAPED_NEXT_LINE, BLANK_STRING), @ts, @te)
       else
-        # It does not. So this is an actual escape sequence, yay!
-        # Two things to consider here.
-        #
-        # 1. The `escape' rule should be pure and so won't raise any
-        #    errors by itself. Instead, it stores them in lambdas.
-        #
-        # 2. Non-interpolated literals do not go through the aforementioned
-        #    rule. As \\ and \' (and variants) are munged, the full token
-        #    should always be written for such literals.
-
-        @escape.call if @escape.respond_to? :call
-
-        if current_literal.regexp?
-          # Regular expressions should include escape sequences in their
-          # escaped form. On the other hand, escaped newlines are removed.
-          current_literal.extend_string(tok.gsub(ESCAPED_NEXT_LINE, BLANK_STRING), @ts, @te)
-        else
-          current_literal.extend_string(@escape || tok, @ts, @te)
-        end
+        current_literal.extend_string(@escape || tok, @ts, @te)
       end
     end
   }
@@ -1115,6 +1089,18 @@ class Parser::Lexer
       c_any       => extend_string;
   *|;
 
+  interp_backslash_delimited := |*
+      interp_code => extend_interp_code;
+      interp_var  => extend_interp_var;
+      c_eol       => extend_string_eol;
+      c_any       => extend_string;
+  *|;
+
+  plain_backslash_delimited := |*
+      c_eol       => extend_string_eol;
+      c_any       => extend_string;
+  *|;
+
   regexp_modifiers := |*
       [A-Za-z]+
       => {
@@ -1669,7 +1655,7 @@ class Parser::Lexer
       # %w(we are the people)
       '%' [A-Za-z]+ c_any
       => {
-        type, delimiter = tok[0..-2], @source[@te - 1].chr
+        type, delimiter = @source[@ts...(@te - 1)], @source[@te - 1].chr
         fgoto *push_literal(type, delimiter, @ts);
       };
 
@@ -1746,9 +1732,6 @@ class Parser::Lexer
           | (c_any - c_space_nl - e_bs) % { @escape = nil }
           )
       => {
-        # Show an error if memorized.
-        @escape.call if @escape.respond_to? :call
-
         value = @escape || tok(@ts + 1)
 
         if version?(18)
@@ -2177,7 +2160,7 @@ class Parser::Lexer
         emit_table(PUNCTUATION)
         @cond.lexpop; @cmdarg.lexpop
 
-        if RBRACE_OR_RPAREN.include?(tok)
+        if RBRACE_OR_RBRACK.include?(tok)
           fnext expr_endarg;
         else # )
           # fnext expr_endfn; ?

data/lib/parser/lexer/dedenter.rb

@@ -9,6 +9,7 @@ module Parser
 
     def dedent(string)
       space_begin = space_end = offset = 0
+      last_index  = string.length - 1
       string.chars.each_with_index do |char, index|
         if @at_line_begin
           if char == ?\n || @indent_level >= @dedent_level
@@ -26,7 +27,7 @@ module Parser
             @indent_level += 8 - @indent_level % 8
             space_end += 1
           end
-        elsif char == ?\n
+        elsif char == ?\n && index == last_index
           @at_line_begin = true
           @indent_level  = 0
           space_begin = space_end = index - offset + 1

data/lib/parser/lexer/literal.rb

@@ -4,7 +4,6 @@ module Parser
 
   class Lexer::Literal
     DELIMITERS = { '(' => ')', '[' => ']', '{' => '}', '<' => '>' }
-    MONOLITHIC = { :tSTRING_BEG => :tSTRING }
 
     TYPES = {
     # type       start token     interpolate?
@@ -79,9 +78,7 @@ module Parser
                       !heredoc?)
 
       # Capture opening delimiter in percent-literals.
-      unless @heredoc_e || @str_type.end_with?(delimiter)
-        @str_type << delimiter
-      end
+      @str_type << delimiter if @str_type.start_with?('%')
 
       clear_buffer
 
@@ -105,6 +102,10 @@ module Parser
       !!@heredoc_e
     end
 
+    def backslash_delimited?
+      @end_delim == '\\'
+    end
+
     def type
       @start_tok
     end
@@ -141,7 +142,7 @@ module Parser
           emit(:tLABEL_END, @end_delim, ts, te + 1)
         elsif @monolithic
           # Emit the string as a single token.
-          emit(MONOLITHIC[@start_tok], @buffer, @str_s, te)
+          emit(:tSTRING, @buffer, @str_s, te)
         else
           # If this is a heredoc, @buffer contains the sentinel now.
           # Just throw it out. Lexer flushes the heredoc after each
@@ -183,10 +184,7 @@ module Parser
     end
 
     def extend_string(string, ts, te)
-      if @buffer_s.nil?
-        @buffer_s = ts
-      end
-
+      @buffer_s ||= ts
       @buffer_e = te
 
       @buffer << string

data/lib/parser/meta.rb

@@ -16,7 +16,7 @@ module Parser
         match_with_lvasgn match_current_line
         module class sclass def defs undef alias args
         cbase arg optarg restarg blockarg block_pass args def kwarg kwoptarg
-        kwrestarg send super zsuper yield block send
+        kwrestarg send csend super zsuper yield block
         and not or if when case while until while_post
         until_post for break next redo return resbody
         kwbegin begin retry preexe postexe iflipflop eflipflop

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.3.0.1'
+  VERSION = '2.3.0.2'
 end

data/test/test_lexer.rb

@@ -1796,6 +1796,14 @@ class TestLexer < Minitest::Test
                    :tREGEXP_OPT,     "")
   end
 
+  def test_regexp_escape_other_meta
+    assert_scanned("/\\.\\$\\*\\+\\.\\?\\|/",
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "\\.\\$\\*\\+\\.\\?\\|",
+                   :tSTRING_END,     "/",
+                   :tREGEXP_OPT,     "")
+  end
+
   def test_regexp_nm
     assert_scanned("/.*/nm",
                    :tREGEXP_BEG,     "/",
@@ -2097,7 +2105,7 @@ class TestLexer < Minitest::Test
 
   def test_string_pct_pct
     assert_scanned("%%blah%",
-                   :tSTRING_BEG,     '%',
+                   :tSTRING_BEG,     '%%',
                    :tSTRING_CONTENT, "blah",
                    :tSTRING_END,     '%')
   end
@@ -2159,6 +2167,48 @@ class TestLexer < Minitest::Test
                    :tSTRING_END,     ')')
   end
 
+  def test_string_pct_backslash
+    assert_scanned("%\\a\\",
+                   :tSTRING_BEG, "%\\",
+                   :tSTRING_CONTENT, "a",
+                   :tSTRING_END, "\\")
+  end
+
+  def test_string_pct_backslash_with_bad_escape
+    # No escapes are allowed in a backslash-delimited string
+    refute_scanned("%\\a\\n\\",
+                   :tSTRING_BEG, "%\\",
+                   :tSTRING_CONTENT, "a",
+                   :tSTRING_END, "\\",
+                   :tIDENTIFIER, "n")
+  end
+
+  def test_string_pct_intertwined_with_heredoc
+    assert_scanned("<<-foo + %\\a\nbar\nfoo\nb\\",
+                   :tSTRING_BEG, "<<\"",
+                   :tSTRING_CONTENT, "bar\n",
+                   :tSTRING_END, "foo",
+                   :tPLUS, "+",
+                   :tSTRING_BEG, "%\\",
+                   :tSTRING_CONTENT, "a\n",
+                   :tSTRING_CONTENT, "b",
+                   :tSTRING_END, "\\")
+  end
+
+  def test_string_pct_q_backslash
+    assert_scanned("%q\\a\\",
+                   :tSTRING_BEG, "%q\\",
+                   :tSTRING_CONTENT, "a",
+                   :tSTRING_END, "\\")
+  end
+
+  def test_string_pct_Q_backslash
+    assert_scanned("%Q\\a\\",
+                   :tSTRING_BEG, "%Q\\",
+                   :tSTRING_CONTENT, "a",
+                   :tSTRING_END, "\\")
+  end
+
   def test_string_single
     assert_scanned("'string'",
                    :tSTRING, "string")

data/test/test_parser.rb

@@ -5286,17 +5286,15 @@ class TestParser < Minitest::Test
       ALL_VERSIONS - %w(1.8 1.9 mac ios 2.0)) # no 1.9 backport
   end
 
-  # We implement broken behavior, and Ruby is not fixed as of 2016-01-14.
   def test_ruby_bug_11989
     assert_parses(
       s(:send, nil, :p,
-        s(:str, "x\n y\n")),
+        s(:str, "x\n   y\n")),
       %Q{p <<~"E"\n  x\\n   y\nE},
       %q{},
       ALL_VERSIONS - %w(1.8 1.9 2.0 2.1 2.2 ios mac))
   end
 
-  # We implement correct behavior, but Ruby is not fixed as of 2016-01-14.
   def test_ruby_bug_11990
     assert_parses(
       s(:send, nil, :p,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.3.0.1
+  version: 2.3.0.2
 platform: ruby
 authors:
 - whitequark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-14 00:00:00.000000000 Z
+date: 2016-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast
@@ -354,7 +354,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A Ruby parser written in pure Ruby.