ripper_ruby_parser 1.4.1 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f75852f6918c25ff33b1edfdd19ed7dc6e09662dd5786b37afe5cc900433db6b
-  data.tar.gz: 1f8b9b066b7dff6170ba7a1c0487a474314a1c67e8d5b63c3720aae4ae498775
+  metadata.gz: ab036613dacd6865ca8103db78c67273d6fad340e3a6a7ed1fe5450c5511cc05
+  data.tar.gz: 182feec3370738d6d5516e5615022a045ed717edf489cc694ac02eda97e6e6bb
 SHA512:
-  metadata.gz: 040b94d89f959fdab917a244cea6649d27dbf1f9fa390753f09b025f6feac6778425a154592d9f252b1f60d28473865e78f6e993d6ba75658cd8d67aeed95d53
-  data.tar.gz: 0d947570348eb461628a027200ad456968261980925087915c9b7ec7f281ec9e789b3d9facf03090f7cecf3f879e4e7fcf1a1ff41a259fb036737d371d6a31e1
+  metadata.gz: 9c5f17c00265aaf9df0741c5afee3b90f000141d4285bce067953bb1feac645779c8e4b9db2319baa9a3f6822b46a9f6edbeef87af1177b22a03cd437fecad51
+  data.tar.gz: 7001633d0756ab5afb2b2813572550f3cea1df75c11e82fe8117e4243aaa6cf4386512b3a7754b76f0a5a8be7c4b49f8b1e5a0148b4465e7aa72658b7c951acc

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.4.2 / 2018-04-03
+
+* Fix handling of strings delimited by %()
+* Handle line continuations in stringlike literals
+  - Handle line continuations in string and regexp literals
+  - Handle escaped line continuations
+  - Hanlde line continuations in word and symbol list literals
+* Force encoding of string literals to UTF-8 if the result is valid
+* Fix handling of range operators with float literals
+
 ## 1.4.1 / 2018-03-31
 
 * Properly pop delimiter stack after parsing a symbol
@@ -7,7 +17,8 @@
 ## 1.4.0 / 2018-03-30
 
 * Handle begin..end blocks with postfix conditionals
-* Correctly handle string variants that do not allow escape sequences
+* Match RubyParser's handling of string literals that do not allow escape
+  sequences
 
 ## 1.3.0 / 2018-02-17
 

data/lib/ripper_ruby_parser/commenting_ripper_parser.rb

@@ -179,10 +179,18 @@ module RipperRubyParser
 
     def on_tstring_content(content)
       content = case @delimiter_stack.last
-                when '"', '`', ':"', /^<</, /^%I/, /^%Q/, /^%W/
+                when /^<</
                   Unescape.unescape(content)
-                when "'", ":'"
+                when '"', '`', ':"', /^%Q.$/, /^%.$/
+                  Unescape.fix_encoding(Unescape.unescape(content))
+                when /^%[WI].$/
+                  Unescape.fix_encoding(Unescape.unescape_wordlist_word(content))
+                when "'", ":'", /^%q.$/
                   Unescape.simple_unescape(content)
+                when '/', /^%r.$/
+                  Unescape.unescape_regexp(content)
+                when /^%[wi].$/
+                  Unescape.simple_unescape_wordlist_word(content)
                 else
                   content
                 end

data/lib/ripper_ruby_parser/sexp_handlers/helper_methods.rb

@@ -36,8 +36,8 @@ module RipperRubyParser
         items.push(*map_process_list(rest))
       end
 
-      def literal?(exp)
-        exp.sexp_type == :lit
+      def integer_literal?(exp)
+        exp.sexp_type == :lit && exp[1].is_a?(Integer)
       end
 
       def reject_void_stmt(body)

data/lib/ripper_ruby_parser/sexp_handlers/operators.rb

@@ -46,7 +46,7 @@ module RipperRubyParser
         _, left, right = exp.shift 3
         left = process(left)
         right = process(right)
-        if literal?(left) && literal?(right)
+        if integer_literal?(left) && integer_literal?(right)
           s(:lit, Range.new(left[1], right[1]))
         else
           s(:dot2, left, right)
@@ -57,7 +57,7 @@ module RipperRubyParser
         _, left, right = exp.shift 3
         left = process(left)
         right = process(right)
-        if literal?(left) && literal?(right)
+        if integer_literal?(left) && integer_literal?(right)
           s(:lit, Range.new(left[1], right[1], true))
         else
           s(:dot3, left, right)

data/lib/ripper_ruby_parser/unescape.rb

@@ -5,6 +5,22 @@ module RipperRubyParser
   module Unescape
     module_function
 
+    ESCAPE_SEQUENCE_REGEXP =
+      /\\(
+        [0-7]{1,3}        | # octal character
+        x[0-9a-fA-F]{1,2} | # hex byte
+        u[0-9a-fA-F]{4}   | # unicode character
+        M-\\C-.           | # meta-ctrl
+        C-\\M-.           | # ctrl-meta
+        M-\\c.            | # meta-ctrl (shorthand)
+        c\\M-.            | # ctrl-meta (shorthand)
+        C-.               | # control (regular)
+        c.                | # control (shorthand)
+        M-.               | # meta
+        \n                | # line continuation
+        .                   # single-character
+      )/x
+
     SINGLE_LETTER_ESCAPES = {
       'a' => "\a",
       'b' => "\b",
@@ -29,40 +45,75 @@ module RipperRubyParser
       end
     end
 
-    def unescape(string)
+    def simple_unescape_wordlist_word(string)
       string.gsub(/\\(
-        [0-7]{1,3}        | # octal character
-        x[0-9a-fA-F]{1,2} | # hex byte
-        u[0-9a-fA-F]{4}   | # unicode character
-        M-\\C-.           | # meta-ctrl
-        C-\\M-.           | # ctrl-meta
-        M-\\c.            | # meta-ctrl (shorthand)
-        c\\M-.            | # ctrl-meta (shorthand)
-        C-.               | # control (regular)
-        c.                | # control (shorthand)
-        M-.               | # meta
-          .                   # single-character
+        '   | # single quote
+        \\  | # backslash
+        \n    # newline
       )/x) do
+        Regexp.last_match[1]
+      end
+    end
+
+    def unescape(string)
+      string.gsub(ESCAPE_SEQUENCE_REGEXP) do
+        bare = Regexp.last_match[1]
+        if bare == "\n"
+          ''
+        else
+          unescaped_value(bare)
+        end
+      end
+    end
+
+    def unescape_wordlist_word(string)
+      string.gsub(ESCAPE_SEQUENCE_REGEXP) do
+        bare = Regexp.last_match[1]
+        unescaped_value(bare)
+      end
+    end
+
+    def fix_encoding(string)
+      unless string.encoding == Encoding::UTF_8
+        dup = string.dup.force_encoding Encoding::UTF_8
+        return dup if dup.valid_encoding?
+      end
+      string
+    end
+
+    def unescape_regexp(string)
+      string.gsub(/\\(\n|\\)/) do
         bare = Regexp.last_match[1]
         case bare
-        when SINGLE_LETTER_ESCAPES_REGEXP
-          SINGLE_LETTER_ESCAPES[bare]
-        when /^x/
-          bare[1..-1].to_i(16).chr
-        when /^u/
-          bare[1..-1].to_i(16).chr(Encoding::UTF_8)
-        when /^(c|C-).$/
-          (bare[-1].ord & 0b1001_1111).chr
-        when /^M-.$/
-          (bare[-1].ord | 0b1000_0000).chr
-        when /^(M-\\C-|C-\\M-|M-\\c|c\\M-).$/
-          (bare[-1].ord & 0b1001_1111 | 0b1000_0000).chr
-        when /^[0-7]+/
-          bare.to_i(8).chr
+        when "\n"
+          ''
         else
-          bare
+          '\\\\'
         end
       end
     end
+
+    def unescaped_value(bare)
+      case bare
+      when SINGLE_LETTER_ESCAPES_REGEXP
+        SINGLE_LETTER_ESCAPES[bare]
+      when /^x/
+        bare[1..-1].to_i(16).chr
+      when /^u/
+        bare[1..-1].to_i(16).chr(Encoding::UTF_8)
+      when /^(c|C-).$/
+        (bare[-1].ord & 0b1001_1111).chr
+      when /^M-.$/
+        (bare[-1].ord | 0b1000_0000).chr
+      when /^(M-\\C-|C-\\M-|M-\\c|c\\M-).$/
+        (bare[-1].ord & 0b1001_1111 | 0b1000_0000).chr
+      when /^[0-7]+/
+        bare.to_i(8).chr
+      when "\n"
+        bare
+      else
+        bare
+      end
+    end
   end
 end

data/lib/ripper_ruby_parser/version.rb

@@ -1,3 +1,3 @@
 module RipperRubyParser
-  VERSION = '1.4.1'.freeze
+  VERSION = '1.4.2'.freeze
 end

data/test/end_to_end/comparison_test.rb

@@ -99,78 +99,4 @@ describe 'Using RipperRubyParser and RubyParser' do
       program.must_be_parsed_as_before
     end
   end
-
-  describe 'for an example with regular expressions with different encoding flags' do
-    it 'gives the same result' do
-      program = <<-END
-        regular = /foo/
-        noenc = /foo/n
-        utf8 = /foo/u
-        euc = /foo/e
-        sjis = /foo/s
-
-        regular = /foo\#{bar}/
-        noenc = /foo\#{bar}/n
-        utf8 = /foo\#{bar}/u
-        euc = /foo\#{bar}/e
-        sjis = /foo\#{bar}/s
-      END
-
-      program.must_be_parsed_as_before
-    end
-  end
-
-  describe 'for an example with __ENCODING__' do
-    it 'gives the same result' do
-      program = 'foo = __ENCODING__'
-      program.must_be_parsed_as_before
-    end
-  end
-
-  describe 'for an example with self[]' do
-    # https://github.com/seattlerb/ruby_parser/issues/250
-    it 'gives the same result' do
-      program = 'self[:foo]'
-      program.must_be_parsed_as_before
-    end
-  end
-
-  describe 'for an example with required keyword arguments and no parentheses' do
-    # https://github.com/seattlerb/ruby_parser/pull/254
-    let(:program) do
-      <<-END
-      def foo a:, b:
-        # body
-      end
-      END
-    end
-
-    it 'gives the same result' do
-      program.must_be_parsed_as_before
-    end
-  end
-
-  describe 'for an example combining begin..end and diverse operators' do
-    let(:program) do
-      <<-END
-      begin end
-      begin; foo; end
-      begin; foo; bar; end
-      - begin; foo; end
-      begin; bar; end + foo
-      foo + begin; bar; end
-      begin; foo; end ? bar : baz
-      foo ? begin; bar; end : baz
-      foo ? bar : begin; baz; end
-      begin; bar; end and foo
-      foo and begin; bar; end
-      begin; foo; end if bar
-      begin; foo; end unless bar
-      END
-    end
-
-    it 'gives the same result' do
-      program.must_be_parsed_as_before
-    end
-  end
 end

data/test/end_to_end/test_comparison_test.rb

@@ -10,7 +10,7 @@ describe 'Using RipperRubyParser and RubyParser' do
     RubyParser.new
   end
 
-  Dir.glob('test/**/*.rb').each do |file|
+  Dir.glob('test/ripper_ruby_parser/**/*.rb').each do |file|
     describe "for #{file}" do
       let :program do
         File.read file

data/test/ripper_ruby_parser/parser_test.rb

@@ -845,54 +845,6 @@ describe RipperRubyParser::Parser do
           must_be_parsed_as s(:call, s(:lit, 1), :!)
       end
 
-      it 'handles the range operator with positive number literals' do
-        '1..2'.
-          must_be_parsed_as s(:lit, 1..2)
-      end
-
-      it 'handles the range operator with negative number literals' do
-        '-1..-2'.
-          must_be_parsed_as s(:lit, -1..-2)
-      end
-
-      it 'handles the range operator with string literals' do
-        "'a'..'z'".
-          must_be_parsed_as s(:dot2,
-                              s(:str, 'a'),
-                              s(:str, 'z'))
-      end
-
-      it 'handles the range operator with non-literals' do
-        'foo..bar'.
-          must_be_parsed_as s(:dot2,
-                              s(:call, nil, :foo),
-                              s(:call, nil, :bar))
-      end
-
-      it 'handles the exclusive range operator with positive number literals' do
-        '1...2'.
-          must_be_parsed_as s(:lit, 1...2)
-      end
-
-      it 'handles the exclusive range operator with negative number literals' do
-        '-1...-2'.
-          must_be_parsed_as s(:lit, -1...-2)
-      end
-
-      it 'handles the exclusive range operator with string literals' do
-        "'a'...'z'".
-          must_be_parsed_as s(:dot3,
-                              s(:str, 'a'),
-                              s(:str, 'z'))
-      end
-
-      it 'handles the exclusive range operator with non-literals' do
-        'foo...bar'.
-          must_be_parsed_as s(:dot3,
-                              s(:call, nil, :foo),
-                              s(:call, nil, :bar))
-      end
-
       it 'handles the ternary operator' do
         'foo ? bar : baz'.
           must_be_parsed_as s(:if,

data/test/ripper_ruby_parser/sexp_handlers/literals_test.rb

@@ -46,6 +46,23 @@ describe RipperRubyParser::Parser do
         result.inspect.must_equal s(:lit, /foo/n).inspect
       end
 
+      it 'works with line continuation' do
+        "/foo\\\nbar/".
+          must_be_parsed_as s(:lit, /foobar/)
+      end
+
+      describe 'for a %r-delimited regex literal' do
+        it 'works for the simple case with escape sequences' do
+          '%r[foo\nbar]'.
+            must_be_parsed_as s(:lit, /foo\nbar/)
+        end
+
+        it 'works with odd delimiters and escape sequences' do
+          '%r_foo\nbar_'.
+            must_be_parsed_as s(:lit, /foo\nbar/)
+        end
+      end
+
       describe 'with interpolations' do
         it 'works for a simple interpolation' do
           '/foo#{bar}baz/'.
@@ -139,6 +156,16 @@ describe RipperRubyParser::Parser do
         result[1].encoding.to_s.must_equal 'UTF-8'
       end
 
+      it 'handles line continuation with double-quoted strings' do
+        "\"foo\\\nbar\"".
+          must_be_parsed_as s(:str, 'foobar')
+      end
+
+      it 'escapes line continuation with double-quoted strings' do
+        "\"foo\\\\\nbar\"".
+          must_be_parsed_as s(:str, "foo\\\nbar")
+      end
+
       describe 'with double-quoted strings with escape sequences' do
         it 'works for strings with escape sequences' do
           '"\\n"'.
@@ -213,6 +240,17 @@ describe RipperRubyParser::Parser do
         it 'works with unicode escapes (unlike RubyParser)' do
           '"foo\\u273bbar"'.must_be_parsed_as s(:str, 'foo✻bar')
         end
+
+        it 'converts to unicode if possible' do
+          '"2\302\275"'.must_be_parsed_as s(:str, '2½')
+        end
+
+        it 'does not convert to unicode if result is not valid' do
+          parser = RipperRubyParser::Parser.new
+          result = parser.parse '"2\x82\302\275"'
+          expected = s(:str, "2\x82\xC2\xBD".force_encoding(Encoding::US_ASCII))
+          result.inspect.must_equal expected.inspect
+        end
       end
 
       describe 'with interpolations' do
@@ -332,6 +370,11 @@ describe RipperRubyParser::Parser do
           "'foo\\\\\\abar'".
             must_be_parsed_as s(:str, 'foo\\\\abar')
         end
+
+        it 'does not process line continuation' do
+          "'foo\\\nbar'".
+            must_be_parsed_as s(:str, "foo\\\nbar")
+        end
       end
 
       describe 'with %Q-delimited strings' do
@@ -344,6 +387,28 @@ describe RipperRubyParser::Parser do
           '%Q[foo\\nbar]'.
             must_be_parsed_as s(:str, "foo\nbar")
         end
+
+        it 'handles line continuation' do
+          "%Q[foo\\\nbar]".
+            must_be_parsed_as s(:str, 'foobar')
+        end
+      end
+
+      describe 'with %-delimited strings' do
+        it 'works for the simple case' do
+          '%(bar)'.
+            must_be_parsed_as s(:str, 'bar')
+        end
+
+        it 'works for escape sequences' do
+          '%(foo\nbar)'.
+            must_be_parsed_as s(:str, "foo\nbar")
+        end
+
+        it 'works for odd delimiters' do
+          '%!foo\nbar!'.
+            must_be_parsed_as s(:str, "foo\nbar")
+        end
       end
 
       describe 'with string concatenation' do
@@ -406,16 +471,45 @@ describe RipperRubyParser::Parser do
           "<<FOO\nbar\\tbaz\nFOO".
             must_be_parsed_as s(:str, "bar\tbaz\n")
         end
+
+        it 'handles line continuation' do
+          "<<FOO\nbar\\\nbaz\nFOO".
+            must_be_parsed_as s(:str, "barbaz\n")
+        end
+
+        it 'escapes line continuation' do
+          "<<FOO\nbar\\\\\nbaz\nFOO".
+            must_be_parsed_as s(:str, "bar\\\nbaz\n")
+        end
+
+        it 'does not convert to unicode even if possible' do
+          parser = RipperRubyParser::Parser.new
+          result = parser.parse "<<FOO\n2\\302\\275\nFOO"
+          expected = s(:str, "2\xC2\xBD\n".force_encoding(Encoding::US_ASCII))
+          result.inspect.must_equal expected.inspect
+        end
       end
     end
 
-    describe 'for word list literals' do
-      it 'works for the simple case with %w' do
+    describe 'for word list literals with %w delimiter' do
+      it 'works for the simple case' do
         '%w(foo bar)'.
           must_be_parsed_as s(:array, s(:str, 'foo'), s(:str, 'bar'))
       end
 
-      it 'works for the simple case with %W' do
+      it 'does not perform interpolation' do
+        '%w(foo\\nbar baz)'.
+          must_be_parsed_as s(:array, s(:str, 'foo\\nbar'), s(:str, 'baz'))
+      end
+
+      it 'handles line continuation' do
+        "%w(foo\\\nbar baz)".
+          must_be_parsed_as s(:array, s(:str, "foo\nbar"), s(:str, 'baz'))
+      end
+    end
+
+    describe 'for word list literals with %W delimiter' do
+      it 'works for the simple case' do
         '%W(foo bar)'.
           must_be_parsed_as s(:array, s(:str, 'foo'), s(:str, 'bar'))
       end
@@ -452,6 +546,13 @@ describe RipperRubyParser::Parser do
                               s(:str, "foo\nbar"),
                               s(:str, 'baz'))
       end
+
+      it 'correctly handles line continuation' do
+        "%W(foo\\\nbar baz)".
+          must_be_parsed_as s(:array,
+                              s(:str, "foo\nbar"),
+                              s(:str, 'baz'))
+      end
     end
 
     describe 'for symbol list literals with %i delimiter' do
@@ -459,6 +560,16 @@ describe RipperRubyParser::Parser do
         '%i(foo bar)'.
           must_be_parsed_as s(:array, s(:lit, :foo), s(:lit, :bar))
       end
+
+      it 'does not perform interpolation' do
+        '%i(foo\\nbar baz)'.
+          must_be_parsed_as s(:array, s(:lit, :"foo\\nbar"), s(:lit, :baz))
+      end
+
+      it 'handles line continuation' do
+        "%i(foo\\\nbar baz)".
+          must_be_parsed_as s(:array, s(:lit, :"foo\nbar"), s(:lit, :baz))
+      end
     end
 
     describe 'for symbol list literals with %I delimiter' do
@@ -491,6 +602,13 @@ describe RipperRubyParser::Parser do
                                 s(:evstr, s(:call, nil, :bar)),
                                 s(:str, 'baz')))
       end
+
+      it 'correctly handles line continuation' do
+        "%I(foo\\\nbar baz)".
+          must_be_parsed_as s(:array,
+                              s(:lit, :"foo\nbar"),
+                              s(:lit, :baz))
+      end
     end
 
     describe 'for character literals' do