ripper_ruby_parser 1.4.2 → 1.5.0

data/test/ripper_ruby_parser/sexp_handlers/literals_test.rb

@@ -1,6 +1,10 @@
+# frozen_string_literal: true
+
 require File.expand_path('../../test_helper.rb', File.dirname(__FILE__))
 
 describe RipperRubyParser::Parser do
+  let(:parser) { RipperRubyParser::Parser.new }
+
   describe '#parse' do
     describe 'for regexp literals' do
       it 'works for a simple regex literal' do
@@ -18,6 +22,11 @@ describe RipperRubyParser::Parser do
           must_be_parsed_as s(:lit, /\)\n\\/)
       end
 
+      it 'does not fix encoding' do
+        '/2\302\275/'.
+          must_be_parsed_as s(:lit, /2\302\275/)
+      end
+
       it 'works for a regex literal with the multiline flag' do
         '/foo/m'.
           must_be_parsed_as s(:lit, /foo/m)
@@ -34,16 +43,13 @@ describe RipperRubyParser::Parser do
       end
 
       it 'works for a regex literal with a combination of flags' do
-        '/foo/ixm'.
-          must_be_parsed_as s(:lit, /foo/ixm)
+        '/foo/ixmn'.
+          must_be_parsed_as s(:lit, /foo/mixn)
       end
 
       it 'works with the no-encoding flag' do
-        parser = RipperRubyParser::Parser.new
-        result = parser.parse '/foo/n'
-        # Use inspect since regular == finds no difference between /foo/n
-        # and /foo/
-        result.inspect.must_equal s(:lit, /foo/n).inspect
+        '/foo/n'.
+          must_be_parsed_as s(:lit, /foo/n)
       end
 
       it 'works with line continuation' do
@@ -72,6 +78,15 @@ describe RipperRubyParser::Parser do
                                 s(:str, 'baz'))
         end
 
+        it 'works for a simple interpolation in extra-compatible mode' do
+          '/foo#{bar}baz/'.
+            must_be_parsed_as s(:dregx,
+                                'foo',
+                                s(:evstr, s(:call, nil, :bar)),
+                                s(:str, 'baz')),
+                              extra_compatible: true
+        end
+
         it 'works for a regex literal with flags and interpolation' do
           '/foo#{bar}/ixm'.
             must_be_parsed_as s(:dregx,
@@ -96,6 +111,14 @@ describe RipperRubyParser::Parser do
                                   s(:call, nil, :bar)), 16)
         end
 
+        it 'works with unicode flag plus other flag' do
+          '/foo#{bar}/un'.
+            must_be_parsed_as s(:dregx,
+                                'foo',
+                                s(:evstr,
+                                  s(:call, nil, :bar)), 48)
+        end
+
         it 'works with the euc-encoding flag' do
           '/foo#{bar}/e'.
             must_be_parsed_as s(:dregx,
@@ -156,6 +179,11 @@ describe RipperRubyParser::Parser do
         result[1].encoding.to_s.must_equal 'UTF-8'
       end
 
+      it 'handles line breaks within double-quoted strings' do
+        "\"foo\nbar\"".
+          must_be_parsed_as s(:str, "foo\nbar")
+      end
+
       it 'handles line continuation with double-quoted strings' do
         "\"foo\\\nbar\"".
           must_be_parsed_as s(:str, 'foobar')
@@ -235,21 +263,31 @@ describe RipperRubyParser::Parser do
           '"foo\\C-Zbar"'.must_be_parsed_as s(:str, "foo\C-Zbar")
         end
 
-        # TODO: Implement remaining escape sequence cases.
-
-        it 'works with unicode escapes (unlike RubyParser)' do
+        it 'works with unicode escapes' do
           '"foo\\u273bbar"'.must_be_parsed_as s(:str, 'foo✻bar')
         end
 
+        it 'works with unicode escapes in extra-compatible mode' do
+          '"foo\\u273bbar"'.
+            must_be_parsed_as s(:str, 'foo✻r'), extra_compatible: true
+        end
+
+        it 'works with unicode escapes with braces' do
+          '"foo\\u{273b}bar"'.must_be_parsed_as s(:str, 'foo✻bar')
+        end
+
         it 'converts to unicode if possible' do
           '"2\302\275"'.must_be_parsed_as s(:str, '2½')
         end
 
+        it 'converts to unicode if possible in extra-compatible mode' do
+          '"2\302\275"'.must_be_parsed_as s(:str, '2½'), extra_compatible: true
+        end
+
         it 'does not convert to unicode if result is not valid' do
-          parser = RipperRubyParser::Parser.new
-          result = parser.parse '"2\x82\302\275"'
-          expected = s(:str, "2\x82\xC2\xBD".force_encoding(Encoding::US_ASCII))
-          result.inspect.must_equal expected.inspect
+          '"2\x82\302\275"'.
+            must_be_parsed_as s(:str,
+                                (+"2\x82\xC2\xBD").force_encoding('ascii-8bit'))
         end
       end
 
@@ -348,6 +386,51 @@ describe RipperRubyParser::Parser do
                                 s(:evstr, s(:array, s(:lit, :foo), s(:str, 'bar'))),
                                 s(:str, "\n"))
         end
+
+        it 'converts to unicode after interpolation' do
+          '"#{foo}2\302\275"'.
+            must_be_parsed_as s(:dstr,
+                                '',
+                                s(:evstr, s(:call, nil, :foo)),
+                                s(:str, '2½'))
+        end
+
+        it 'convert single null byte to unicode after interpolation' do
+          '"#{foo}\0"'.
+            must_be_parsed_as s(:dstr,
+                                '',
+                                s(:evstr, s(:call, nil, :foo)),
+                                s(:str, "\u0000"))
+        end
+      end
+
+      describe 'with interpolations and escape sequences in extra-compatible mode' do
+        it 'does not convert to unicode after interpolation' do
+          '"#{foo}2\302\275"'.
+            must_be_parsed_as s(:dstr,
+                                '',
+                                s(:evstr, s(:call, nil, :foo)),
+                                s(:str, (+"2\xC2\xBD").force_encoding('ascii-8bit'))),
+                              extra_compatible: true
+        end
+
+        it 'keeps single null byte as ascii after interpolation' do
+          '"#{foo}\0"'.
+            must_be_parsed_as s(:dstr,
+                                '',
+                                s(:evstr, s(:call, nil, :foo)),
+                                s(:str, (+"\x00").force_encoding('ascii-8bit'))),
+                              extra_compatible: true
+        end
+
+        it 'converts string with null to unicode after interpolation' do
+          '"#{foo}bar\0"'.
+            must_be_parsed_as s(:dstr,
+                                '',
+                                s(:evstr, s(:call, nil, :foo)),
+                                s(:str, "bar\x00")),
+                              extra_compatible: true
+        end
       end
 
       describe 'with single quoted strings' do
@@ -375,6 +458,12 @@ describe RipperRubyParser::Parser do
           "'foo\\\nbar'".
             must_be_parsed_as s(:str, "foo\\\nbar")
         end
+
+        it 'handles escape sequences correctly in extra-compatible mode' do
+          "'foo\\'bar\\\nbaz\\aqux'".
+            must_be_parsed_as s(:str, "foo'bar\\\nbaz\\aqux"),
+                              extra_compatible: true
+        end
       end
 
       describe 'with %Q-delimited strings' do
@@ -388,12 +477,39 @@ describe RipperRubyParser::Parser do
             must_be_parsed_as s(:str, "foo\nbar")
         end
 
+        it 'works for multi-line strings' do
+          "%Q[foo\nbar]".
+            must_be_parsed_as s(:str, "foo\nbar")
+        end
+
         it 'handles line continuation' do
           "%Q[foo\\\nbar]".
             must_be_parsed_as s(:str, 'foobar')
         end
       end
 
+      describe 'with %q-delimited strings' do
+        it 'works for the simple case' do
+          '%q[bar]'.
+            must_be_parsed_as s(:str, 'bar')
+        end
+
+        it 'does not handle for escape sequences' do
+          '%q[foo\\nbar]'.
+            must_be_parsed_as s(:str, 'foo\nbar')
+        end
+
+        it 'works for multi-line strings' do
+          "%q[foo\nbar]".
+            must_be_parsed_as s(:str, "foo\nbar")
+        end
+
+        it 'handles line continuation' do
+          "%q[foo\\\nbar]".
+            must_be_parsed_as s(:str, "foo\\\nbar")
+        end
+      end
+
       describe 'with %-delimited strings' do
         it 'works for the simple case' do
           '%(bar)'.
@@ -405,6 +521,16 @@ describe RipperRubyParser::Parser do
             must_be_parsed_as s(:str, "foo\nbar")
         end
 
+        it 'works for multiple lines' do
+          "%(foo\nbar)".
+            must_be_parsed_as s(:str, "foo\nbar")
+        end
+
+        it 'works with line continuations' do
+          "%(foo\\\nbar)".
+            must_be_parsed_as s(:str, 'foobar')
+        end
+
         it 'works for odd delimiters' do
           '%!foo\nbar!'.
             must_be_parsed_as s(:str, "foo\nbar")
@@ -467,11 +593,58 @@ describe RipperRubyParser::Parser do
             must_be_parsed_as s(:str, "bar\n")
         end
 
+        it 'works with multiple lines' do
+          "<<FOO\nbar\nbaz\nFOO".
+            must_be_parsed_as s(:str, "bar\nbaz\n")
+        end
+
+        it 'works for the indentable case' do
+          "<<-FOO\n  bar\n  FOO".
+            must_be_parsed_as s(:str, "  bar\n")
+        end
+
+        it 'works for the automatically outdenting case' do
+          "  <<~FOO\n  bar\n  FOO".
+            must_be_parsed_as s(:str, "bar\n")
+        end
+
         it 'works for escape sequences' do
           "<<FOO\nbar\\tbaz\nFOO".
             must_be_parsed_as s(:str, "bar\tbaz\n")
         end
 
+        it 'works for escape sequences in extra-compatible mode' do
+          "<<FOO\nbar\\tbaz\nFOO".
+            must_be_parsed_as s(:str, "bar\tbaz\n"),
+                              extra_compatible: true
+        end
+
+        it 'does not unescape with single quoted version' do
+          "<<'FOO'\nbar\\tbaz\nFOO".
+            must_be_parsed_as s(:str, "bar\\tbaz\n")
+        end
+
+        it 'does not unescape with single quoted version in extra-compatible mode' do
+          "<<'FOO'\nbar\\tbaz\nFOO".
+            must_be_parsed_as s(:str, "bar\\tbaz\n"),
+                              extra_compatible: true
+        end
+
+        it 'works with multiple lines with the single quoted version' do
+          "<<'FOO'\nbar\nbaz\nFOO".
+            must_be_parsed_as s(:str, "bar\nbaz\n")
+        end
+
+        it 'does not unescape with indentable single quoted version' do
+          "<<-'FOO'\n  bar\\tbaz\n  FOO".
+            must_be_parsed_as s(:str, "  bar\\tbaz\n")
+        end
+
+        it 'does not unescape the automatically outdenting single quoted version' do
+          "<<~'FOO'\n  bar\\tbaz\n  FOO".
+            must_be_parsed_as s(:str, "bar\\tbaz\n")
+        end
+
         it 'handles line continuation' do
           "<<FOO\nbar\\\nbaz\nFOO".
             must_be_parsed_as s(:str, "barbaz\n")
@@ -482,11 +655,16 @@ describe RipperRubyParser::Parser do
             must_be_parsed_as s(:str, "bar\\\nbaz\n")
         end
 
-        it 'does not convert to unicode even if possible' do
-          parser = RipperRubyParser::Parser.new
-          result = parser.parse "<<FOO\n2\\302\\275\nFOO"
-          expected = s(:str, "2\xC2\xBD\n".force_encoding(Encoding::US_ASCII))
-          result.inspect.must_equal expected.inspect
+        it 'converts to unicode' do
+          "<<FOO\n2\\302\\275\nFOO".
+            must_be_parsed_as s(:str, "2½\n")
+        end
+
+        it 'does not convert to unicode in extra-compatible mode' do
+          "<<FOO\n2\\302\\275\nFOO".
+            must_be_parsed_as s(:str,
+                                (+"2\xC2\xBD\n").force_encoding('ascii-8bit')),
+                              extra_compatible: true
         end
       end
     end
@@ -506,6 +684,11 @@ describe RipperRubyParser::Parser do
         "%w(foo\\\nbar baz)".
           must_be_parsed_as s(:array, s(:str, "foo\nbar"), s(:str, 'baz'))
       end
+
+      it 'handles escaped spaces' do
+        '%w(foo bar\ baz)'.
+          must_be_parsed_as s(:array, s(:str, 'foo'), s(:str, 'bar baz'))
+      end
     end
 
     describe 'for word list literals with %W delimiter' do
@@ -514,6 +697,11 @@ describe RipperRubyParser::Parser do
           must_be_parsed_as s(:array, s(:str, 'foo'), s(:str, 'bar'))
       end
 
+      it 'handles escaped spaces' do
+        '%W(foo bar\ baz)'.
+          must_be_parsed_as s(:array, s(:str, 'foo'), s(:str, 'bar baz'))
+      end
+
       it 'correctly handles interpolation' do
         "%W(foo \#{bar} baz)".
           must_be_parsed_as  s(:array,
@@ -547,12 +735,45 @@ describe RipperRubyParser::Parser do
                               s(:str, 'baz'))
       end
 
+      it 'converts to unicode if possible' do
+        '%W(2\302\275)'.must_be_parsed_as s(:array, s(:str, '2½'))
+      end
+
+      it 'does not convert to unicode if possible in extra-compatible mode' do
+        '%W(2\302\275)'.
+          must_be_parsed_as s(:array,
+                              s(:str, (+"2\xC2\xBD").force_encoding('ascii-8bit'))),
+                            extra_compatible: true
+      end
+
       it 'correctly handles line continuation' do
         "%W(foo\\\nbar baz)".
           must_be_parsed_as s(:array,
                               s(:str, "foo\nbar"),
                               s(:str, 'baz'))
       end
+
+      it 'correctly handles multiple lines' do
+        "%W(foo\nbar baz)".
+          must_be_parsed_as s(:array,
+                              s(:str, 'foo'),
+                              s(:str, 'bar'),
+                              s(:str, 'baz'))
+      end
+
+      it 'handles escaped spaces in extra-compatible mode' do
+        '%W(foo bar\ baz)'.
+          must_be_parsed_as s(:array, s(:str, 'foo'), s(:str, 'bar baz')),
+                            extra_compatible: true
+      end
+
+      it 'correctly handles line continuation in extra-compatible mode' do
+        "%W(foo\\\nbar baz)".
+          must_be_parsed_as s(:array,
+                              s(:str, "foo\nbar"),
+                              s(:str, 'baz')),
+                            extra_compatible: true
+      end
     end
 
     describe 'for symbol list literals with %i delimiter' do
@@ -609,6 +830,14 @@ describe RipperRubyParser::Parser do
                               s(:lit, :"foo\nbar"),
                               s(:lit, :baz))
       end
+
+      it 'correctly handles multiple lines' do
+        "%I(foo\nbar baz)".
+          must_be_parsed_as s(:array,
+                              s(:lit, :foo),
+                              s(:lit, :bar),
+                              s(:lit, :baz))
+      end
     end
 
     describe 'for character literals' do
@@ -629,27 +858,27 @@ describe RipperRubyParser::Parser do
 
       it 'works for escaped character literals with meta' do
         '?\\M-a'.
-          must_be_parsed_as s(:str, "\xE1".force_encoding('ascii-8bit'))
+          must_be_parsed_as s(:str, (+"\xE1").force_encoding('ascii-8bit'))
       end
 
       it 'works for escaped character literals with meta plus shorthand ctrl' do
         '?\\M-\\ca'.
-          must_be_parsed_as s(:str, "\x81".force_encoding('ascii-8bit'))
+          must_be_parsed_as s(:str, (+"\x81").force_encoding('ascii-8bit'))
       end
 
       it 'works for escaped character literals with shorthand ctrl plus meta' do
         '?\\c\\M-a'.
-          must_be_parsed_as s(:str, "\x81".force_encoding('ascii-8bit'))
+          must_be_parsed_as s(:str, (+"\x81").force_encoding('ascii-8bit'))
       end
 
       it 'works for escaped character literals with meta plus ctrl' do
         '?\\M-\\C-a'.
-          must_be_parsed_as s(:str, "\x81".force_encoding('ascii-8bit'))
+          must_be_parsed_as s(:str, (+"\x81").force_encoding('ascii-8bit'))
       end
 
       it 'works for escaped character literals with ctrl plus meta' do
         '?\\C-\\M-a'.
-          must_be_parsed_as s(:str, "\x81".force_encoding('ascii-8bit'))
+          must_be_parsed_as s(:str, (+"\x81").force_encoding('ascii-8bit'))
       end
     end
 
@@ -664,6 +893,25 @@ describe RipperRubyParser::Parser do
           must_be_parsed_as s(:lit, :@foo)
       end
 
+      it 'works for symbols that look like class names' do
+        ':Foo'.
+          must_be_parsed_as s(:lit, :Foo)
+      end
+
+      it 'works for symbols that look like keywords' do
+        ':class'.must_be_parsed_as s(:lit, :class)
+      end
+
+      it 'works for :__LINE__' do
+        ':__LINE__'.
+          must_be_parsed_as s(:lit, :__LINE__)
+      end
+
+      it 'works for :__FILE__' do
+        ':__FILE__'.
+          must_be_parsed_as s(:lit, :__FILE__)
+      end
+
       it 'works for simple dsyms' do
         ':"foo"'.
           must_be_parsed_as s(:lit, :foo)
@@ -676,11 +924,28 @@ describe RipperRubyParser::Parser do
                               s(:evstr, s(:call, nil, :bar)))
       end
 
+      it 'works for dsyms with interpolations at the start' do
+        ':"#{bar}"'.
+          must_be_parsed_as s(:dsym,
+                              '',
+                              s(:evstr, s(:call, nil, :bar)))
+      end
+
       it 'works for dsyms with escape sequences' do
         ':"foo\nbar"'.
           must_be_parsed_as s(:lit, :"foo\nbar")
       end
 
+      it 'works for dsyms with multiple lines' do
+        ":\"foo\nbar\"".
+          must_be_parsed_as s(:lit, :"foo\nbar")
+      end
+
+      it 'works for dsyms with line continuations' do
+        ":\"foo\\\nbar\"".
+          must_be_parsed_as s(:lit, :foobar)
+      end
+
       it 'works with single quoted dsyms' do
         ":'foo'".
           must_be_parsed_as s(:lit, :foo)
@@ -691,10 +956,31 @@ describe RipperRubyParser::Parser do
           must_be_parsed_as s(:lit, :'foo\'bar')
       end
 
+      it 'works with single quoted dsyms with multiple lines' do
+        ":'foo\nbar'".
+          must_be_parsed_as s(:lit, :"foo\nbar")
+      end
+
+      it 'works with single quoted dsyms with line continuations' do
+        ":'foo\\\nbar'".
+          must_be_parsed_as s(:lit, :"foo\\\nbar")
+      end
+
       it 'works with single quoted dsyms with embedded backslashes' do
         ":'foo\\abar'".
           must_be_parsed_as s(:lit, :"foo\\abar")
       end
+
+      it 'works with barewords that need to be interpreted as symbols' do
+        'alias foo bar'.
+          must_be_parsed_as s(:alias,
+                              s(:lit, :foo), s(:lit, :bar))
+      end
+
+      it 'assigns a line number to the result' do
+        result = parser.parse ':foo'
+        result.line.must_equal 1
+      end
     end
 
     describe 'for backtick string literals' do
@@ -710,10 +996,26 @@ describe RipperRubyParser::Parser do
                               s(:evstr, s(:call, nil, :bar)))
       end
 
+      it 'works for backtick strings interpolated at the start' do
+        '`#{foo}`'.
+          must_be_parsed_as s(:dxstr, '',
+                              s(:evstr, s(:call, nil, :foo)))
+      end
+
       it 'works for backtick strings with escape sequences' do
         '`foo\\n`'.
           must_be_parsed_as s(:xstr, "foo\n")
       end
+
+      it 'works for backtick strings with multiple lines' do
+        "`foo\nbar`".
+          must_be_parsed_as s(:xstr, "foo\nbar")
+      end
+
+      it 'works for backtick strings with line continuations' do
+        "`foo\\\nbar`".
+          must_be_parsed_as s(:xstr, 'foobar')
+      end
     end
 
     describe 'for array literals' do
@@ -818,6 +1120,11 @@ describe RipperRubyParser::Parser do
         '+1'.
           must_be_parsed_as s(:lit, 1)
       end
+
+      it 'works for rationals' do
+        '1000r'.
+          must_be_parsed_as s(:lit, 1000r)
+      end
     end
   end
 end