ruby_parser 3.8.2 → 3.8.3

data/lib/ruby_parser_extras.rb

@@ -91,7 +91,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = "3.8.2" unless constants.include? "VERSION" # SIGH
+  VERSION = "3.8.3" unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -875,11 +875,73 @@ module RubyParserStuff
     str = val[0]
     str.force_encoding("ASCII-8BIT") unless str.valid_encoding? unless RUBY_VERSION < "1.9"
     result = s(:str, str)
-    self.lexer.lineno += str.count("\n") + self.lexer.extra_lineno
-    self.lexer.extra_lineno = 0
+    self.lexer.fixup_lineno str.count("\n")
     result
   end
 
+  def new_qword_list_entry val
+    str = val[1]
+    str.force_encoding("ASCII-8BIT") unless str.valid_encoding? unless RUBY_VERSION < "1.9"
+    result = s(:str, str)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_qword_list
+    result = s(:array)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_word_list
+    result = s(:array)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_word_list_entry val
+    result = val[1][0] == :evstr ? s(:dstr, "", val[1]) : val[1]
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_qsym_list
+    result = s(:array)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_qsym_list_entry val
+    result = s(:lit, val[1].to_sym)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_symbol_list
+    result = s(:array)
+    self.lexer.fixup_lineno
+    result
+  end
+
+  def new_symbol_list_entry val
+    _list, sym, _nil = val # TODO: use _list
+    result = val[1]
+
+    result ||= s(:str, "")
+
+    case sym[0]
+    when :dstr then
+      sym[0] = :dsym
+    when :str then
+      sym = s(:lit, sym.last.to_sym)
+    else
+      debug20 24
+      sym = s(:dsym, "", sym || s(:str, ""))
+    end
+    self.lexer.fixup_lineno
+    sym
+  end
+
   def new_super args
     if args && args.node_type == :block_pass then
       s(:super, args)

data/test/test_ruby_lexer.rb

@@ -89,6 +89,8 @@ class TestRubyLexer < Minitest::Test
     case value
     when Float then
       assert_in_epsilon value, act_value, 0.001, msg
+    when NilClass then
+      assert_nil act_value, msg
     else
       assert_equal value, act_value,       msg
     end
@@ -967,7 +969,17 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_float_e_bad_double_e
-    refute_lex "1e2e3"
+    assert_lex3("1e2e3",
+                nil,
+                :tFLOAT, 100, :expr_end,
+                :tIDENTIFIER, "e3", :expr_end)
+  end
+
+  def test_yylex_float_if_modifier
+    assert_lex3("1e2if",
+                nil,
+                :tFLOAT, 100, :expr_end,
+                :kIF_MOD, "if", :expr_beg)
   end
 
   def test_yylex_float_e_bad_trailing_underscore
@@ -1409,6 +1421,13 @@ class TestRubyLexer < Minitest::Test
     refute_lex "0d42__24"
   end
 
+  def test_yylex_integer_if_modifier
+    assert_lex3("123if",
+                nil,
+                :tINTEGER, 123, :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+  end
+
   def test_yylex_question_eh_a__18
     setup_lexer_class Ruby18Parser
 
@@ -2698,25 +2717,80 @@ class TestRubyLexer < Minitest::Test
                :tRCURLY, "}", :expr_endarg,   0, 0)
   end
 
-  def test_ruby21_new_numbers
-    skip "Don't have imaginary and rational literal lexing yet"
+  def test_ruby21_rational_literal
+    setup_lexer_class Ruby21Parser
 
+    assert_lex3("10r",      nil, :tRATIONAL, Rational(10), :expr_end)
+    assert_lex3("0x10r",      nil, :tRATIONAL, Rational(16), :expr_end)
+    assert_lex3("0o10r",      nil, :tRATIONAL, Rational(8), :expr_end)
+    assert_lex3("0or",      nil, :tRATIONAL, Rational(0), :expr_end)
+    assert_lex3("0b10r",      nil, :tRATIONAL, Rational(2), :expr_end)
+    assert_lex3("1.5r",    nil, :tRATIONAL, Rational(15, 10), :expr_end)
+    assert_lex3("15e3r",    nil, :tRATIONAL, Rational(15000), :expr_end)
+    assert_lex3("15e-3r",    nil, :tRATIONAL, Rational(15, 1000), :expr_end)
+    assert_lex3("1.5e3r",    nil, :tRATIONAL, Rational(1500), :expr_end)
+    assert_lex3("1.5e-3r",    nil, :tRATIONAL, Rational(15, 10000), :expr_end)
+
+    assert_lex3("-10r", nil,
+                :tUMINUS_NUM, "-", :expr_beg,
+                :tRATIONAL, Rational(10), :expr_end)
+  end
+
+  def test_ruby21_imaginary_literal
     setup_lexer_class Ruby21Parser
 
-    assert_lex3("10r",      nil, :tRATIONAL, "10r", :expr_end)
-    assert_lex3("1.5r",     nil, :tRATIONAL, "1.5r", :expr_end)
+    assert_lex3("1i",       nil, :tIMAGINARY, Complex(0, 1), :expr_end)
+    assert_lex3("0x10i",    nil, :tIMAGINARY, Complex(0, 16), :expr_end)
+    assert_lex3("0o10i",    nil, :tIMAGINARY, Complex(0, 8), :expr_end)
+    assert_lex3("0oi",      nil, :tIMAGINARY, Complex(0, 0), :expr_end)
+    assert_lex3("0b10i",    nil, :tIMAGINARY, Complex(0, 2), :expr_end)
+    assert_lex3("1.5i",    nil, :tIMAGINARY, Complex(0, 1.5), :expr_end)
+    assert_lex3("15e3i",    nil, :tIMAGINARY, Complex(0, 15000), :expr_end)
+    assert_lex3("15e-3i",    nil, :tIMAGINARY, Complex(0, 0.015), :expr_end)
+    assert_lex3("1.5e3i",    nil, :tIMAGINARY, Complex(0, 1500), :expr_end)
+    assert_lex3("1.5e-3i",    nil, :tIMAGINARY, Complex(0, 0.0015), :expr_end)
+
+    assert_lex3("-10i", nil,
+                :tUMINUS_NUM, "-", :expr_beg,
+                :tIMAGINARY, Complex(0, 10), :expr_end)
+  end
 
-    assert_lex3("1i",       nil, :tIMAGINARY, "1i", :expr_end)
-    assert_lex3("1+2i",     nil, :tIMAGINARY, "1+2i", :expr_end)
-    assert_lex3("1.2+3.4i", nil, :tIMAGINARY, "1.2+3.4i", :expr_end)
-    assert_lex3("4r+3i",    nil, :tIMAGINARY, "4r+3i", :expr_end)
-    assert_lex3("4r+3ri",   nil, :tIMAGINARY, "4r+3i", :expr_end)
+  def test_ruby21_rational_imaginary_literal
+    setup_lexer_class Ruby21Parser
 
-    assert_lex3("4i+3r",    nil, :tIMAGINARY, "4r+3i", :expr_end) # HACK
-    assert_lex3("1i+2ri",   nil, :tIMAGINARY, "4r+3i", :expr_end) # HACK
+    assert_lex3("1ri",       nil, :tIMAGINARY, Complex(0, Rational(1)), :expr_end)
+    assert_lex3("0x10ri",    nil, :tIMAGINARY, Complex(0, Rational(16)), :expr_end)
+    assert_lex3("0o10ri",    nil, :tIMAGINARY, Complex(0, Rational(8)), :expr_end)
+    assert_lex3("0ori",      nil, :tIMAGINARY, Complex(0, Rational(0)), :expr_end)
+    assert_lex3("0b10ri",    nil, :tIMAGINARY, Complex(0, Rational(2)), :expr_end)
+    assert_lex3("1.5ri",    nil, :tIMAGINARY, Complex(0, Rational("1.5")), :expr_end)
+    assert_lex3("15e3ri",    nil, :tIMAGINARY, Complex(0, Rational("15e3")), :expr_end)
+    assert_lex3("15e-3ri",    nil, :tIMAGINARY, Complex(0, Rational("15e-3")), :expr_end)
+    assert_lex3("1.5e3ri",    nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), :expr_end)
+    assert_lex3("1.5e-3ri",    nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), :expr_end)
+
+    assert_lex3("-10ri", nil,
+                :tUMINUS_NUM, "-", :expr_beg,
+                :tIMAGINARY, Complex(0, Rational(10)), :expr_end)
+  end
+
+  def test_ruby21_imaginary_literal_with_succeeding_keyword
+    skip "Currently does not tokenize correctly"
+
+    setup_lexer_class Ruby21Parser
 
-    assert_lex3("1+2ri",    nil, :tIMAGINARY, "1+3ri", :expr_end)
-    refute_lex("1+2ir", :tINTEGER, 1)
+    assert_lex3("1if", nil,
+                :tINTEGER, 1, :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+    assert_lex3("1rif", nil,
+                :tRATIONAL, Rational(1), :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+    assert_lex3("1.0if", nil,
+                :tFLOAT, 1.0, :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+    assert_lex3("1.0rif", nil,
+                :tRATIONAL, Rational("1.0"), :expr_end,
+                :kIF_MOD, "if", :expr_beg)
 
     flunk
   end

data/test/test_ruby_parser.rb

@@ -45,6 +45,11 @@ class RubyParserTestCase < ParseTreeTestCase
     assert_equal pt, result
   end
 
+  def refute_parse rb
+    self.result = processor.parse rb
+    assert_nil result
+  end
+
   def assert_syntax_error rb, emsg
     e = nil
     assert_silent do
@@ -302,10 +307,9 @@ module TestRubyParserShared
 
   def test_bug_comment_eq_begin
     rb = "\n\n#\n=begin\nblah\n=end\n\n"
-    pt = nil
     exp = rb.strip + "\n"
 
-    assert_parse rb, pt
+    refute_parse rb
     assert_equal exp, processor.lexer.comments
   end
 
@@ -366,10 +370,7 @@ module TestRubyParserShared
   end
 
   def test_empty
-    rb = ""
-    pt = nil
-
-    assert_parse rb, pt
+    refute_parse ""
   end
 
   def test_evstr_evstr
@@ -1401,8 +1402,29 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
-  def test_qwords_line_breaks
-    skip "not yet"
+  def test_array_line_breaks
+    # It seems like arrays are roughly created when a certain element is created
+    # In ruby > 1.9 it seems like that is after the last element, so the array
+    # itself is assigned line 3 (since the last element is on line 3) and for
+    # ruby <= 1.9 it seems to get created after the first element, so the array
+    # itself is assigned line 2 (since the first element is on line 2).
+    # This seems to happen since arrays like this are created with a line in
+    # ruby_parser.yy like `result = s(:array, val[0])`. So, the array is not
+    # created by itself. The creation of the array itself is deferred until there
+    # is an element to create it with. That seems to mess up line numbers
+    # for the array. Luckily, the arary elements all seemt to get the correct
+    # line number.
+    start_line = self.class.to_s =~ /1[89]/ ? 2 : 3
+    rb = "[\n'a',\n'b']\n1"
+    pt = s(:block,
+           s(:array,
+             s(:str, "a").line(2),
+             s(:str, "b").line(3)).line(start_line),
+           s(:lit, 1).line(4))
+    assert_parse rb, pt
+  end
+
+  def test_non_interpolated_word_array_line_breaks
 
     rb = "%w(\na\nb\n)\n1"
     pt = s(:block,
@@ -1410,7 +1432,17 @@ module TestRubyParserShared
              s(:str, "a").line(2),
              s(:str, "b").line(3)).line(1),
            s(:lit, 1).line(5))
+    assert_parse rb, pt
+  end
+
+  def test_interpolated_word_array_line_breaks
 
+    rb = "%W(\na\nb\n)\n1"
+    pt = s(:block,
+           s(:array,
+             s(:str, "a").line(2),
+             s(:str, "b").line(3)).line(1),
+           s(:lit, 1).line(5))
     assert_parse rb, pt
   end
 
@@ -2187,6 +2219,28 @@ module TestRubyParserShared19to22
 end
 
 module TestRubyParserShared20to22
+  def test_non_interpolated_symbol_array_line_breaks
+
+    rb = "%i(\na\nb\n)\n1"
+    pt = s(:block,
+           s(:array,
+             s(:lit, :a).line(2),
+             s(:lit, :b).line(3)).line(1),
+           s(:lit, 1).line(5))
+    assert_parse rb, pt
+  end
+
+  def test_interpolated_symbol_array_line_breaks
+
+    rb = "%I(\na\nb\n)\n1"
+    pt = s(:block,
+           s(:array,
+             s(:lit, :a).line(2),
+             s(:lit, :b).line(3)).line(1),
+           s(:lit, 1).line(5))
+    assert_parse rb, pt
+  end
+
   def test_defs_kwarg
     rb = "def self.a b: 1\nend"
     pt = s(:defs, s(:self), :a, s(:args, s(:kwarg, :b, s(:lit, 1))), s(:nil))
@@ -3504,8 +3558,47 @@ class TestRuby23Parser < RubyParserTestCase
 
     assert_parse rb, pt
   end
-end
 
+  def test_ruby21_numbers
+    rb = "[1i, 2r, 3ri]"
+    pt = s(:array, s(:lit, Complex(0, 1)), s(:lit, Rational(2)), s(:lit, Complex(0, Rational(3))))
+
+    assert_parse rb, pt
+  end
+
+  def test_float_with_if_modifier
+    rb = "1.0if true"
+    pt = s(:if, s(:true), s(:lit, 1.0), nil)
+
+    assert_parse rb, pt
+  end
+
+  def test_integer_with_if_modifier
+    rb = "1_234if true"
+    pt = s(:if, s(:true), s(:lit, 1234), nil)
+
+    assert_parse rb, pt
+  end
+
+  def test_slashy_newlines_within_string
+    rb = %(puts "hello\\
+ my\\
+ dear\\
+ friend"
+
+a + b
+    )
+
+    pt = s(:block,
+            s(:call, nil, :puts, s(:str, "hello my dear friend").line(1)).line(1),
+            s(:call, s(:call, nil, :a).line(6),
+              :+,
+              s(:call, nil, :b).line(6)).line(6)
+          ).line(1)
+
+    assert_parse rb, pt
+  end
+end
 
 [18, 19, 20, 21, 22, 23].each do |v|
   describe "block args arity #{v}" do

data/test/test_ruby_parser_extras.rb

@@ -182,7 +182,7 @@ class TestEnvironment < Minitest::Test
 
   def test_var_scope_static
     @env.extend
-    assert_equal nil, @env[:blah]
+    assert_nil @env[:blah]
     @env.unextend
     assert_equal 42, @env[:blah]
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_parser
 version: !ruby/object:Gem::Version
-  version: 3.8.2
+  version: 3.8.3
 platform: ruby
 authors:
 - Ryan Davis
@@ -10,9 +10,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBAzANBgkqhkiG9w0BAQUFADBFMRMwEQYDVQQDDApyeWFu
+  MIIDijCCAnKgAwIBAgIBATANBgkqhkiG9w0BAQUFADBFMRMwEQYDVQQDDApyeWFu
   ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTE1MDkxOTIwNTEyMloXDTE2MDkxODIwNTEyMlowRTETMBEGA1UE
+  GRYDY29tMB4XDTE2MDkyNjAxNTczNVoXDTE3MDkyNjAxNTczNVowRTETMBEGA1UE
   AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
   JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
   b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
@@ -20,16 +20,17 @@ cert_chain:
   oOvjtt5P8+GSK9zLzxQP0gVLS/D0FmoE44XuDr3iQkVS2ujU5zZL84mMNqNB1znh
   GiadM9GHRaDiaxuX0cIUBj19T01mVE2iymf9I6bEsiayK/n6QujtyCbTWsAS9Rqt
   qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
-  gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
-  HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBBQUAA4IB
-  AQB+Hx8xUgrpZa4P8H8gR8zme5kISwQrG80MbpqJV6/G3/ZicRFhN5sjwu0uHGue
-  bd9Cymf6oIRwHVarJux2M32T6bL07Hmi07w2QaPc3MnMKB/D46SRZ2JSSGPFRBTc
-  SilobMRoGs/7B15uGFUEnNrCB/ltMqhwwSx1r++UQPfeySHEV9uqu03E5Vb7J37O
-  2Er6PLXHRiYsIycD1LkMi6YnixdITRHmrqJYE2rsjaIfpIehiusVAPHkNf7qbpHq
-  qx3h45R1CAsObX0SQDIT+rRbQrtKz1GHIZTOFYvEJjUY1XmRTZupD3CJ8Q7sDqSy
-  NLq5jm1fq6Y9Uolu3RJbmycf
+  gBEfoTEGr7Zii72cx+sCAwEAAaOBhDCBgTAJBgNVHRMEAjAAMAsGA1UdDwQEAwIE
+  sDAdBgNVHQ4EFgQUR8V72Z3+v+2P9abCnL4wjx32T+EwIwYDVR0RBBwwGoEYcnlh
+  bmQtcnVieUB6ZW5zcGlkZXIuY29tMCMGA1UdEgQcMBqBGHJ5YW5kLXJ1YnlAemVu
+  c3BpZGVyLmNvbTANBgkqhkiG9w0BAQUFAAOCAQEAIGzgp0aZ2W9+v96ujmBcQHoC
+  buy0iU68MVj2VlxMyfr1KPZIh1OyhU4UO4zrkREcH8ML70v9cYHNvOd9oynRHnvC
+  l2tj/fD3YJ0AEkJxGrYwRWQmvMfC4bJ02bC1+rVOUIXXKp3+cUmiN4sTniof8VFo
+  bo/YYP4c7erpERa+9hrqygg6WQbJlk2YRlH3JXPFjmu869i2dcbR5ZLOAeEy+axH
+  E4oJcnPkJAr0rw504JGtlZtONZQblwmRJOIdXzolaE3NRGUzGVOUSptZppAKiavY
+  fO6tdKQc/5RfA8oQEkg8hrxA5PQSz4TOFJGLpFvIapEk6tMruQ0bHgkhr9auXg==
   -----END CERTIFICATE-----
-date: 2016-05-05 00:00:00.000000000 Z
+date: 2016-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sexp_processor
@@ -46,61 +47,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '4.1'
 - !ruby/object:Gem::Dependency
-  name: rdoc
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - <
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - <
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '11'
 - !ruby/object:Gem::Dependency
-  name: racc
+  name: oedipus_lex
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.6
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.4.6
+        version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '4.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
-  name: oedipus_lex
+  name: racc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 1.4.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 1.4.6
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement