ruby_parser 3.3.0 → 3.4.0

data/lib/ruby_parser_extras.rb

@@ -59,28 +59,8 @@ class RPStringScanner < StringScanner
     end
   end
 
-  def current_line # HAHA fuck you (HACK)
-    string_to_pos[/\A.*__LINE__/m].split(/\n/).size
-  end
-
-  def extra_lines_added
-    @extra_lines_added ||= 0
-  end
-
-  def extra_lines_added= val
-    @extra_lines_added = val
-  end
-
-  def lineno
-    string[0...charpos].count("\n") + 1 - extra_lines_added
-  end
-
-  # TODO: once we get rid of these, we can make things like
-  # TODO: current_line and lineno much more accurate and easy to do
-
   def unread_many str # TODO: remove this entirely - we should not need it
     warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
-    self.extra_lines_added += str.count("\n") - 1
     begin
       string[charpos, 0] = str
     rescue IndexError
@@ -111,7 +91,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = "3.3.0" unless constants.include? "VERSION" # SIGH
+  VERSION = "3.4.0" unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -148,15 +128,6 @@ module RubyParserStuff
     raise RubyParser::SyntaxError, msg
   end
 
-  def arg_add(node1, node2) # TODO: nuke
-    return s(:arglist, node2) unless node1
-
-    node1[0] = :arglist if node1[0] == :array
-    return node1 << node2 if node1[0] == :arglist
-
-    return s(:arglist, node1, node2)
-  end
-
   def arg_blk_pass node1, node2 # TODO: nuke
     node1 = s(:arglist, node1) unless [:arglist, :call_args, :array, :args].include? node1.first
     node1 << node2 if node2
@@ -391,6 +362,7 @@ module RubyParserStuff
   end
 
   def gettable(id)
+    lineno = id.lineno if id.respond_to? :lineno
     id = id.to_sym if String === id
 
     result = case id.to_s
@@ -411,7 +383,7 @@ module RubyParserStuff
                end
              end
 
-    result.line(result.line - 1) if result.line and lexer.src.bol?
+    result.line lineno if lineno
 
     raise "identifier #{id.inspect} is not valid" unless result
 
@@ -457,7 +429,7 @@ module RubyParserStuff
     list
   end
 
-  def literal_concat head, tail
+  def literal_concat head, tail # TODO: ugh. rewrite
     return tail unless head
     return head unless tail
 
@@ -476,8 +448,10 @@ module RubyParserStuff
       end
     when :dstr then
       if htype == :str then
+        lineno = head.line
         tail[1] = head[-1] + tail[1]
         head = tail
+        head.line = lineno
       else
         tail[0] = :array
         tail[1] = s(:str, tail[1])
@@ -563,7 +537,7 @@ module RubyParserStuff
   end
 
   def argl x
-    x = s(:arglist, x) if x and x[0] != :arglist
+    x = s(:arglist, x) if x and x[0] == :array
     x
   end
 
@@ -587,12 +561,13 @@ module RubyParserStuff
     # TODO: need a test with f(&b) to produce block_pass
     # TODO: need a test with f(&b) { } to produce warning
 
-    args ||= s(:arglist)
-    args[0] = :arglist if [:args, :array, :call_args].include? args.first
-    args = s(:arglist, args) unless args.first == :arglist
-
-    # HACK quick hack to make this work quickly... easy to clean up above
-    result.concat args[1..-1]
+    if args
+      if [:arglist, :args, :array, :call_args].include? args.first
+        result.concat args.sexp_body
+      else
+        result << args
+      end
+    end
 
     line = result.grep(Sexp).map(&:line).compact.min
     result.line = line if line
@@ -600,9 +575,8 @@ module RubyParserStuff
     result
   end
 
-  def new_case expr, body
+  def new_case expr, body, line
     result = s(:case, expr)
-    line = (expr || body).line
 
     while body and body.node_type == :when
       result << body
@@ -794,8 +768,14 @@ module RubyParserStuff
                   begin
                     Regexp.new(node[1], o)
                   rescue RegexpError => e
-                    warn "Ignoring: #{e.message}"
-                    Regexp.new(node[1], Regexp::ENC_NONE)
+                    warn "WA\RNING: #{e.message} for #{node[1].inspect} #{options.inspect}"
+                    begin
+                      warn "WA\RNING: trying to recover with ENC_UTF8"
+                      Regexp.new(node[1], Regexp::ENC_UTF8)
+                    rescue RegexpError => e
+                      warn "WA\RNING: trying to recover with ENC_NONE"
+                      Regexp.new(node[1], Regexp::ENC_NONE)
+                    end
                   end
                 end
     when :dstr then
@@ -919,8 +899,10 @@ module RubyParserStuff
   end
 
   def next_token
-    if self.lexer.advance then
-      return self.lexer.token, self.lexer.yacc_value
+    token = self.lexer.next_token
+
+    if token and token.first != RubyLexer::EOF then
+      return token
     else
       return [false, '$end']
     end
@@ -932,18 +914,13 @@ module RubyParserStuff
     rhs = value_expr rhs
 
     case lhs[0]
-    when :gasgn, :iasgn, :lasgn, :masgn, :cdecl, :cvdecl, :cvasgn then
-      lhs << rhs
-    when :attrasgn then
+    when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn then
       lhs << rhs
-    when :call then
-      args = lhs.pop unless Symbol === lhs.last
-      lhs.concat arg_add(args, rhs)[1..-1]
     when :const then
       lhs[0] = :cdecl
       lhs << rhs
     else
-      raise "unknown lhs #{lhs.inspect}"
+      raise "unknown lhs #{lhs.inspect} w/ #{rhs.inspect}"
     end
 
     lhs
@@ -1033,10 +1010,12 @@ module RubyParserStuff
       str = handle_encoding str
 
       self.file = file.dup
-      self.lexer.src = str
 
       @yydebug = ENV.has_key? 'DEBUG'
 
+      # HACK -- need to get tests passing more than have graceful code
+      self.lexer.ss = RPStringScanner.new str
+
       do_parse
     end
   end
@@ -1086,7 +1065,7 @@ module RubyParserStuff
 
   def s(*args)
     result = Sexp.new(*args)
-    result.line ||= lexer.lineno if lexer.src          # otherwise...
+    result.line ||= lexer.lineno if lexer.ss          # otherwise...
     result.file = self.file
     result
   end
@@ -1369,6 +1348,16 @@ unless "".respond_to?(:grep) then
   end
 end
 
+class String
+  ##
+  # This is a hack used by the lexer to sneak in line numbers at the
+  # identifier level. This should be MUCH smaller than making
+  # process_token return [value, lineno] and modifying EVERYTHING that
+  # reduces tIDENTIFIER.
+
+  attr_accessor :lineno
+end
+
 class Sexp
   attr_writer :paren
 
@@ -1386,12 +1375,10 @@ class Sexp
     self.value.to_sym
   end
 
-  def add x
-    concat x
-  end
+  alias :add :<<
 
   def add_all x
-    raise "no: #{self.inspect}.add_all #{x.inspect}" # TODO: need a test to trigger this
+    self.concat x.sexp_body
   end
 
   def block_pass?

data/test/test_ruby_lexer.rb

@@ -20,7 +20,7 @@ class TestRubyLexer < Minitest::Test
 
   def setup_lexer input, exp_sexp = nil
     setup_new_parser
-    lex.src = input
+    lex.ss = RPStringScanner.new(input)
     lex.lex_state = self.lex_state
   end
 
@@ -71,11 +71,11 @@ class TestRubyLexer < Minitest::Test
   end
 
   def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
-    adv = @lex.advance
+    adv = @lex.next_token
 
     assert adv, "no more tokens"
 
-    act_token, act_value = @lex.token, @lex.yacc_value
+    act_token, act_value = adv
 
     msg = message {
       act = [act_token, act_value, @lex.lex_state,
@@ -98,20 +98,20 @@ class TestRubyLexer < Minitest::Test
   end
 
   def assert_read_escape expected, input
-    @lex.src = input
+    @lex.ss.string = input
     assert_equal expected, @lex.read_escape, input
   end
 
   def assert_read_escape_bad input # TODO: rename refute_read_escape
-    @lex.src = input
+    @lex.ss.string = input
     assert_raises RubyParser::SyntaxError do
       @lex.read_escape
     end
   end
 
   def refute_lexeme
-    x = @lex.advance
-    y = [@lex.token, @lex.yacc_value]
+    x = y = @lex.next_token
+
     refute x, "not empty: #{y.inspect}"
   end
 
@@ -146,10 +146,10 @@ class TestRubyLexer < Minitest::Test
 
   ## Tests:
 
-  def test_advance
-    assert @lex.advance # blah
-    assert @lex.advance # blah
-    refute @lex.advance # nada
+  def test_next_token
+    assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
+    assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
+    assert_nil @lex.next_token
   end
 
   def test_unicode_ident
@@ -834,6 +834,14 @@ class TestRubyLexer < Minitest::Test
                 :kEND,      "end", :expr_end)
   end
 
+  def test_yylex_is_your_spacebar_broken?
+    assert_lex3(":a!=:b",
+                nil,
+                :tSYMBOL, "a",  :expr_end,
+                :tNEQ,    "!=", :expr_beg,
+                :tSYMBOL, "b",  :expr_end)
+  end
+
   def test_yylex_do_cond
     assert_lex3("x do 42 end",
                 nil,
@@ -2538,8 +2546,9 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_underscore_end
-    @lex.src = "__END__\n"
-    refute_lexeme
+    assert_lex3("__END__\n",
+                nil,
+                RubyLexer::EOF, RubyLexer::EOF, nil)
   end
 
   def test_yylex_uplus

data/test/test_ruby_parser.rb

@@ -603,14 +603,8 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
-  STARTING_LINE = {
-    "case_no_expr"                       => 2, # TODO this should be 1
-    "structure_unused_literal_wwtt"      => 3, # yes, 3... odd test
-  }
-
   def after_process_hook klass, node, data, input_name, output_name
-    expected = STARTING_LINE[node] || 1
-    assert_equal expected, @result.line, "should have proper line number"
+    assert_equal 1, @result.line, "should have proper line number"
   end
 
   def test_parse_line_block
@@ -639,13 +633,10 @@ module TestRubyParserShared
            s(:call, nil, :b).line(2),
            s(:call, nil, :c).line(3))
 
-    # see skipped test: test_parse_line_block_inline_comment2
     assert_parse rb, pt
   end
 
-  def test_parse_line_block_inline_comment2
-    skip "I can't fix this yet. Very involved. (process_token to return lineno)"
-
+  def test_parse_line_block_inline_multiline_comment
     rb = "a\nb # comment\n# another comment\nc"
     pt = s(:block,
            s(:call, nil, :a).line(1),
@@ -655,14 +646,12 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
-  def test_parse_line_block_inline_comment3
-    skip "not yet"
-
+  def test_parse_line_block_inline_comment_leading_newlines
     rb = "\n\n\na\nb # comment\n# another comment\nc"
     pt = s(:block,
            s(:call, nil, :a).line(4),
-           s(:call, nil, :b).line(6),
-           s(:call, nil, :c).line(8)).line(1)
+           s(:call, nil, :b).line(5),
+           s(:call, nil, :c).line(7)).line(4)
 
     assert_parse rb, pt
   end
@@ -681,6 +670,20 @@ module TestRubyParserShared
     assert_equal 2, result[3].line,   "call should have line number"
   end
 
+  def test_parse_line_call_ivar_line_break_paren
+    rb = "a(@b\n)"
+    pt = s(:call, nil, :a, s(:ivar, :@b).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_parse_line_call_ivar_arg_no_parens_line_break
+    rb = "a @b\n"
+    pt = s(:call, nil, :a, s(:ivar, :@b).line(1)).line(1)
+
+    assert_parse rb, pt
+  end
+
   def test_parse_line_defn_no_parens
     pt = s(:defn, :f, s(:args), s(:nil))
 
@@ -708,6 +711,15 @@ module TestRubyParserShared
     assert_equal 4, body.return.line, "return should have line number"
   end
 
+  def test_parse_line_evstr_after_break
+    rb = "\"a\"\\\n\"\#{b}\""
+    pt = s(:dstr, "a",
+           s(:evstr,
+             s(:call, nil, :b).line(2)).line(2)).line(1)
+
+    assert_parse rb, pt
+  end
+
   def test_parse_line_iter_call_parens
     rb = "f(a) do |x, y|\n  x + y\nend"
 
@@ -727,15 +739,12 @@ module TestRubyParserShared
     rb = "f a do |x, y|\n  x + y\nend"
 
     pt = s(:iter,
-           s(:call, nil, :f, s(:call, nil, :a)),
-           s(:args, :x, :y),
-           s(:call, s(:lvar, :x), :+, s(:lvar, :y)))
+           s(:call, nil, :f, s(:call, nil, :a).line(1)).line(1),
+           s(:args, :x, :y).line(1),
+           s(:call, s(:lvar, :x).line(2), :+,
+             s(:lvar, :y).line(2)).line(2)).line(1)
 
-    assert_parse_line rb, pt, 1
-
-    assert_equal 1, result[1].line,   "call should have line number"
-    assert_equal 1, result[2].line,   "masgn should have line number"
-    assert_equal 2, result[3].line,   "call should have line number"
+    assert_parse rb, pt
   end
 
   def test_parse_line_heredoc
@@ -797,6 +806,15 @@ module TestRubyParserShared
     assert_equal 3, result.if.return.lit.line
   end
 
+  def test_parse_line_trailing_newlines
+    rb = "a \nb"
+    pt = s(:block,
+           s(:call, nil, :a).line(1),
+           s(:call, nil, :b).line(2)).line(1)
+
+    assert_parse rb, pt
+  end
+
   def test_bug_and
     rb = "true and []"
     pt = s(:and, s(:true), s(:array))
@@ -1044,7 +1062,7 @@ module TestRubyParserShared
                s(:call, nil, :p, s(:str, "a").line(2)).line(2),
                s(:lasgn, :b, s(:lit, 1).line(3)).line(3),
                s(:call, nil, :p, s(:lvar, :b).line(4)).line(4),
-               s(:lasgn, :c, s(:lit, 1).line(5)).line(5)).line(2), # TODO line 2?
+               s(:lasgn, :c, s(:lit, 1).line(5)).line(5)).line(2),
              nil).line(1),
            s(:call, nil, :a).line(7)).line(1)
 
@@ -2051,6 +2069,16 @@ class TestRuby18Parser < RubyParserTestCase
     self.processor = Ruby18Parser.new
   end
 
+  def test_call_space_before_paren_args
+    rb = "a (:b, :c, :d => :e)"
+    pt = s(:call, nil, :a,
+           s(:lit, :b),
+           s(:lit, :c),
+           s(:hash, s(:lit, :d), s(:lit, :e)))
+
+    assert_parse rb, pt
+  end
+
   def test_flip2_env_lvar
     rb = "if a..b then end"
     pt = s(:if, s(:flip2, s(:call, nil, :a), s(:call, nil, :b)), nil, nil)
@@ -2848,6 +2876,38 @@ class TestRuby20Parser < RubyParserTestCase
     self.processor = Ruby20Parser.new
   end
 
+  def test_block_call_dot_op2_brace_block
+    rb = "a.b c() do d end.e do |f| g end"
+    pt = s(:iter,
+           s(:call,
+             s(:iter,
+               s(:call, s(:call, nil, :a), :b, s(:call, nil, :c)),
+               s(:args),
+               s(:call, nil, :d)),
+             :e),
+           s(:args, :f),
+           s(:call, nil, :g))
+
+
+    assert_parse rb, pt
+  end
+
+  def test_block_call_dot_op2_cmd_args_do_block
+    rb = "a.b c() do d end.e f do |g| h end"
+    pt = s(:iter,
+           s(:call,
+             s(:iter,
+               s(:call, s(:call, nil, :a), :b, s(:call, nil, :c)),
+               s(:args),
+               s(:call, nil, :d)),
+             :e,
+             s(:call, nil, :f)),
+           s(:args, :g),
+           s(:call, nil, :h))
+
+    assert_parse rb, pt
+  end
+
   def test_defn_kwarg_val
     rb = "def f(a, b:1) end"
     pt = s(:defn, :f, s(:args, :a, s(:kwarg, :b, s(:lit, 1))), s(:nil))