rdoc 6.0.4 → 6.1.0.beta1

data/lib/rdoc/parser/ruby.rb

@@ -8,8 +8,6 @@
 #       by Keiju ISHITSUKA (Nippon Rational Inc.)
 #
 
-$TOKEN_DEBUG ||= nil
-
 ##
 # Extracts code elements from a source file returning a TopLevel object
 # containing the constituent file elements.
@@ -141,6 +139,7 @@ $TOKEN_DEBUG ||= nil
 # standard rdocable item following it.
 
 require 'ripper'
+require_relative 'ripper_state_lex'
 
 class RDoc::Parser::Ruby < RDoc::Parser
 
@@ -178,7 +177,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     @size = 0
     @token_listeners = nil
     content = RDoc::Encoding.remove_magic_comment content
-    @scanner = RDoc::RipperStateLex.parse(content)
+    @scanner = RDoc::Parser::RipperStateLex.parse(content)
     @content = content
     @scanner_point = 0
     @prev_seek = nil
@@ -249,10 +248,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
     tk = get_tk
 
     while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
-      if first_line and tk[:text] =~ /\A#!/ then
+      comment_body = retrieve_comment_body(tk)
+      if first_line and comment_body =~ /\A#!/ then
         skip_tkspace
         tk = get_tk
-      elsif first_line and tk[:text] =~ /\A#\s*-\*-/ then
+      elsif first_line and comment_body =~ /\A#\s*-\*-/ then
         first_line = false
         skip_tkspace
         tk = get_tk
@@ -261,11 +261,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
         first_comment_tk_kind = tk[:kind]
 
         first_line = false
-        comment << tk[:text]
+        comment << comment_body
         tk = get_tk
 
         if :on_nl === tk then
-          skip_tkspace false
+          skip_tkspace_without_nl
           tk = get_tk
         end
       end
@@ -280,7 +280,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Consumes trailing whitespace from the token stream
 
   def consume_trailing_spaces # :nodoc:
-    skip_tkspace false
+    skip_tkspace_without_nl
   end
 
   ##
@@ -352,7 +352,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       given_name << '::'
     end
 
-    skip_tkspace false
+    skip_tkspace_without_nl
     given_name << name_t[:text]
 
     is_self = name_t[:kind] == :on_op && name_t[:text] == '<<'
@@ -376,7 +376,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       record_location container
 
       get_tk
-      skip_tkspace false
+      skip_tkspace_without_nl
       name_t = get_tk
       unless :on_const == name_t[:kind] || :on_ident == name_t[:kind]
         raise RDoc::Error, "Invalid class or module definition: #{given_name}"
@@ -388,7 +388,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
     end
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     return [container, name_t, given_name, new_modules]
   end
@@ -408,7 +408,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     res = get_constant
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     get_tkread # empty out read buffer
 
@@ -431,7 +431,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def get_constant
     res = ""
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
 
     while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do
@@ -444,28 +444,83 @@ class RDoc::Parser::Ruby < RDoc::Parser
   end
 
   ##
-  # Get a constant that may be surrounded by parens
+  # Get an included module that may be surrounded by parens
 
-  def get_constant_with_optional_parens
-    skip_tkspace false
+  def get_included_module_with_optional_parens
+    skip_tkspace_without_nl
+    get_tkread
+    tk = get_tk
+    end_token = get_end_token tk
+    return '' unless end_token
 
     nest = 0
+    continue = false
+    only_constant = true
 
-    while :on_lparen == (tk = peek_tk)[:kind] do
-      get_tk
-      skip_tkspace
-      nest += 1
-    end
-
-    name = get_constant
-
-    while nest > 0
-      skip_tkspace
+    while tk != nil do
+      is_element_of_constant = false
+      case tk[:kind]
+      when :on_semicolon then
+        break if nest == 0
+      when :on_lbracket then
+        nest += 1
+      when :on_rbracket then
+        nest -= 1
+      when :on_lbrace then
+        nest += 1
+      when :on_rbrace then
+        nest -= 1
+        if nest <= 0
+          # we might have a.each { |i| yield i }
+          unget_tk(tk) if nest < 0
+          break
+        end
+      when :on_lparen then
+        nest += 1
+      when end_token[:kind] then
+        if end_token[:kind] == :on_rparen
+          nest -= 1
+          break if nest <= 0
+        else
+          break if nest <= 0
+        end
+      when :on_rparen then
+        nest -= 1
+      when :on_comment, :on_embdoc then
+        @read.pop
+        if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
+          (!continue or (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0) then
+          break if !continue and nest <= 0
+        end
+      when :on_comma then
+        continue = true
+      when :on_ident then
+        continue = false if continue
+      when :on_kw then
+        case tk[:text]
+        when 'def', 'do', 'case', 'for', 'begin', 'class', 'module'
+          nest += 1
+        when 'if', 'unless', 'while', 'until', 'rescue'
+          # postfix if/unless/while/until/rescue must be EXPR_LABEL
+          nest += 1 unless (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0
+        when 'end'
+          nest -= 1
+          break if nest == 0
+        end
+      when :on_const then
+        is_element_of_constant = true
+      when :on_op then
+        is_element_of_constant = true if '::' == tk[:text]
+      end
+      only_constant = false unless is_element_of_constant
       tk = get_tk
-      nest -= 1 if :on_rparen == tk[:kind]
     end
 
-    name
+    if only_constant
+      get_tkread_clean(/\s+/, ' ')
+    else
+      ''
+    end
   end
 
   ##
@@ -479,17 +534,17 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def get_end_token tk # :nodoc:
     case tk[:kind]
     when :on_lparen
-      {
-        :kind => :on_rparen,
-        :text => ')'
-      }
+      token = RDoc::Parser::RipperStateLex::Token.new
+      token[:kind] = :on_rparen
+      token[:text] = ')'
+      token
     when :on_rparen
       nil
     else
-      {
-        :kind => :on_nl,
-        :text => "\n"
-      }
+      token = RDoc::Parser::RipperStateLex::Token.new
+      token[:kind] = :on_nl
+      token[:text] = "\n"
+      token
     end
   end
 
@@ -628,7 +683,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     if args.size > 0 then
       name = args[0]
       rw = "R"
-      skip_tkspace false
+      skip_tkspace_without_nl
       tk = get_tk
 
       if :on_comma == tk[:kind] then
@@ -739,9 +794,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
       when end_token
         if end_token == :on_rparen
           nest -= 1
-          break if RDoc::RipperStateLex.end?(tk) and nest <= 0
+          break if RDoc::Parser::RipperStateLex.end?(tk) and nest <= 0
         else
-          break if RDoc::RipperStateLex.end?(tk)
+          break if RDoc::Parser::RipperStateLex.end?(tk)
         end
       when :on_comment, :on_embdoc
         unget_tk(tk)
@@ -878,7 +933,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     line_no = tk[:line_no]
 
     name = tk[:text]
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     return unless name =~ /^\w+$/
 
@@ -904,7 +959,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
           break if nest == 0
         end
       end
-      skip_tkspace false
+      skip_tkspace_without_nl
       is_array_or_hash = true
     end
 
@@ -959,7 +1014,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then
         nest += 1
       elsif (:on_kw == tk[:kind] && %w{do if unless case begin}.include?(tk[:text])) then
-        if (tk[:state] & RDoc::RipperStateLex::EXPR_LABEL) == 0
+        if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0
           nest += 1
         end
       elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) ||
@@ -967,7 +1022,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
         nest -= 1
       elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
         unget_tk tk
-        if nest <= 0 and RDoc::RipperStateLex.end?(tk) then
+        if nest <= 0 and RDoc::Parser::RipperStateLex.end?(tk) then
           body = get_tkread_clean(/^[ \t]+/, '')
           read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
           break
@@ -983,7 +1038,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
           break
         end
       elsif :on_nl == tk[:kind] then
-        if nest <= 0 and RDoc::RipperStateLex.end?(tk) then
+        if nest <= 0 and RDoc::Parser::RipperStateLex.end?(tk) then
           unget_tk tk
           break
         end
@@ -1047,10 +1102,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     record_location meth
 
     meth.start_collecting_tokens
-    indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
-    position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+    indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
+    position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
     position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
-    newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+    newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
     meth.add_tokens [position_comment, newline, indent]
 
     meth.params =
@@ -1090,10 +1145,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth.line      = line_no
 
     meth.start_collecting_tokens
-    indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
-    position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+    indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
+    position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
     position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
-    newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+    newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
     meth.add_tokens [position_comment, newline, indent]
 
     meth.call_seq = signature
@@ -1117,7 +1172,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     loop do
       skip_tkspace_comment
 
-      name = get_constant_with_optional_parens
+      name = get_included_module_with_optional_parens
 
       unless name.empty? then
         obj = container.add klass, name, comment
@@ -1241,7 +1296,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     add_token tk
     add_token_listener self
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
     singleton = !!$~
@@ -1258,10 +1313,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     remove_token_listener self
 
     meth.start_collecting_tokens
-    indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
-    position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+    indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
+    position_comment = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
     position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
-    newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+    newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
     meth.add_tokens [position_comment, newline, indent]
     meth.add_tokens @token_stream
 
@@ -1361,10 +1416,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth.line   = line_no
 
     meth.start_collecting_tokens
-    indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
-    token = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+    indent = RDoc::Parser::RipperStateLex::Token.new(1, 1, :on_sp, ' ' * column)
+    token = RDoc::Parser::RipperStateLex::Token.new(line_no, 1, :on_comment)
     token[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
-    newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+    newline = RDoc::Parser::RipperStateLex::Token.new(0, 0, :on_nl, "\n")
     meth.add_tokens [token, newline, indent]
     meth.add_tokens @token_stream
 
@@ -1430,7 +1485,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def parse_method_name container # :nodoc:
     skip_tkspace
     name_t = get_tk
-    back_tk = skip_tkspace(false)
+    back_tk = skip_tkspace_without_nl
     singleton = false
 
     dot = get_tk
@@ -1518,7 +1573,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_method_or_yield_parameters(method = nil,
                                        modifiers = RDoc::METHOD_MODIFIERS)
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
     end_token = get_end_token tk
     return '' unless end_token
@@ -1530,6 +1585,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
       case tk[:kind]
       when :on_semicolon then
         break if nest == 0
+      when :on_lbracket then
+        nest += 1
+      when :on_rbracket then
+        nest -= 1
       when :on_lbrace then
         nest += 1
       when :on_rbrace then
@@ -1553,7 +1612,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       when :on_comment, :on_embdoc then
         @read.pop
         if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
-          (!continue or (tk[:state] & RDoc::RipperStateLex::EXPR_LABEL) != 0) then
+          (!continue or (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) != 0) then
           if method && method.block_params.nil? then
             unget_tk tk
             read_documentation_modifiers method, modifiers
@@ -1587,7 +1646,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     return if  method.block_params
 
-    skip_tkspace false
+    skip_tkspace_without_nl
     read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
   end
 
@@ -1638,19 +1697,30 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a rescue
 
   def parse_rescue
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     while tk = get_tk
       case tk[:kind]
       when :on_nl, :on_semicolon, :on_comment then
         break
       when :on_comma then
-        skip_tkspace false
+        skip_tkspace_without_nl
 
         get_tk if :on_nl == peek_tk[:kind]
       end
 
-      skip_tkspace false
+      skip_tkspace_without_nl
+    end
+  end
+
+  ##
+  # Retrieve comment body without =begin/=end
+
+  def retrieve_comment_body(tk)
+    if :on_embdoc == tk[:kind]
+      tk[:text].gsub(/\A=begin.*\n/, '').gsub(/=end\n?\z/, '')
+    else
+      tk[:text]
     end
   end
 
@@ -1707,11 +1777,12 @@ class RDoc::Parser::Ruby < RDoc::Parser
           end
 
           while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do
-            comment += tk[:text]
-            comment += "\n" unless "\n" == tk[:text].chars.to_a.last
+            comment_body = retrieve_comment_body(tk)
+            comment += comment_body
+            comment += "\n" unless "\n" == comment_body.chars.to_a.last
 
-            if tk[:text].size > 1 && "\n" == tk[:text].chars.to_a.last then
-              skip_tkspace false # leading spaces
+            if comment_body.size > 1 && "\n" == comment_body.chars.to_a.last then
+              skip_tkspace_without_nl # leading spaces
             end
             tk = get_tk
           end
@@ -1758,7 +1829,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
           end
 
         when 'until', 'while' then
-          if (tk[:state] & RDoc::RipperStateLex::EXPR_LABEL) == 0
+          if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0
             nest += 1
             skip_optional_do_after_expression
           end
@@ -1774,7 +1845,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
           skip_optional_do_after_expression
 
         when 'case', 'do', 'if', 'unless', 'begin' then
-          if (tk[:state] & RDoc::RipperStateLex::EXPR_LABEL) == 0
+          if (tk[:state] & RDoc::Parser::RipperStateLex::EXPR_LABEL) == 0
             nest += 1
           end
 
@@ -1895,7 +1966,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
 
     loop do
-      skip_tkspace false
+      skip_tkspace_without_nl
 
       tk1 = get_tk
       if tk1.nil? || :on_comma != tk1[:kind] then
@@ -2044,7 +2115,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # See also RDoc::Markup::PreProcess#handle_directive
 
   def read_documentation_modifiers context, allowed
-    skip_tkspace(false)
+    skip_tkspace_without_nl
     directive, value = read_directive allowed
 
     return unless directive
@@ -2122,7 +2193,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # while, until, and for have an optional do
 
   def skip_optional_do_after_expression
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
 
     b_nest = 0
@@ -2154,7 +2225,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       tk = get_tk
     end
 
-    skip_tkspace false
+    skip_tkspace_without_nl
 
     get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text]
   end
@@ -2163,9 +2234,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # skip the var [in] part of a 'for' statement
 
   def skip_for_variable
-    skip_tkspace false
+    skip_tkspace_without_nl
     get_tk
-    skip_tkspace false
+    skip_tkspace_without_nl
     tk = get_tk
     unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text]
   end
@@ -2184,7 +2255,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def skip_tkspace_comment(skip_nl = true)
     loop do
-      skip_tkspace skip_nl
+      skip_nl ? skip_tkspace : skip_tkspace_without_nl
       next_tk = peek_tk
       return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind])
       get_tk