yard 0.7.5 → 0.8.0

data/lib/yard/parser/ruby/legacy/ruby_parser.rb

@@ -11,6 +11,7 @@ module YARD
 
           def parse
             @parse ||= StatementList.new(@source)
+            self
           end
 
           def tokenize

data/lib/yard/parser/ruby/legacy/statement.rb

@@ -2,7 +2,12 @@ module YARD
   module Parser::Ruby::Legacy
     class Statement
       attr_reader :tokens, :comments, :block
-      attr_accessor :comments_range, :group
+      attr_accessor :comments_range
+
+      # @deprecated Groups are now defined by directives
+      # @see Tags::GroupDirective
+      attr_accessor :group
+
       attr_accessor :comments_hash_flag
 
       def initialize(tokens, block = nil, comments = nil)
@@ -31,7 +36,7 @@ module YARD
       end
 
       def show
-        "\t #{line}: #{first_line}"
+        "\t#{line}: #{first_line}"
       end
 
       # @return [Fixnum] the first line of Ruby source

data/lib/yard/parser/ruby/legacy/statement_list.rb

@@ -14,7 +14,6 @@ module YARD
       #
       # @param [TokenList, String] content the tokens to create the list from
       def initialize(content)
-        @group = nil
         @shebang_line = nil
         @encoding_line = nil
         if content.is_a? TokenList
@@ -69,12 +68,15 @@ module YARD
           sanitize_block
           @statement.pop if [TkNL, TkSPACE, TkSEMICOLON].include?(@statement.last.class)
           stmt = Statement.new(@statement, @block, @comments)
-          stmt.group = @group
           if @comments && @comments_line
             stmt.comments_range = (@comments_line..(@comments_line + @comments.size - 1))
             stmt.comments_hash_flag = @comments_hash_flag
           end
           stmt
+        elsif @comments
+          @statement << TkCOMMENT.new(@comments_line, 0)
+          @statement.first.set_text("# " + @comments.join("\n# "))
+          Statement.new(@statement, nil, @comments)
         else
           nil
         end
@@ -111,23 +113,6 @@ module YARD
         end
       end
 
-      def preprocess_token(tk)
-        if tk.is_a?(TkCOMMENT)
-          case tk.text
-          when /\A# @group\s+(.+)\s*\Z/
-            @group = $1
-            true
-          when /\A# @endgroup\s*\Z/
-            @group = nil
-            true
-          else
-            false
-          end
-        else
-          false
-        end
-      end
-
       # Processes a single token
       #
       # @param [RubyToken::Token] tk the token to process
@@ -135,7 +120,6 @@ module YARD
         # p tk.class, tk.text, @state, @level, @current_block, "<br/>"
         case @state
         when :first_statement
-          return if preprocess_token(tk)
           return if process_initial_comment(tk)
           return if @statement.empty? && [TkSPACE, TkNL, TkCOMMENT].include?(tk.class)
           @comments_last_line = nil
@@ -235,10 +219,17 @@ module YARD
             return
           end
         end
+
         return if !@statement.empty? && @comments
         return if @first_line && tk.line_no > @first_line
 
-        @comments = nil if @comments_last_line && @comments_last_line < tk.line_no - 1
+        if @comments_last_line && @comments_last_line < tk.line_no - 1
+          if @comments && @statement.empty?
+            @tokens.unshift(tk)
+            return @done = true
+          end
+          @comments = nil
+        end
         @comments_line = tk.line_no unless @comments
 
         # Remove the "#" and up to 1 space before the text
@@ -356,7 +347,7 @@ module YARD
       # @return [Boolean] whether or not the current statement's parentheses and blocks
       #   are balanced after +tk+
       def balances?(tk)
-        unless [TkALIAS, TkDEF].include?(@last_ns_tk.class) || @before_last_ns_tk.class == TkALIAS 
+        unless [TkALIAS, TkDEF].include?(@last_ns_tk.class) || @before_last_ns_tk.class == TkALIAS
           if [TkLPAREN, TkLBRACK, TkLBRACE, TkDO, TkBEGIN].include?(tk.class)
             @level += 1
           elsif OPEN_BLOCK_TOKENS.include?(tk.class)

data/lib/yard/parser/ruby/ruby_parser.rb

@@ -4,9 +4,9 @@ module YARD
   module Parser
     module Ruby
       # Ruby 1.9 parser
-      # @attr_reader encoding_line
-      # @attr_reader shebang_line
-      # @attr_reader enumerator
+      # @!attribute [r] encoding_line
+      # @!attribute [r] shebang_line
+      # @!attribute [r] enumerator
       class RubyParser < Parser::Base
         def initialize(source, filename)
           @parser = RipperParser.new(source, filename)
@@ -33,6 +33,7 @@ module YARD
           @source = source
           @tokens = []
           @comments = {}
+          @comments_range = {}
           @comments_flags = {}
           @heredoc_tokens = nil
           @heredoc_state = nil
@@ -40,9 +41,9 @@ module YARD
           @ns_charno = 0
           @list = []
           @charno = 0
-          @groups = []
           @shebang_line = nil
           @encoding_line = nil
+          @file_encoding = nil
         end
 
         def parse
@@ -58,6 +59,15 @@ module YARD
           ast.children
         end
 
+        def file_encoding
+          return nil unless defined?(::Encoding)
+          return @file_encoding if @file_encoding
+          return Encoding.default_internal unless @encoding_line
+          if match = @encoding_line.match(SourceParser::ENCODING_LINE)
+            @file_encoding = match.captures.last
+          end
+        end
+
         private
 
         MAPPINGS = {
@@ -234,6 +244,11 @@ module YARD
             @tokens[-1] = [:symbol, ":" + data]
           elsif @heredoc_state == :started
             @heredoc_tokens << [token, data]
+
+            # fix ripper encoding of heredoc bug
+            # (see http://bugs.ruby-lang.org/issues/6200)
+            data.force_encoding(file_encoding) if file_encoding
+
             @heredoc_state = :ended if token == :heredoc_end
           elsif (token == :nl || token == :comment) && @heredoc_state == :ended
             @heredoc_tokens.unshift([token, data])
@@ -244,7 +259,7 @@ module YARD
             @tokens << [token, data]
             if token == :heredoc_beg
               @heredoc_state = :started
-              @heredoc_tokens = [] 
+              @heredoc_tokens = []
             end
           end
         end
@@ -274,6 +289,9 @@ module YARD
         undef on_embdoc
         undef on_embdoc_end
         undef on_parse_error
+        undef on_bodystmt
+        undef on_top_const_ref
+        undef on_const_path_ref
 
         def on_program(*args)
           args.first
@@ -299,7 +317,7 @@ module YARD
         def on_assoclist_from_args(*args)
           args.first
         end
-        
+
         def on_unary(op, val)
           map = @map[op.to_s[0,1]]
           lstart, sstart = *(map ? map.pop : [lineno, @ns_charno - 1])
@@ -340,7 +358,7 @@ module YARD
           end
           node
         end
-        
+
         def on_lbracket(tok)
           (@map[:lbracket] ||= []) << [lineno, charno]
           visit_ns_token(:lbracket, tok, false)
@@ -378,7 +396,7 @@ module YARD
             end
           eof
         end
-        
+
         def on_qwords_new(*args)
           node = LiteralNode.new(:qwords_literal, args)
           if @map[:qwords_beg]
@@ -388,7 +406,7 @@ module YARD
           end
           node
         end
-        
+
         def on_qwords_add(list, item)
           last = @source[@ns_charno,1] == "\n" ? @ns_charno - 1 : @ns_charno
           list.source_range = (list.source_range.first..last)
@@ -467,32 +485,30 @@ module YARD
             end
           end
 
+          ch = charno
           visit_ns_token(:comment, comment)
           if not_comment
             @last_ns_token = nil
             return
           end
-          case comment
-          when /\A#+ @group\s+(.+)\s*\Z/
-            @groups.unshift [lineno, $1]
-            return
-          when /\A#+ @endgroup\s*\Z/
-            @groups.unshift [lineno, nil]
-            return
-          end
 
+          source_range = ch..(charno-1)
           comment = comment.gsub(/^(\#+)\s{0,1}/, '').chomp
           append_comment = @comments[lineno - 1]
+
           hash_flag = $1 == '##' ? true : false
 
           if append_comment && @comments_last_column == column
             @comments.delete(lineno - 1)
             @comments_flags[lineno] = @comments_flags[lineno - 1]
             @comments_flags.delete(lineno - 1)
+            range = @comments_range.delete(lineno - 1)
+            source_range = range.first..source_range.last
             comment = append_comment + "\n" + comment
           end
 
           @comments[lineno] = comment
+          @comments_range[lineno] = source_range
           @comments_flags[lineno] = hash_flag if !append_comment
           @comments_last_column = column
         end
@@ -523,22 +539,67 @@ module YARD
             (node.line - 2).upto(node.line) do |line|
               comment = @comments[line]
               if comment && !comment.empty?
-                node.docstring_hash_flag = @comments_flags[line]
-                node.docstring = comment
-                node.docstring_range = ((line - comment.count("\n"))..line)
-                @comments.delete(line)
-                @comments_flags.delete(line)
+                add_comment(line, node)
                 break
               end
             end
-            if node.type == :def || node.type == :defs || node.call?
-              @groups.each do |group|
-                if group.first < node.line
-                  break node.group = group.last
+
+            @comments.keys.each do |line|
+              if node.line > line
+                add_comment(line, nil, node)
+              end
+            end
+          end
+
+          # insert any lone unadded comments before node
+          root.traverse do |node|
+            next if node.type == :list || node.parent.type != :list
+            @comments.keys.each do |line|
+              if node.line_range.include?(line)
+                pick = nil
+                node.traverse do |subnode|
+                  next unless subnode.type == :list
+                  pick ||= subnode
+                  next unless subnode.line_range.include?(line)
+                  pick = subnode
                 end
+                add_comment(line, nil, pick, true) if pick
               end
             end
+          end if @comments.size > 0
+
+          # insert all remaining comments
+          @comments.each do |line, comment|
+            add_comment(line, nil, root, true)
+          end
+
+          @comments = {}
+        end
+
+        def add_comment(line, node = nil, before_node = nil, into = false)
+          comment = @comments[line]
+          source_range = @comments_range[line]
+          line_range = ((line - comment.count("\n"))..line)
+          if node.nil?
+            node = CommentNode.new(:comment, [comment], :line => line_range, :char => source_range)
+            if into
+              before_node.push(node)
+              before_node.unfreeze
+              node.parent = before_node
+            elsif before_node
+              parent_node = before_node.parent
+              idx = parent_node.index(before_node)
+              parent_node.insert(idx, node)
+              parent_node.unfreeze
+              node.parent = parent_node
+            end
           end
+          node.docstring = comment
+          node.docstring_hash_flag = @comments_flags[line]
+          node.docstring_range = line_range
+          @comments.delete(line)
+          @comments_range.delete(line)
+          @comments_flags.delete(line)
         end
 
         def freeze_tree(node = nil)

data/lib/yard/parser/source_parser.rb

@@ -1,8 +1,6 @@
 require 'stringio'
 require 'ostruct'
 
-begin require 'continuation'; rescue LoadError; end
-
 module YARD
   module Parser
     # Raised when an object is recognized but cannot be documented. This
@@ -13,19 +11,38 @@ module YARD
     # Raised when the parser sees a Ruby syntax error
     class ParserSyntaxError < UndocumentableError; end
 
-    # A LoadOrderError occurs when a handler needs to modify a
-    # {CodeObjects::NamespaceObject} (usually by adding a child to it)
-    # that has not yet been resolved.
+    # Responsible for parsing a list of files in order. The
+    # {#parse} method of this class can be called from the
+    # {SourceParser#globals} globals state list to re-enter
+    # parsing for the remainder of files in the list recursively.
     #
-    # @see Handers::Base#ensure_loaded!
-    class LoadOrderError < Exception
-      # @return [Continuation] the context representing the
-      #   point at which the load order error occurred.
-      attr_accessor :context
-      
-      # @param [Continuation] context see {#context}
-      def initialize(context)
-        @context = context
+    # @see Processor#parse_remaining_files
+    class OrderedParser
+      # @return [Array<String>] the list of remaining files to parse
+      attr_accessor :files
+
+      # Creates a new OrderedParser with the global state and a list
+      # of files to parse.
+      #
+      # @note OrderedParser sets itself as the +ordered_parser+ key on
+      #   global_state for later use in {Handlers::Processor}.
+      # @param [OpenStruct] global_state a structure containing all global
+      #   state during parsing
+      # @param [Array<String>] files the list of files to parse
+      def initialize(global_state, files)
+        @global_state = global_state
+        @files = files.dup
+        @global_state.ordered_parser = self
+      end
+
+      # Parses the remainder of the {#files} list.
+      #
+      # @see Processor#parse_remaining_files
+      def parse
+        while file = files.shift
+          log.debug("Processing #{file}...")
+          SourceParser.new(SourceParser.parser_type, @global_state).parse(file)
+        end
       end
     end
 
@@ -44,7 +61,7 @@ module YARD
     class SourceParser
       SHEBANG_LINE  = /\A\s*#!\S+/
       ENCODING_LINE = /\A(?:\s*#*!.*\r?\n)?\s*(?:#+|\/\*+|\/\/+).*coding\s*[:=]{1,2}\s*([a-z\d_\-]+)/i
-      
+
       # Byte order marks for various encodings
       # @since 0.7.0
       ENCODING_BYTE_ORDER_MARKS = {
@@ -163,20 +180,20 @@ module YARD
         def validated_parser_type(type)
           !defined?(::Ripper) && type == :ruby ? :ruby18 : type
         end
-        
+
         # @group Parser Callbacks
-        
-        # Registers a callback to be called before a list of files is parsed 
-        # via {parse}. The block passed to this method will be called on 
+
+        # Registers a callback to be called before a list of files is parsed
+        # via {parse}. The block passed to this method will be called on
         # subsequent parse calls.
-        # 
+        #
         # @example Installing a simple callback
         #   SourceParser.before_parse_list do |files, globals|
         #     puts "Starting to parse..."
         #   end
         #   YARD.parse('lib/**/*.rb')
         #   # prints "Starting to parse..."
-        # 
+        #
         # @example Setting global state
         #   SourceParser.before_parse_list do |files, globals|
         #     globals.method_count = 0
@@ -190,16 +207,16 @@ module YARD
         #   end
         #   YARD.parse
         #   # Prints: "Found 37 methods"
-        # 
+        #
         # @example Using a global callback to cancel parsing
         #   SourceParser.before_parse_list do |files, globals|
         #     return false if files.include?('foo.rb')
         #   end
-        # 
+        #
         #   YARD.parse(['foo.rb', 'bar.rb']) # callback cancels this method
         #   YARD.parse('bar.rb') # parses normally
-        # 
-        # @yield [files, globals] the yielded block is called once before 
+        #
+        # @yield [files, globals] the yielded block is called once before
         #   parsing all files
         # @yieldparam [Array<String>] files the list of files that will be parsed.
         # @yieldparam [OpenStruct] globals a global structure to store arbitrary
@@ -213,18 +230,18 @@ module YARD
         def before_parse_list(&block)
           before_parse_list_callbacks << block
         end
-        
-        # Registers a callback to be called after a list of files is parsed 
-        # via {parse}. The block passed to this method will be called on 
+
+        # Registers a callback to be called after a list of files is parsed
+        # via {parse}. The block passed to this method will be called on
         # subsequent parse calls.
-        # 
+        #
         # @example Printing results after parsing occurs
         #   SourceParser.after_parse_list do
         #     puts "Finished parsing!"
         #   end
         #   YARD.parse
         #   # Prints "Finished parsing!" after parsing files
-        # @yield [files, globals] the yielded block is called once before 
+        # @yield [files, globals] the yielded block is called once before
         #   parsing all files
         # @yieldparam [Array<String>] files the list of files that will be parsed.
         # @yieldparam [OpenStruct] globals a global structure to store arbitrary
@@ -237,14 +254,14 @@ module YARD
         def after_parse_list(&block)
           after_parse_list_callbacks << block
         end
-        
-        # Registers a callback to be called before an individual file is parsed. 
-        # The block passed to this method will be called on subsequent parse 
+
+        # Registers a callback to be called before an individual file is parsed.
+        # The block passed to this method will be called on subsequent parse
         # calls.
-        # 
+        #
         # To register a callback that is called before the entire list of files
         # is processed, see {before_parse_list}.
-        # 
+        #
         # @example Installing a simple callback
         #   SourceParser.before_parse_file do |parser|
         #     puts "I'm parsing #{parser.file}"
@@ -254,18 +271,18 @@ module YARD
         #   "I'm parsing lib/foo.rb"
         #   "I'm parsing lib/foo_bar.rb"
         #   "I'm parsing lib/last_file.rb"
-        # 
+        #
         # @example Cancel parsing of any test_*.rb files
         #   SourceParser.before_parse_file do |parser|
         #     return false if parser.file =~ /^test_.+\.rb$/
         #   end
-        # 
+        #
         # @yield [parser] the yielded block is called once before each
         #   file that is parsed. This might happen many times for a single
         #   codebase.
         # @yieldparam [SourceParser] parser the parser object that will {#parse}
         #   the file.
-        # @yieldreturn [Boolean] if the block returns +false+, parsing for 
+        # @yieldreturn [Boolean] if the block returns +false+, parsing for
         #   the file is cancelled.
         # @return [Proc] the yielded block
         # @see after_parse_file
@@ -274,14 +291,14 @@ module YARD
         def before_parse_file(&block)
           before_parse_file_callbacks << block
         end
-        
-        # Registers a callback to be called after an individual file is parsed. 
-        # The block passed to this method will be called on subsequent parse 
+
+        # Registers a callback to be called after an individual file is parsed.
+        # The block passed to this method will be called on subsequent parse
         # calls.
-        # 
+        #
         # To register a callback that is called after the entire list of files
         # is processed, see {after_parse_list}.
-        # 
+        #
         # @example Printing the length of each file after it is parsed
         #   SourceParser.after_parse_file do |parser|
         #     puts "#{parser.file} is #{parser.contents.size} characters"
@@ -290,8 +307,8 @@ module YARD
         #   # prints:
         #   "lib/foo.rb is 1240 characters"
         #   "lib/foo_bar.rb is 248 characters"
-        # 
-        # @yield [parser] the yielded block is called once after each file 
+        #
+        # @yield [parser] the yielded block is called once after each file
         #   that is parsed. This might happen many times for a single codebase.
         # @yieldparam [SourceParser] parser the parser object that parsed
         #   the file.
@@ -303,104 +320,88 @@ module YARD
         def after_parse_file(&block)
           after_parse_file_callbacks << block
         end
-        
+
         # @return [Array<Proc>] the list of callbacks to be called before
         #   parsing a list of files. Should only be used for testing.
         # @since 0.7.0
         def before_parse_list_callbacks
           @before_parse_list_callbacks ||= []
         end
-        
+
         # @return [Array<Proc>] the list of callbacks to be called after
         #   parsing a list of files. Should only be used for testing.
         # @since 0.7.0
         def after_parse_list_callbacks
           @after_parse_list_callbacks ||= []
         end
-        
+
         # @return [Array<Proc>] the list of callbacks to be called before
         #   parsing a file. Should only be used for testing.
         # @since 0.7.0
         def before_parse_file_callbacks
           @before_parse_file_callbacks ||= []
         end
-        
+
         # @return [Array<Proc>] the list of callbacks to be called after
         #   parsing a file. Should only be used for testing.
         # @since 0.7.0
         def after_parse_file_callbacks
           @after_parse_file_callbacks ||= []
         end
-        
+
         # @endgroup
 
         private
 
-        # Parses a list of files in a queue. If a {LoadOrderError} is caught,
-        # the file is moved to the back of the queue with a Continuation object
-        # that can continue processing the file.
+        # Parses a list of files in a queue.
         #
         # @param [Array<String>] files a list of files to queue for parsing
         # @return [void]
         def parse_in_order(*files)
           global_state = OpenStruct.new
           files = files.sort_by {|x| x.length if x }
-          files_copy = files.dup
-          
+
           before_parse_list_callbacks.each do |cb|
-            return if cb.call(files_copy, global_state) == false
+            return if cb.call(files, global_state) == false
           end
-          
-          while file = files.shift
-            begin
-              if file.is_a?(Array) && file.last.is_a?(Continuation)
-                log.debug("Re-processing #{file.first}")
-                file.last.call
-              elsif file.is_a?(String)
-                log.debug("Processing #{file}...")
-                new(parser_type, true, global_state).parse(file)
-              end
-            rescue LoadOrderError => e
-              # Out of order file. Push the context to the end and we'll call it
-              files.push([file, e.context])
-            end
-          end
-          
+
+          OrderedParser.new(global_state, files).parse
+
           after_parse_list_callbacks.each do |cb|
-            cb.call(files_copy, global_state)
+            cb.call(files, global_state)
           end
         end
       end
 
       register_parser_type :ruby,   Ruby::RubyParser
       register_parser_type :ruby18, Ruby::Legacy::RubyParser
-      register_parser_type :c,      CParser, ['c', 'cc', 'cxx', 'cpp']
+      register_parser_type :c,      C::CParser, ['c', 'cc', 'cxx', 'cpp']
 
       self.parser_type = :ruby
 
       # @return [String] the filename being parsed by the parser.
-      attr_reader :file
+      attr_accessor :file
 
       # @return [Symbol] the parser type associated with the parser instance.
       #   This should be set by the {#initialize constructor}.
       attr_reader :parser_type
-      
+
       # @return [OpenStruct] an open struct containing arbitrary global state
       #   shared between files and handlers.
       # @since 0.7.0
       attr_reader :globals
-      
+
       # @return [String] the contents of the file to be parsed
       # @since 0.7.0
       attr_reader :contents
 
-      # Creates a new parser object for code parsing with a specific parser type.
+      # @overload initialize(parser_type = SourceParser.parser_type, globals = nil)
+      #   Creates a new parser object for code parsing with a specific parser type.
       #
-      # @param [Symbol] parser_type the parser type to use
-      # @param [Boolean] load_order_errors whether or not to raise the {LoadOrderError}
-      # @param [OpenStruct] globals global state to be re-used across separate source files
-      def initialize(parser_type = SourceParser.parser_type, load_order_errors = false, globals = nil)
-        @load_order_errors = load_order_errors
+      #   @param [Symbol] parser_type the parser type to use
+      #   @param [OpenStruct] globals global state to be re-used across separate source files
+      def initialize(parser_type = SourceParser.parser_type, globals1 = nil, globals2 = nil)
+        globals = [true, false].include?(globals1) ? globals2 : globals1
         @file = '(stdin)'
         @globals = globals || OpenStruct.new
         self.parser_type = parser_type
@@ -430,18 +431,18 @@ module YARD
 
         @contents = content
         @parser = parser_class.new(content, file)
-        
+
         self.class.before_parse_file_callbacks.each do |cb|
           return @parser if cb.call(self) == false
         end
-          
+
         @parser.parse
         post_process
-        
+
         self.class.after_parse_file_callbacks.each do |cb|
           cb.call(self)
         end
-        
+
         @parser
       rescue ArgumentError, NotImplementedError => e
         log.warn("Cannot parse `#{file}': #{e.message}")
@@ -488,7 +489,7 @@ module YARD
       def post_process
         return unless @parser.respond_to? :enumerator
         return unless enumerator = @parser.enumerator
-        post = Handlers::Processor.new(@file, @load_order_errors, @parser_type, @globals)
+        post = Handlers::Processor.new(self)
         post.process(enumerator)
       end