yard 0.2.3.5 → 0.4.0

data/lib/yard/parser/ruby/legacy/ruby_lex.rb

@@ -253,8 +253,8 @@ module YARD
         [:TkDOLLAR,     TkUnknownChar,  "\$"], #"
       ]
 
-      # {reading => token_class}
-      # {reading => [token_class, *opt]}
+      # { reading => token_class }
+      # { reading => [token_class, *opt] }
       TkReading2Token = {}
       TkSymbol2Token = {}
 

data/lib/yard/parser/ruby/legacy/statement_list.rb

@@ -16,7 +16,7 @@ module YARD
         if content.is_a? TokenList
           @tokens = content.dup
         elsif content.is_a? String
-          @tokens = TokenList.new(content)
+          @tokens = TokenList.new(content.gsub("\r", ""))
         else 
           raise ArgumentError, "Invalid content for StatementList: #{content.inspect}:#{content.class}"
         end
@@ -110,11 +110,12 @@ module YARD
       #
       # @param [RubyToken::Token] tk the token to process
       def process_token(tk)
-        #p tk.text, @state, @level, @current_block, "<br/>"
+        # p tk.text, @state, @level, @current_block, "<br/>"
         case @state
         when :first_statement
           return if process_initial_comment(tk)
           return if @statement.empty? && [TkSPACE, TkNL, TkCOMMENT].include?(tk.class)
+          @comments_last_line = nil
           return if process_simple_block_opener(tk)
           push_token(tk)
           return if process_complex_block_opener(tk)
@@ -175,15 +176,14 @@ module YARD
       # @param [RubyToken::Token] tk the token to process
       # @return [Boolean] whether or not +tk+ was processed as an initial comment
       def process_initial_comment(tk)
-        return unless tk.class == TkCOMMENT
-
-        # Two new-lines in a row will destroy any comment blocks
-        if @last_tk.class == TkNL && @before_last_tk &&
-            (@before_last_tk.class == TkNL || @before_last_tk.class == TkSPACE)
-          @comments = nil
-          return
+        if @statement.empty? && (@comments_last_line || 0) < tk.line_no - 2
+          @comments = nil 
         end
         
+        return unless tk.class == TkCOMMENT
+        return if !@statement.empty? && @comments
+        
+        @comments = nil if @comments_last_line && @comments_last_line < tk.line_no - 1
         @comments_line = tk.line_no unless @comments
 
         # Remove the "#" and up to 1 space before the text
@@ -192,6 +192,7 @@ module YARD
         @comments ||= []
         @comments << tk.text.gsub(/^#+\s{0,1}/, '')
         @comments.pop if @comments.size == 1 && @comments.first =~ /^\s*$/
+        @comments_last_line = tk.line_no
         true
       end
 

data/lib/yard/parser/ruby/ruby_parser.rb

@@ -39,7 +39,6 @@ module YARD
           :END => "END",
           :alias => "alias",
           :array => :lbracket,
-          :aref => :lbracket,
           :arg_paren => :lparen,
           :begin => "begin",
           :blockarg => "&",
@@ -150,7 +149,8 @@ module YARD
         [:kw, :op].each do |event|
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
             def on_#{event}(tok)
-              unless @tokens.last && @tokens.last[0] == :symbeg
+              unless @last_ns_token == [:kw, "def"] ||
+                  (@tokens.last && @tokens.last[0] == :symbeg)
                 (@map[tok] ||= []) << [lineno, charno]
               end
               visit_ns_token(:#{event}, tok, true)
@@ -185,6 +185,7 @@ module YARD
         def visit_ns_token(token, data, ast_token = false)
           add_token(token, data)
           ch = charno
+          @last_ns_token = [token, data]
           @charno += data.length
           @ns_charno = charno
           if ast_token
@@ -205,7 +206,7 @@ module YARD
         end
 
         def on_body_stmt(*args)
-          args.first
+          args.compact.size == 1 ? args.first : AstNode.new(:list, args)
         end
         
         def on_assoc_new(*args)
@@ -224,6 +225,29 @@ module YARD
           args.first
         end
         
+        def on_aref(*args)
+          ll, lc = *@map[:aref].pop
+          sr = args.first.source_range.first..lc
+          lr = args.first.line_range.first..ll
+          AstNode.new(:aref, args, char: sr, line: lr)
+        end
+        
+        def on_rbracket(tok)
+          (@map[:aref] ||= []) << [lineno, charno]
+          visit_ns_token(:rbracket, tok, false)
+        end
+        
+        [:if_mod, :unless_mod, :while_mod].each do |kw|
+          node_class = AstNode.node_class_for(kw)
+          module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            def on_#{kw}(*args)
+              sr = args.last.source_range.first..args.first.source_range.last
+              lr = args.last.line_range.first..args.first.line_range.last
+              #{node_class}.new(:#{kw}, args, line: lr, char: sr)
+            end
+          eof
+        end
+        
         def on_qwords_new
           visit_event AstNode.new(:qwords_literal, [])
         end
@@ -239,6 +263,11 @@ module YARD
         def on_string_content(*args)
           AstNode.new(:string_content, args, listline: lineno..lineno, listchar: charno..charno)
         end
+        
+        def on_rescue(exc, *args)
+          exc = AstNode.new(:list, exc) if exc
+          visit_event AstNode.new(:rescue, [exc, *args])
+        end
 
         def on_void_stmt
           AstNode.new(:void_stmt, [], line: lineno..lineno, char: charno...charno)

data/lib/yard/parser/source_parser.rb

@@ -3,19 +3,51 @@ require 'continuation' unless RUBY18
 
 module YARD
   module Parser
+    # Raised when an object is recognized but cannot be documented. This
+    # generally occurs when the Ruby syntax used to declare an object is
+    # too dynamic in nature. 
     class UndocumentableError < Exception; end
+    
+    # Raised when the parser sees a Ruby syntax error
     class ParserSyntaxError < UndocumentableError; end
+    
+    # A LoadOrderError occurs when a handler needs to modify a 
+    # {CodeObjects::NamespaceObject} (usually by adding a child to it)
+    # that has not yet been resolved. 
+    # 
+    # @see Handers::Base#ensure_loaded!
     class LoadOrderError < Exception; end
     
-    # Responsible for parsing a source file into the namespace
+    # Responsible for parsing a source file into the namespace. Parsing
+    # also invokes handlers to process the parsed statements and generate
+    # any code objects that may be recognized.
+    # 
+    # @see Handlers::Base
+    # @see CodeObjects::Base
     class SourceParser 
       class << self
+        # The default parser type to use for blocks of code. Change this
+        # attribute to apply a new parser type for all newly parsed
+        # blocks of code.
+        # 
+        # @return [Symbol] the parser type
         attr_accessor :parser_type
         
+        # Sets the parser and makes sure it's a valid type
+        # 
+        # @param [Symbol] value the new parser type
+        # @return [nil] 
         def parser_type=(value)
           @parser_type = validated_parser_type(value)
         end
         
+        # Parses a path or set of paths
+        # 
+        # @param [String, Array<String>] paths a path, glob, or list of paths to
+        #   parse
+        # @param [Fixnum] level the logger level to use during parsing. See
+        #   {YARD::Logger}
+        # @return the parser object that was used to parse the source. 
         def parse(paths = "lib/**/*.rb", level = log.level)
           log.debug("Parsing #{paths} with `#{parser_type}` parser")
           files = [paths].flatten.map {|p| p.include?("*") ? Dir[p] : p }.flatten
@@ -25,21 +57,43 @@ module YARD
           end
         end
       
+        # Parses a string +content+
+        # 
+        # @param [String] content the block of code to parse
+        # @param [Symbol] ptype the parser type to use. See {parser_type}.
+        # @return the parser object that was used to parse +content+
         def parse_string(content, ptype = parser_type)
           new(ptype).parse(StringIO.new(content))
         end
         
+        # Tokenizes but does not parse the block of code
+        # 
+        # @param [String] content the block of code to tokenize
+        # @param [Symbol] ptype the parser type to use. See {parser_type}.
+        # @return [Array] a list of tokens
         def tokenize(content, ptype = parser_type)
           new(ptype).tokenize(content)
         end
         
+        # Returns the validated parser type. Basically, enforces that :ruby
+        # type is never set from Ruby 1.8
+        # 
+        # @param [Symbol] type the parser type to set
+        # @return [Symbol] the validated parser type
         def validated_parser_type(type)
           RUBY18 && type == :ruby ? :ruby18 : type
         end
 
         private
         
+        # Parses a list of files in a queue. If a {LoadOrderError} is caught,
+        # the file is moved to the back of the queue with a Continuation object
+        # that can continue processing the file.
+        # 
+        # @param [Array<String>] files a list of files to queue for parsing
+        # @return [nil]
         def parse_in_order(*files)
+          files = files.sort_by {|x| x.length if x }
           while file = files.shift
             begin
               if file.is_a?(Array) && file.last.is_a?(Continuation)
@@ -59,19 +113,28 @@ module YARD
 
       self.parser_type = :ruby
       
-      attr_reader :file, :parser_type
+      # The filename being parsed by the parser.
+      attr_reader :file
+      
+      # The parser type associated with the parser instance. This should
+      # be set by the {#initialize constructor}.
+      attr_reader :parser_type
 
+      # Creates a new parser object for code parsing with a specific parser type.
+      # 
+      # @param [Symbol] parser_type the parser type to use
+      # @param [Boolean] load_order_errors whether or not to raise the {LoadOrderError}
       def initialize(parser_type = SourceParser.parser_type, load_order_errors = false)
         @load_order_errors = load_order_errors
         @file = '(stdin)'
         self.parser_type = parser_type
       end
 
-      ##
-      # Creates a new SourceParser that parses a file and returns
-      # analysis information about it.
+      # The main parser method. This should not be called directly. Instead,
+      # use the class methods {parse} and {parse_string}.
       #
       # @param [String, #read, Object] content the source file to parse
+      # @return the parser object used to parse the source
       def parse(content = __FILE__)
         case content
         when String
@@ -91,6 +154,10 @@ module YARD
         log.warn(e.message.capitalize)
       end
       
+      # Tokenizes but does not parse the block of code using the current {#parser_type}
+      # 
+      # @param [String] content the block of code to tokenize
+      # @return [Array] a list of tokens
       def tokenize(content)
         case parser_type
         when :c
@@ -106,6 +173,8 @@ module YARD
       
       private
 
+      # Runs a {Handlers::Processor} object to post process the parsed statements.
+      # @return [nil] 
       def post_process
         post = Handlers::Processor.new(@file, @load_order_errors, @parser_type)
         post.process(@parser.enumerator)
@@ -115,6 +184,10 @@ module YARD
         @parser_type = self.class.validated_parser_type(value)
       end
       
+      # Guesses the parser type to use depending on the file extension.
+      # 
+      # @param [String] filename the filename to use to guess the parser type
+      # @return [Symbol] a parser type that matches the filename
       def parser_type_for_filename(filename)
         case (File.extname(filename)[1..-1] || "").downcase
         when "c", "cpp", "cxx"
@@ -124,6 +197,10 @@ module YARD
         end
       end
       
+      # Selects a parser to use to parse +content+.
+      # 
+      # @param [String] content the block of code to parse
+      # @return the resulting parser object
       def parse_statements(content)
         case parser_type
         when :c

data/lib/yard/rake/yardoc_task.rb

@@ -4,13 +4,42 @@ require 'rake/tasklib'
 module YARD
   module Rake
     
+    # The rake task to run {CLI::Yardoc} and generate documentation.
     class YardocTask < ::Rake::TaskLib
+      # The name of the task
+      # @return [String] the task name
       attr_accessor :name
+      
+      # Options to pass to {CLI::Yardoc}
+      # @return [Hash] the options passed to the commandline utility
       attr_accessor :options
+      
+      # The Ruby source files (and any extra documentation files separated by '-')
+      # to process
+      # @return [Array<String>] a list of files
       attr_accessor :files
-      attr_accessor :before, :after
+      
+      # Runs a +Proc+ before the task
+      # @return [Proc] a proc to call before running the task
+      attr_accessor :before
+      
+      # Runs a +Proc+ after the task
+      # @return [Proc] a proc to call after running the task
+      attr_accessor :after
+      
+      # @return [Proc] an optional lambda to run against all objects being
+      #   generated. Any object that the lambda returns false for will be
+      #   excluded from documentation. 
+      # @see Verifier
+      attr_accessor :verifier
 
-      def initialize(name = :yardoc)
+      # Creates a new task with name +name+.
+      # 
+      # @param [String, Symbol] name the name of the rake task
+      # @yield a block to allow any options to be modified on the task
+      # @yieldparam [YardocTask] _self the task object to allow any parameters
+      #   to be changed.
+      def initialize(name = :yard)
         @name = name
         @options = []
         @files = []
@@ -22,11 +51,17 @@ module YARD
         define
       end
       
+      protected
+      
+      # Defines the rake task
+      # @return [nil] 
       def define
         desc "Generate YARD Documentation"
         task(name) do
           before.call if before.is_a?(Proc)
-          YARD::CLI::Yardoc.run *(options + files) 
+          yardoc = YARD::CLI::Yardoc.new
+          yardoc.options[:verifier] = verifier if verifier
+          yardoc.run *(options + files) 
           after.call if after.is_a?(Proc)
         end
       end

data/lib/yard/registry.rb

@@ -2,6 +2,18 @@ require 'singleton'
 require 'find'
 
 module YARD
+  # The +Registry+ is the centralized data store for all {CodeObjects} created
+  # during parsing. The storage is a key value store with the object's path
+  # (see {CodeObjects::Base#path}) as the key and the object itself as the value.
+  # Object paths must be unique to be stored in the Registry. All lookups for 
+  # objects are done on the singleton Registry instance using the {Registry#at} 
+  # or {Registry#resolve} methods.
+  # 
+  # The registry is saved to a "yardoc" file, which can be loaded back to 
+  # perform any lookups.
+  # 
+  # This class is a singleton class. Any method called on the class will be
+  # delegated to the instance.
   class Registry 
     DEFAULT_YARDOC_FILE = ".yardoc"
     
@@ -10,17 +22,43 @@ module YARD
     @objects = {}
 
     class << self
+      # Holds the objects cache. This attribute should never be accessed
+      # directly.
+      # @return [Array<CodeObjects::Base>] the objects cache
       attr_reader :objects
       
+      # Clears the registry and cache
+      # @return [nil] 
       def clear
         instance.clear 
         objects.clear
       end
     end
 
+    # Gets/sets the yardoc filename
+    # @return [String] the yardoc filename
+    # @see DEFAULT_YARDOC_FILE
     attr_accessor :yardoc_file
+    
+    # The assumed types of a list of paths
+    # @return [{String => Symbol}] a set of unresolved paths and their assumed type
     attr_reader :proxy_types
     
+    # Loads the registry and/or parses a list of files
+    # 
+    # @example Loads the yardoc file or parses files 'a', 'b' and 'c' (but not both)
+    #   Registry.load(['a', 'b', 'c'])
+    # @example Reparses files 'a' and 'b' regardless if yardoc file exists
+    #   Registry.load(['a', 'b'], true)
+    # @param [String, Array] files if +files+ is an Array, it should represent
+    #   a list of files that YARD should parse into the registry. If reload is
+    #   set to false and the yardoc file already exists, these files are skipped.
+    #   If files is a String, it should represent the yardoc file to load
+    #   into the registry.
+    # @param [Boolean] reload if reload is false and a yardoc file already
+    #   exists, any files passed in will be ignored.
+    # @return [Boolean] true if the registry was successfully loaded 
+    # @raise [ArgumentError] if files is not a String or Array
     def load(files = [], reload = false)
       if files.is_a?(Array)
         if File.exists?(yardoc_file) && !reload
@@ -39,6 +77,10 @@ module YARD
       end
     end
     
+    # Loads a yardoc file directly
+    # 
+    # @param [String] file the yardoc file to load.
+    # @return [nil] 
     def load_yardoc(file = yardoc_file)
       return false unless File.exists?(file)
       ns, pt = *Marshal.load(IO.read(file))
@@ -46,11 +88,27 @@ module YARD
       proxy_types.update(pt)
     end
     
+    # Saves the registry to +file+
+    # 
+    # @param [String] file the yardoc file to save to
+    # @return [Boolean] true if the file was saved
     def save(file = yardoc_file)
-      File.open(file, "w") {|f| Marshal.dump([@namespace, @proxy_types], f) }
+      File.open(file, "wb") {|f| Marshal.dump([@namespace, @proxy_types], f) }
       true
     end
 
+    # Returns all objects in the registry that match one of the types provided
+    # in the +types+ list (if +types+ is provided).
+    # 
+    # @example Returns all objects
+    #   Registry.all
+    # @example Returns all classes and modules
+    #   Registry.all(:class, :module)
+    # @param [Array<Symbol>] types an optional list of types to narrow the
+    #   objects down by. Equivalent to performing a select: 
+    #     +Registry.all.select {|o| types.include(o.type) }+
+    # @return [Array<CodeObjects::Base>] the list of objects found
+    # @see CodeObjects::Base#type
     def all(*types)
       namespace.values.select do |obj| 
         if types.empty?
@@ -64,37 +122,83 @@ module YARD
       end + (types.include?(:root) ? [root] : [])
     end
     
+    # Returns the paths of all of the objects in the registry.
+    # @return [Array<String>] all of the paths in the registry.
     def paths
       namespace.keys.map {|k| k.to_s }
     end
-      
+    
+    # Returns the object at a specific path.
+    # @param [String, :root] path the pathname to look for. If +path+ is +root+,
+    #   returns the {#root} object.
+    # @return [CodeObjects::Base] the object at path
+    # @return [nil] if no object is found
     def at(path) path.to_s.empty? ? root : namespace[path] end
     alias_method :[], :at
     
+    # The root namespace object.
+    # @return [CodeObjects::RootObject] the root object in the namespace
     def root; namespace[:root] end
     
+    # Deletes an object from the registry
+    # @param [CodeObjects::Base] object the object to remove
+    # @return [nil] 
     def delete(object) 
       namespace.delete(object.path)
       self.class.objects.delete(object.path)
     end
 
+    # Clears the registry
+    # @return [nil] 
     def clear
       @namespace = SymbolHash.new
       @namespace[:root] = CodeObjects::RootObject.new(nil, :root)
       @proxy_types = {}
     end
 
+    # Creates the Registry
+    # @return [Registry]
     def initialize
       @yardoc_file = DEFAULT_YARDOC_FILE
       clear
     end
   
+    # Registers a new object with the registry
+    # 
+    # @param [CodeObjects::Base] object the object to register
+    # @return [CodeObjects::Base] the registered object
     def register(object)
       self.class.objects[object.path] = object
       return if object.is_a?(CodeObjects::Proxy)
       namespace[object.path] = object
     end
 
+    # Attempts to find an object by name starting at +namespace+, performing
+    # a lookup similar to Ruby's method of resolving a constant in a namespace.
+    # 
+    # @example Looks for instance method #reverse starting from A::B::C
+    #   Registry.resolve(P("A::B::C"), "#reverse")
+    # @example Looks for a constant in the root namespace
+    #   Registry.resolve(nil, 'CONSTANT')
+    # @example Looks for a class method respecting the inheritance tree
+    #   Registry.resolve(myclass, 'mymethod', true)
+    # @example Looks for a constant but returns a proxy if not found
+    #   Registry.resolve(P('A::B::C'), 'D', false, true) # => #<yardoc proxy A::B::C::D>
+    # @example Looks for a complex path from a namespace
+    #   Registry.resolve(P('A::B'), 'B::D') # => #<yardoc class A::B::D>
+    # @param [CodeObjects::NamespaceObject, nil] namespace the starting namespace
+    #   (module or class). If +nil+ or +:root+, starts from the {#root} object.
+    # @param [String, Symbol] name the name (or complex path) to look for from
+    #   +namespace+.
+    # @param [Boolean] inheritance Follows inheritance chain (mixins, superclass)
+    #   when performing name resolution if set to +true+.
+    # @param [Boolean] proxy_fallback If +true+, returns a proxy representing
+    #   the unresolved path (namespace + name) if no object is found.
+    # @return [CodeObjects::Base] the object if it is found
+    # @return [CodeObjects::Proxy] a Proxy representing the object if 
+    #   +proxy_fallback+ is +true+.
+    # @return [nil] if +proxy_fallback+ is +false+ and no object was found.
+    # @see P
     def resolve(namespace, name, inheritance = false, proxy_fallback = false)
       if namespace.is_a?(CodeObjects::Proxy)
         return proxy_fallback ? CodeObjects::Proxy.new(namespace, name) : nil
@@ -143,8 +247,14 @@ module YARD
 
     private
   
+    # The internal namespace hash
+    # @return [Hash{String => CodeObjects::Base}] the path/object hash
     attr_accessor :namespace
 
+    # Attempts to resolve a name in a namespace
+    # 
+    # @param [CodeObjects::NamespaceObject] namespace the starting namespace
+    # @param [String] name the name to look for
     def partial_resolve(namespace, name)
       [CodeObjects::NSEP, CodeObjects::CSEP, ''].each do |s|
         next if s.empty? && name =~ /^\w/