yard 0.5.5 → 0.5.6

data/lib/yard/handlers/ruby/struct_handler_methods.rb

@@ -0,0 +1,128 @@
+module YARD::Handlers::Ruby::StructHandlerMethods
+  include YARD::CodeObjects
+  
+  # Extracts the user's defined @member tag for a given class and its member. Returns
+  # nil if the user did not define a @member tag for this struct entry.
+  #
+  # @param [ClassObject] klass the class whose tags we're searching
+  # @param [String] member the name of the struct member we need
+  # @param [Symbol] type reader method, or writer method?
+  # @return [Tags::Tag, nil] the tag matching the request, or nil if not found
+  def member_tag_for_member(klass, member, type = :read)
+    specific_tag = type == :read ? :attr_reader : :attr_writer
+    (klass.tags(specific_tag) + klass.tags(:attr)).find {|tag| tag.name == member}
+  end
+  
+  # Determines whether to create an attribute method based on the class's
+  # tags.
+  #
+  # @param [ClassObject] klass the class whose tags we're searching
+  # @param [String] member the name of the struct member we need
+  # @param [Symbol] type (:read) reader method, or writer method?
+  # @return [Boolean] should the attribute be created?
+  def create_member_method?(klass, member, type = :read)
+    return true if (klass.tags(:attr) + klass.tags(:attr_reader) + klass.tags(:attr_writer)).empty?
+    return true if member_tag_for_member(klass, member, type)
+    return !member_tag_for_member(klass, member, :write) if type == :read
+    return !member_tag_for_member(klass, member, :read)
+  end
+  
+  # Gets the return type for the member in a nicely formatted string. Used
+  # to be injected into auto-generated docstrings.
+  #
+  # @param [ClassObject] klass the class whose tags we're searching
+  # @param [String] member the name of the struct member whose return type we need
+  # @return [String] the user-declared type of the struct member, or [Object] if
+  #   the user did not define a type for this member.
+  def return_type_from_tag(member_tag)
+    (member_tag && member_tag.types) ? member_tag.types : "Object"
+  end
+  
+  # Creates the auto-generated docstring for the getter method of a struct's
+  # member. This is used so the generated documentation will look just like that
+  # of an attribute defined using attr_accessor.
+  #
+  # @param [ClassObject] klass the class whose members we're working with
+  # @param [String] member the name of the member we're generating documentation for
+  # @return [String] a docstring to be attached to the getter method for this member
+  def add_reader_tags(klass, new_method, member)
+    member_tag = member_tag_for_member(klass, member, :read)
+    return_type = return_type_from_tag(member_tag)
+    getter_doc_text = member_tag ? member_tag.text : "Returns the value of attribute #{member}"
+    new_method.docstring.replace(getter_doc_text)
+    new_method.docstring.add_tag YARD::Tags::Tag.new(:return, "the current value of #{member}", return_type)
+  end
+  
+  # Creates the auto-generated docstring for the setter method of a struct's
+  # member. This is used so the generated documentation will look just like that
+  # of an attribute defined using attr_accessor.
+  #
+  # @param [ClassObject] klass the class whose members we're working with
+  # @param [String] member the name of the member we're generating documentation for
+  # @return [String] a docstring to be attached to the setter method for this member
+  def add_writer_tags(klass, new_method, member)
+    member_tag = member_tag_for_member(klass, member, :write)
+    return_type = return_type_from_tag(member_tag)
+    setter_doc_text = member_tag ? member_tag.text : "Sets the attribute #{member}"
+    new_method.docstring.replace(setter_doc_text)
+    new_method.docstring.add_tag YARD::Tags::Tag.new(:param, "the value to set the attribute #{member} to.", return_type, "value")
+    new_method.docstring.add_tag YARD::Tags::Tag.new(:return, "the newly set value", return_type)
+  end
+  
+  # Creates and registers a class object with the given name and superclass name.
+  # Returns it for further use.
+  #
+  # @param [String] classname the name of the class
+  # @param [String] superclass the name of the superclass
+  # @return [ClassObject] the class object for further processing/method attaching
+  def create_class(classname, superclass)
+    register ClassObject.new(namespace, classname) do |o|
+      o.superclass = superclass if superclass
+      o.superclass.type = :class if o.superclass.is_a?(Proxy)
+    end
+  end
+  
+  # Creates the setter (writer) method and attaches it to the class as an attribute.
+  # Also sets up the docstring to prettify the documentation output.
+  #
+  # @param [ClassObject] klass the class to attach the method to
+  # @param [String] member the name of the member we're generating a method for
+  def create_writer(klass, member)
+    # We want to convert these members into attributes just like
+    # as if they were declared using attr_accessor.
+    new_meth = register MethodObject.new(klass, "#{member}=", :instance) do |o|
+      o.parameters = [['value', nil]]
+      o.signature ||= "def #{member}=(value)"
+      o.source ||= "#{o.signature}\n  @#{member} = value\nend"
+    end
+    add_writer_tags(klass, new_meth, member)
+    klass.attributes[:instance][member][:write] = new_meth
+  end
+  
+  # Creates the getter (reader) method and attaches it to the class as an attribute.
+  # Also sets up the docstring to prettify the documentation output.
+  #
+  # @param [ClassObject] klass the class to attach the method to
+  # @param [String] member the name of the member we're generating a method for
+  def create_reader(klass, member)
+    new_meth = register MethodObject.new(klass, member, :instance) do |o|
+      o.signature ||= "def #{member}"
+      o.source ||= "#{o.signature}\n  @#{member}\nend"
+    end
+    add_reader_tags(klass, new_meth, member)
+    klass.attributes[:instance][member][:read] = new_meth
+  end
+  
+  # Creates the given member methods and attaches them to the given ClassObject.
+  #
+  # @param [ClassObject] klass the class to generate attributes for
+  # @param [Array<String>] members a list of member names
+  def create_attributes(klass, members)
+    # For each parameter, add reader and writers
+    members.each do |member|
+      klass.attributes[:instance][member] = SymbolHash[:read => nil, :write => nil]
+      create_writer klass, member if create_member_method?(klass, member, :write)
+      create_reader klass, member if create_member_method?(klass, member, :read)
+    end
+  end
+end

data/lib/yard/parser/base.rb

@@ -0,0 +1,55 @@
+module YARD
+  module Parser
+    # Represents the abstract base parser class that parses source code in
+    # a specific way. A parser should implement {#parse}, {#tokenize} and 
+    # {#enumerator}.
+    # 
+    # == Registering a Custom Parser
+    # To register a parser, see {SourceParser.register_parser_type}
+    # 
+    # @abstract
+    # @see #parse
+    # @see #tokenize
+    # @see #enumerator
+    class Base
+      # Convenience method to create a new parser and {#parse}
+      def self.parse(source, filename = nil)
+        new(source, filename).parse
+      end
+      
+      # This default constructor does nothing. The subclass is responsible for
+      # storing the source contents and filename if they are required.
+      # @param [String] source the source contents
+      # @param [String] filename the name of the file if from disk
+      def initialize(source, filename)
+        raise NotImplementedError, "invalid parser implementation"
+      end
+      
+      # This method should be implemented to parse the source and return itself.
+      # @abstract
+      # @return [Base] this method should return itself
+      def parse
+        raise NotImplementedError, "#{self.class} must implement #parse"
+      end
+      
+      # This method should be implemented to tokenize given source
+      # @abstract
+      # @return [Array] a list/tree of lexical tokens
+      def tokenize
+        raise NotImplementedError, "#{self.class} does not support tokenization"
+      end
+      
+      # This method should be implemented to return a list of semantic tokens
+      # representing the source code to be post-processed. Otherwise the method
+      # should return nil.
+      # 
+      # @abstract
+      # @return [Array] a list of semantic tokens representing the source code
+      #   to be post-processed
+      # @return [nil] if no post-processing should be done
+      def enumerator
+        nil
+      end
+    end
+  end
+end

data/lib/yard/parser/c_parser.rb

@@ -4,7 +4,7 @@
 
 module YARD
   module Parser
-    class CParser
+    class CParser < Base
       def initialize(source, file = '(stdin)')
         @file = file
         @namespaces = {}
@@ -18,6 +18,10 @@ module YARD
         parse_includes
       end
       
+      def tokenize
+        raise NotImplementedError, "no tokenization support for C/C++ files"
+      end
+      
       private
       
       def remove_var_prefix(var)

data/lib/yard/parser/ruby/ast_node.rb

@@ -38,8 +38,8 @@ module YARD
       # list, like Strings or Symbols representing names. To return only 
       # the AstNode children of the node, use {#children}.
       class AstNode < Array
-        attr_accessor :type, :parent, :docstring, :file, :full_source, :source
-        attr_accessor :source_range, :line_range, :docstring_range
+        attr_accessor :type, :parent, :docstring, :docstring_range, :source
+        attr_writer :source_range, :line_range, :file, :full_source
         alias comments docstring
         alias comments_range docstring_range
         alias to_s source
@@ -235,12 +235,12 @@ module YARD
           q.group(3, 's(', ')') do
             q.seplist(objs, nil, :each) do |v| 
               if v == :__last__
-                q.seplist(options, nil, :each) do |k, v| 
+                q.seplist(options, nil, :each) do |k, v2| 
                   q.group(3) do 
                     q.text k
                     q.group(3) do 
                       q.text ': '
-                      q.pp v 
+                      q.pp v2 
                     end
                   end
                 end

data/lib/yard/parser/ruby/legacy/ruby_parser.rb

@@ -0,0 +1,26 @@
+module YARD
+  module Parser
+    module Ruby
+      module Legacy
+        # Legacy Ruby parser
+        class RubyParser < Parser::Base
+          def initialize(source, filename)
+            @source = source
+          end
+          
+          def parse
+            @parse ||= StatementList.new(@source)
+          end
+          
+          def tokenize
+            @tokenize ||= TokenList.new(@source)
+          end
+          
+          def enumerator
+            @parse
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yard/parser/ruby/legacy/statement_list.rb

@@ -24,10 +24,6 @@ module YARD
         parse_statements
       end
       
-      def enumerator
-        self
-      end
-
       private
 
       def parse_statements

data/lib/yard/parser/ruby/ruby_parser.rb

@@ -3,12 +3,25 @@ require 'ripper'
 module YARD
   module Parser
     module Ruby
-      class RubyParser < Ripper
+      # Ruby 1.9 parser
+      class RubyParser < Parser::Base
+        def initialize(source, filename)
+          @parser = RipperParser.new(source, filename)
+        end
+        
+        def parse; @parser.parse end
+        def tokenize; @parser.tokens end
+        def enumerator; @parser.enumerator end
+      end
+      
+      # Internal parser class
+      class RipperParser < Ripper
         attr_reader :ast, :charno, :comments, :file, :tokens
         alias root ast
 
         def initialize(source, filename, *args)
           super
+          @last_ns_token = nil
           @file = filename
           @source = source
           @tokens = []
@@ -107,6 +120,7 @@ module YARD
             eof
           elsif /_add(_.+)?\z/ =~ event
             module_eval(<<-eof, __FILE__, __LINE__ + 1)
+              undef on_#{event} if instance_method(:on_#{event})
               def on_#{event}(list, item)
                 list.push(item)
                 list
@@ -114,12 +128,14 @@ module YARD
             eof
           elsif MAPPINGS.has_key?(event)
             module_eval(<<-eof, __FILE__, __LINE__ + 1)
+              undef on_#{event} if instance_method(:on_#{event})
               def on_#{event}(*args)
                 visit_event #{node_class}.new(:#{event}, args)
               end
             eof
           else
             module_eval(<<-eof, __FILE__, __LINE__ + 1)
+              undef on_#{event} if instance_method(:on_#{event})
               def on_#{event}(*args)
                 #{node_class}.new(:#{event}, args, listline: lineno..lineno, listchar: charno...charno)
               end
@@ -130,6 +146,7 @@ module YARD
         SCANNER_EVENTS.each do |event|
           ast_token = AST_TOKENS.include?(event)
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            undef on_#{event} if instance_method(:on_#{event})
             def on_#{event}(tok)
               visit_ns_token(:#{event}, tok, #{ast_token.inspect})
             end
@@ -139,6 +156,7 @@ module YARD
         REV_MAPPINGS.select {|k| k.is_a?(Symbol) }.each do |event, value|
           ast_token = AST_TOKENS.include?(event)
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            undef on_#{event} if instance_method(:on_#{event})
             def on_#{event}(tok)
               (@map[:#{event}] ||= []) << [lineno, charno]
               visit_ns_token(:#{event}, tok, #{ast_token.inspect})
@@ -148,6 +166,7 @@ module YARD
         
         [:kw, :op].each do |event|
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            undef on_#{event} if instance_method(:on_#{event})
             def on_#{event}(tok)
               unless @last_ns_token == [:kw, "def"] ||
                   (@tokens.last && @tokens.last[0] == :symbeg)
@@ -160,6 +179,7 @@ module YARD
 
         [:sp, :nl, :ignored_nl].each do |event|
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            undef on_#{event} if instance_method(:on_#{event})
             def on_#{event}(tok)
               add_token(:#{event}, tok)
               @charno += tok.length
@@ -201,6 +221,28 @@ module YARD
           end
         end
 
+        undef on_program
+        undef on_bodystmt
+        undef on_assoc_new
+        undef on_hash
+        undef on_bare_assoc_hash
+        undef on_assoclist_from_args
+        undef on_aref
+        undef on_rbracket
+        undef on_qwords_new
+        undef on_string_literal
+        undef on_lambda
+        undef on_string_content
+        undef on_rescue
+        undef on_void_stmt
+        undef on_params
+        undef on_label
+        undef on_comment
+        undef on_embdoc_beg
+        undef on_embdoc
+        undef on_embdoc_end
+        undef on_parse_error
+
         def on_program(*args)
           args.first
         end
@@ -241,6 +283,7 @@ module YARD
         [:if_mod, :unless_mod, :while_mod].each do |kw|
           node_class = AstNode.node_class_for(kw)
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
+            undef on_#{kw} if instance_method(:on_#{kw})
             def on_#{kw}(*args)
               sr = args.last.source_range.first..args.first.source_range.last
               lr = args.last.line_range.first..args.first.line_range.last
@@ -340,7 +383,7 @@ module YARD
         def insert_comments
           root.traverse do |node|
             next if node.type == :list || node.parent.type != :list
-            node.line.downto(node.line - 2) do |line|
+            (node.line - 2).upto(node.line) do |line|
               comment = @comments[line]
               if comment && !comment.empty?
                 node.docstring = comment

data/lib/yard/parser/source_parser.rb

@@ -22,22 +22,19 @@ module YARD
     # also invokes handlers to process the parsed statements and generate
     # any code objects that may be recognized.
     # 
+    # == Custom Parsers
+    # SourceParser allows custom parsers to be registered and called when
+    # a certain filetype is recognized. To register a parser and hook it
+    # up to a set of file extensions, call {register_parser_type}
+    # 
+    # @see register_parser_type
     # @see Handlers::Base
     # @see CodeObjects::Base
     class SourceParser 
       class << self
-        # The default parser type to use for blocks of code. Change this
-        # attribute to apply a new parser type for all newly parsed
-        # blocks of code.
-        # 
-        # @return [Symbol] the parser type
-        attr_accessor :parser_type
-        undef parser_type=
+        # @return [Symbol] the default parser type (defaults to :ruby)
+        attr_reader :parser_type
         
-        # Sets the parser and makes sure it's a valid type
-        # 
-        # @param [Symbol] value the new parser type
-        # @return [void] 
         def parser_type=(value)
           @parser_type = validated_parser_type(value)
         end
@@ -86,6 +83,49 @@ module YARD
           new(ptype).tokenize(content)
         end
         
+        # Registers a new parser type.
+        # 
+        # @example Registering a parser for "java" files
+        #   SourceParser.register_parser_type :java, JavaParser, 'java'
+        # @param [Symbol] type a symbolic name for the parser type
+        # @param [Base] parser_klass a class that implements parsing and tokenization
+        # @param [Array<String>, String, Regexp] extensions a list of extensions or a
+        #   regex to match against the file extension
+        # @return [void]
+        # @see Parser::Base
+        def register_parser_type(type, parser_klass, extensions = nil)
+          unless Base > parser_klass
+            raise ArgumentError, "expecting parser_klass to be a subclass of YARD::Parser::Base"
+          end
+          parser_type_extensions[type.to_sym] = extensions if extensions
+          parser_types[type.to_sym] = parser_klass
+        end
+        
+        # @return [Hash{Symbol=>Object}] a list of registered parser types
+        def parser_types; @@parser_types ||= {} end
+        def parser_types=(value) @@parser_types = value end
+        
+        # @return [Hash] a list of registered parser type extensions
+        def parser_type_extensions; @@parser_type_extensions ||= {} end
+        def parser_type_extensions=(value) @@parser_type_extensions = value end
+
+        # Finds a parser type that is registered for the extension. If no
+        # type is found, the default Ruby type is returned.
+        # 
+        # @return [Symbol] the parser type to be used for the extension
+        def parser_type_for_extension(extension)
+          type = parser_type_extensions.find do |t, exts|
+            if exts.is_a?(Array)
+              exts.include?(extension)
+            elsif exts.is_a?(String)
+              exts == extension
+            elsif exts.is_a?(Regexp)
+              extension =~ exts
+            end
+          end
+          validated_parser_type(type ? type.first : :ruby)
+        end
+        
         # Returns the validated parser type. Basically, enforces that :ruby
         # type is never set from Ruby 1.8
         # 
@@ -94,7 +134,7 @@ module YARD
         def validated_parser_type(type)
           RUBY18 && type == :ruby ? :ruby18 : type
         end
-
+        
         private
         
         # Parses a list of files in a queue. If a {LoadOrderError} is caught,
@@ -124,6 +164,10 @@ module YARD
 
       self.parser_type = :ruby
       
+      register_parser_type :ruby,   Ruby::RubyParser if RUBY19
+      register_parser_type :ruby18, Ruby::Legacy::RubyParser
+      register_parser_type :c,      CParser, ['c', 'cc', 'cxx', 'cpp']
+      
       # The filename being parsed by the parser.
       attr_reader :file
       
@@ -163,7 +207,8 @@ module YARD
           content = content.read if content.respond_to? :read
         end
         
-        @parser = parse_statements(content)
+        @parser = parser_class.new(content, file)
+        @parser.parse
         post_process
         @parser
       rescue ArgumentError, NotImplementedError => e
@@ -177,16 +222,8 @@ module YARD
       # @param [String] content the block of code to tokenize
       # @return [Array] a list of tokens
       def tokenize(content)
-        case parser_type
-        when :c
-          raise NotImplementedError, "no support for C/C++ files"
-        when :ruby18
-          Ruby::Legacy::TokenList.new(content)
-        when :ruby
-          Ruby::RubyParser.parse(content).tokens
-        else
-          raise ArgumentError, "invalid parser type or unrecognized file"
-        end
+        @parser = parser_class.new(content, file)
+        @parser.tokenize
       end
       
       private
@@ -205,8 +242,9 @@ module YARD
       # @return [void] 
       def post_process
         return unless @parser.respond_to? :enumerator
+        return unless enumerator = @parser.enumerator
         post = Handlers::Processor.new(@file, @load_order_errors, @parser_type)
-        post.process(@parser.enumerator)
+        post.process(enumerator)
       end
 
       def parser_type=(value)
@@ -218,29 +256,15 @@ module YARD
       # @param [String] filename the filename to use to guess the parser type
       # @return [Symbol] a parser type that matches the filename
       def parser_type_for_filename(filename)
-        case (File.extname(filename)[1..-1] || "").downcase
-        when "c", "cpp", "cxx"
-          :c
-        else # when "rb", "rbx", "erb"
-          parser_type == :ruby18 ? :ruby18 : :ruby
-        end
+        ext = (File.extname(filename)[1..-1] || "").downcase
+        type = self.class.parser_type_for_extension(ext)
+        parser_type == :ruby18 && type == :ruby ? :ruby18 : type
       end
       
-      # Selects a parser to use to parse +content+.
-      # 
-      # @param [String] content the block of code to parse
-      # @return the resulting parser object
-      def parse_statements(content)
-        case parser_type
-        when :c
-          CParser.new(content, file).parse
-        when :ruby18
-          Ruby::Legacy::StatementList.new(content)
-        when :ruby
-          Ruby::RubyParser.parse(content, file)
-        else
-          raise ArgumentError, "invalid parser type or unrecognized file"
-        end
+      def parser_class
+        klass = self.class.parser_types[parser_type]
+        raise ArgumentError, "invalid parser type '#{parser_type}' or unrecognized file", caller[1..-1] if !klass
+        klass
       end
     end
   end