yard 0.7.5 → 0.8.0

data/lib/yard/core_ext/symbol_hash.rb

@@ -57,7 +57,7 @@ class SymbolHash < Hash
   # @return [Boolean] whether the key exists
   def has_key?(key) super(key.to_sym) end
 
-  # Updates the object with the contents of another Hash object
+  # Updates the object with the contents of another Hash object.
   # This method modifies the original SymbolHash object
   #
   # @param [Hash] hash the hash object to copy the values from

data/lib/yard/docstring.rb

@@ -30,13 +30,39 @@ module YARD
     def hash_flag=(v) @hash_flag = v == nil ? false : v end
 
     # Matches a tag at the start of a comment line
-    META_MATCH = /^@([a-z_0-9]+)(?:\s+(.*))?$/i
+    # @deprecated Use {DocstringParser::META_MATCH}
+    META_MATCH = DocstringParser::META_MATCH
 
     # @group Creating a Docstring Object
 
+    # Creates a new docstring without performing any parsing through
+    # a {DocstringParser}. This method is called by +DocstringParser+
+    # when creating the new docstring object.
+    #
+    # @param [String] text the textual portion of the docstring
+    # @param [Array<Tag>] tags the list of tag objects in the docstring
+    # @param [CodeObjects::Base, nil] object the object associated with the
+    #   docstring. May be nil.
+    # @param [String] raw_data the complete docstring, including all
+    #   original formatting and any unparsed tags/directives.
+    def self.new!(text, tags = [], object = nil, raw_data = nil)
+      docstring = allocate
+      docstring.replace(text, false)
+      docstring.object = object
+      docstring.add_tag(*tags)
+      docstring.instance_variable_set("@all", raw_data) if raw_data
+      docstring
+    end
+
     # Creates a new docstring with the raw contents attached to an optional
-    # object.
+    # object. Parsing will be done by the {DocstringParser} class.
     #
+    # @note To properly parse directives with proper parser context within
+    #   handlers, you should not use this method to create a Docstring.
+    #   Instead, use {DocstringParser}, which takes a handler object that
+    #   can pass parser state onto directives. If a Docstring is created
+    #   with this method, directives do not have access to any parser
+    #   state, and may not function as expected.
     # @example
     #   Docstring.new("hello world\n@return Object return", someobj)
     #
@@ -68,16 +94,16 @@ module YARD
 
     # Replaces the docstring with new raw content. Called by {#all=}.
     # @param [String] content the raw comments to be parsed
-    def replace(content)
+    def replace(content, parse = true)
       content = content.join("\n") if content.is_a?(Array)
       @tags, @ref_tags = [], []
       @all = content
-      super parse_comments(content)
+      super(parse ? parse_comments(content) : content)
     end
     alias all= replace
-    
+
     # Deep-copies a docstring
-    # 
+    #
     # @note This method creates a new docstring with new tag lists, but does
     #   not create new individual tags. Modifying the tag objects will still
     #   affect the original tags.
@@ -124,10 +150,10 @@ module YARD
       @summary += '.' unless @summary.empty?
       @summary
     end
-    
+
     # Reformats and returns a raw representation of the tag data using the
     # current tag and docstring data, not the original text.
-    # 
+    #
     # @return [String] the updated raw formatted docstring data
     # @since 0.7.0
     # @todo Add Tags::Tag#to_raw and refactor
@@ -153,33 +179,10 @@ module YARD
 
     # @group Creating and Accessing Meta-data
 
-    # Creates a tag from the {Tags::DefaultFactory tag factory}.
-    # 
-    # To add an already created tag object, use {#add_tag}
-    #
-    # @param [String] tag_name the tag name
-    # @param [String] tag_buf the text attached to the tag with newlines removed.
-    # @return [Tags::Tag, Tags::RefTag] a tag
-    def create_tag(tag_name, tag_buf)
-      if tag_buf =~ /\A\s*(?:(\S+)\s+)?\(\s*see\s+(\S+)\s*\)\s*\Z/
-        return create_ref_tag(tag_name, $1, $2)
-      end
-
-      tag_factory = Tags::Library.instance
-      tag_method = "#{tag_name}_tag"
-      if tag_name && tag_factory.respond_to?(tag_method)
-        add_tag(*[tag_factory.send(tag_method, tag_buf)].flatten)
-      else
-        log.warn "Unknown tag @#{tag_name}" + (object ? " in file `#{object.file}` near line #{object.line}" : "")
-      end
-    rescue Tags::TagFormatError
-      log.warn "Invalid tag format for @#{tag_name}" + (object ? " in file `#{object.file}` near line #{object.line}" : "")
-    end
-
     # Adds a tag or reftag object to the tag list. If you want to parse
-    # tag data based on the {Tags::DefaultFactory} tag factory, use {#create_tag}
-    # instead.
-    # 
+    # tag data based on the {Tags::DefaultFactory} tag factory, use
+    # {DocstringParser} instead.
+    #
     # @param [Tags::Tag, Tags::RefTag] tags list of tag objects to add
     # @return [void]
     def add_tag(*tags)
@@ -188,7 +191,7 @@ module YARD
         when Tags::Tag
           tag.object = object
           @tags << tag
-        when Tags::RefTag
+        when Tags::RefTag, Tags::RefTagList
           @ref_tags << tag
         else
           raise ArgumentError, "expected Tag or RefTag, got #{tag.class} (at index #{i})"
@@ -226,7 +229,7 @@ module YARD
     def has_tag?(name)
       tags.any? {|tag| tag.tag_name.to_s == name.to_s }
     end
-    
+
     # Delete all tags with +name+
     # @param [String] name the tag name
     # @return [void]
@@ -234,10 +237,10 @@ module YARD
     def delete_tags(name)
       delete_tag_if {|tag| tag.tag_name.to_s == name.to_s }
     end
-    
+
     # Deletes all tags where the block returns true
     # @yieldparam [Tags::Tag] tag the tag that is being tested
-    # @yieldreturn [Boolean] true if the tag should be deleted 
+    # @yieldreturn [Boolean] true if the tag should be deleted
     # @return [void]
     # @since 0.7.0
     def delete_tag_if(&block)
@@ -270,11 +273,6 @@ module YARD
       list.map {|t| t.tags }.flatten
     end
 
-    # Creates a {Tags::RefTag}
-    def create_ref_tag(tag_name, name, object_name)
-      @ref_tags << Tags::RefTagList.new(tag_name, P(object, object_name), name)
-    end
-
     # Parses out comments split by newlines into a new code object
     #
     # @param [String] comments
@@ -284,48 +282,10 @@ module YARD
     # @return [String] the non-metadata portion of the comments to
     #   be used as a docstring
     def parse_comments(comments)
-      comments = comments.split(/\r?\n/) if comments.is_a?(String)
-      return '' if !comments || comments.empty?
-      docstring = ""
-
-      indent, last_indent = comments.first[/^\s*/].length, 0
-      orig_indent = 0
-      last_line = ""
-      tag_name, tag_klass, tag_buf = nil, nil, []
-
-      (comments+['']).each_with_index do |line, index|
-        indent = line[/^\s*/].length
-        empty = (line =~ /^\s*$/ ? true : false)
-        done = comments.size == index
-
-        if tag_name && (((indent < orig_indent && !empty) || done ||
-            (indent == 0 && !empty)) || (indent <= last_indent && line =~ META_MATCH))
-          create_tag(tag_name, tag_buf.join("\n"))
-          tag_name, tag_buf, = nil, []
-          orig_indent = 0
-        end
-
-        # Found a meta tag
-        if line =~ META_MATCH
-          tag_name, tag_buf = $1, [($2 || '')]
-        elsif tag_name && indent >= orig_indent && !empty
-          orig_indent = indent if orig_indent == 0
-          # Extra data added to the tag on the next line
-          last_empty = last_line =~ /^[ \t]*$/ ? true : false
-
-          tag_buf << '' if last_empty
-          tag_buf << line.gsub(/^[ \t]{#{orig_indent}}/, '')
-        elsif !tag_name
-          # Regular docstring text
-          docstring << line << "\n"
-        end
-
-        last_indent = indent
-        last_line = line
-      end
-
-      # Remove trailing/leading whitespace / newlines
-      docstring.gsub!(/\A[\r\n\s]+|[\r\n\s]+\Z/, '')
+      parser = DocstringParser.new
+      parser.parse(comments, object)
+      add_tag(*parser.tags)
+      parser.text
     end
   end
 end

data/lib/yard/docstring_parser.rb

@@ -0,0 +1,269 @@
+require 'ostruct'
+
+module YARD
+  # Parses text and creates a {Docstring} object to represent documentation
+  # for a {CodeObjects::Base}. To create a new docstring, you should initialize
+  # the parser and call {#parse} followed by {#to_docstring}.
+  #
+  # @example Creating a Docstring with a DocstringParser
+  #   DocstringParser.new.parse("text here").to_docstring
+  # @since 0.8.0
+  class DocstringParser
+    # Creates a callback that is called after a docstring is successfully
+    # parsed. Use this method to perform sanity checks on a docstring's
+    # tag data, or add any extra tags automatically to a docstring.
+    #
+    # @yield [parser] a block to be called after a docstring is parsed
+    # @yieldparam [DocstringParser] parser the docstring parser object
+    #   with all directives and tags created.
+    # @yieldreturn [void]
+    # @return [void]
+    def self.after_parse(&block)
+      self.after_parse_callbacks << block
+    end
+
+    # @return [Array<Proc>] the {#after_parse} callback proc objects
+    # @private
+    def self.after_parse_callbacks
+      @after_parse_callbacks ||= []
+    end
+
+    # @return [String] the parsed text portion of the docstring,
+    #   with tags removed.
+    attr_reader :text
+
+    # @return [String] the complete input string to the parser.
+    attr_reader :raw_text
+
+    # @return [Array<Tag>] the list of meta-data tags identified
+    #   by the parser
+    attr_reader :tags
+
+    # @return [Array<Directive>] a list of directives identified
+    #   by the parser. This list will not be passed on to the
+    #   Docstring object.
+    attr_reader :directives
+
+    # @return [OpenStruct] any arbitrary state to be passed between
+    #   tags during parsing. Mainly used by directives to coordinate
+    #   behaviour (so that directives can be aware of other directives
+    #   used in a docstring).
+    attr_reader :state
+
+    # @return [CodeObjects::Base, nil] the object associated with
+    #   the docstring being parsed. May be nil if the docstring is
+    #   not attached to any object.
+    attr_accessor :object
+
+    # @return [Handlers::Base, nil] the handler parsing this
+    #   docstring. May be nil if this docstring parser is not
+    #   initialized through
+    attr_accessor :handler
+
+    # @return [Tags::Library] the tag library being used to
+    #   identify registered tags in the docstring.
+    attr_accessor :library
+
+    # The regular expression to match the tag syntax
+    META_MATCH = /^@(!)?((?:\w\.?)+)(?:\s+(.*))?$/i
+
+    # Creates a new parser to parse docstring data
+    #
+    # @param [Tags::Library] library a tag library for recognizing
+    #   tags.
+    def initialize(library = Tags::Library.instance)
+      @text = ""
+      @raw_text = ""
+      @tags = []
+      @directives = []
+      @library = library
+      @object = nil
+      @handler = nil
+      @state = OpenStruct.new
+    end
+
+    # Parses all content and returns itself.
+    #
+    # @param [String] content the docstring text to parse
+    # @param [CodeObjects::Base] object the object that the docstring
+    #   is attached to. Will be passed to directives to act on
+    #   this object.
+    # @param [Handlers::Base, nil] handler the handler object that is
+    #   parsing this object. May be nil if this parser is not being
+    #   called from a {Parser::SourceParser} context.
+    # @return [self] the parser object. To get the docstring,
+    #   call {#to_docstring}.
+    # @see #to_docstring
+    def parse(content, object = nil, handler = nil)
+      @object = object
+      @handler = handler
+      @raw_text = content
+      text = parse_content(content)
+      # Remove trailing/leading whitespace / newlines
+      @text = text.gsub(/\A[\r\n\s]+|[\r\n\s]+\Z/, '')
+      call_directives_after_parse
+      call_after_parse_callbacks
+      self
+    end
+
+    # @return [Docstring] translates parsed text into
+    #   a Docstring object.
+    def to_docstring
+      Docstring.new!(text, tags, object, raw_text)
+    end
+
+    private
+
+    # Parses all text and tags
+    def parse_content(content)
+      content = content.split(/\r?\n/) if content.is_a?(String)
+      return '' if !content || content.empty?
+      docstring = ""
+
+      indent, last_indent = content.first[/^\s*/].length, 0
+      orig_indent = 0
+      directive = false
+      last_line = ""
+      tag_name, tag_klass, tag_buf = nil, nil, []
+
+      (content+['']).each_with_index do |line, index|
+        indent = line[/^\s*/].length
+        empty = (line =~ /^\s*$/ ? true : false)
+        done = content.size == index
+
+        if tag_name && (((indent < orig_indent && !empty) || done ||
+            (indent == 0 && !empty)) || (indent <= last_indent && line =~ META_MATCH))
+          buf = tag_buf.join("\n")
+          if directive || tag_is_directive?(tag_name)
+            directive = create_directive(tag_name, buf)
+            if directive
+              docstring << parse_content(directive.expanded_text).chomp
+            end
+          else
+            create_tag(tag_name, buf)
+          end
+          tag_name, tag_buf, directive = nil, [], false
+          orig_indent = 0
+        end
+
+        # Found a meta tag
+        if line =~ META_MATCH
+          directive, tag_name, tag_buf = $1, $2, [($3 || '')]
+        elsif tag_name && indent >= orig_indent && !empty
+          orig_indent = indent if orig_indent == 0
+          # Extra data added to the tag on the next line
+          last_empty = last_line =~ /^[ \t]*$/ ? true : false
+
+          tag_buf << '' if last_empty
+          tag_buf << line.gsub(/^[ \t]{#{orig_indent}}/, '')
+        elsif !tag_name
+          # Regular docstring text
+          docstring << line << "\n"
+        end
+
+        last_indent = indent
+        last_line = line
+      end
+
+      docstring
+    end
+
+    private
+
+    # Creates a tag from the {Tags::DefaultFactory tag factory}.
+    #
+    # To add an already created tag object, use {#add_tag}
+    #
+    # @param [String] tag_name the tag name
+    # @param [String] tag_buf the text attached to the tag with newlines removed.
+    # @return [Tags::Tag, Tags::RefTag] a tag
+    def create_tag(tag_name, tag_buf)
+      if tag_buf =~ /\A\s*(?:(\S+)\s+)?\(\s*see\s+(\S+)\s*\)\s*\Z/
+        return create_ref_tag(tag_name, $1, $2)
+      end
+
+      if library.has_tag?(tag_name)
+        @tags += [library.tag_create(tag_name, tag_buf)].flatten
+      else
+        log.warn "Unknown tag @#{tag_name}" +
+          (object ? " in file `#{object.file}` near line #{object.line}" : "")
+      end
+    rescue Tags::TagFormatError
+      log.warn "Invalid tag format for @#{tag_name}" +
+        (object ? " in file `#{object.file}` near line #{object.line}" : "")
+    end
+
+    # Creates a {Tags::RefTag}
+    def create_ref_tag(tag_name, name, object_name)
+      @tags << Tags::RefTagList.new(tag_name, P(object, object_name), name)
+    end
+
+    # Creates a new directive using the registered {#library}
+    # @return [Directive] the directive object that is created
+    def create_directive(tag_name, tag_buf)
+      if library.has_directive?(tag_name)
+        dir = library.directive_create(tag_name, tag_buf, self)
+        if dir.is_a?(Tags::Directive)
+          @directives << dir
+          dir
+        end
+      else
+        log.warn "Unknown directive @!#{tag_name}" +
+          (object ? " in file `#{object.file}` near line #{object.line}" : "")
+        nil
+      end
+    rescue Tags::TagFormatError
+      log.warn "Invalid directive format for @!#{tag_name}" +
+        (object ? " in file `#{object.file}` near line #{object.line}" : "")
+      nil
+    end
+
+    # Calls the {Directive#after_parse} callback on all the
+    # created directives.
+    def call_directives_after_parse
+      directives.each do |dir|
+        dir.after_parse
+      end
+    end
+
+    # Backward compatibility to detect old tags that should be specified
+    # as directives in 0.8 and onward.
+    def tag_is_directive?(tag_name)
+      list = %w(attribute endgroup group macro method scope visibility)
+      list.include?(tag_name)
+    end
+
+    # Calls all {after_parse} callbacks
+    def call_after_parse_callbacks
+      self.class.after_parse_callbacks.each do |cb|
+        cb.call(self)
+      end
+    end
+
+    # Define a callback to check that @param tags are properly named
+    after_parse do |parser|
+      next unless parser.object
+      next unless parser.object.is_a?(CodeObjects::MethodObject)
+      next if parser.object.parameters.empty? # method has no params or
+                                                # YARD couldn't detect any.
+                                                # but don't warn user (?)
+      names = parser.object.parameters.map {|l| l.first.gsub(/\W/, '') }
+      seen_names = []
+      infile_info = "\n    in file `#{parser.object.file}' " +
+                    "near line #{parser.object.line}"
+      parser.tags.each do |tag|
+        next if tag.is_a?(Tags::RefTagList) # we don't handle this yet
+        next unless tag.tag_name == "param"
+        if names.include?(tag.name)
+          seen_names << tag.name
+        elsif seen_names.include?(tag.name)
+          log.warn "@param tag has duplicate parameter name: " +
+            "#{tag.name} #{infile_info}"
+        else
+          log.warn "@param tag has unknown parameter name: " +
+            "#{tag.name} #{infile_info}"
+        end
+      end
+    end
+  end
+end