yard 0.2.0

data/lib/source_parser.rb

@@ -0,0 +1,246 @@
+require 'stringio'
+require File.dirname(__FILE__) + '/ruby_lex' 
+require File.dirname(__FILE__) + '/namespace'
+require File.dirname(__FILE__) + '/code_object'
+require File.dirname(__FILE__) + '/handlers/all_handlers'
+
+module YARD
+  ##
+  # Responsible for parsing a source file into the namespace
+  class SourceParser 
+    attr_reader :file
+    
+    def self.parse(content)
+      new.parse(content)
+    end
+    
+    def self.parse_string(content)
+      new.parse(StringIO.new(content))
+    end
+    
+    attr_accessor :current_namespace
+    
+    def initialize
+      @current_namespace = NameStruct.new(Namespace.root)
+    end
+    
+    ##
+    # Creates a new SourceParser that parses a file and returns
+    # analysis information about it.
+    #
+    # @param [String, TokenList, StatementList] content the source file to parse
+    def parse(content = __FILE__)
+      case content
+      when String
+        @file = content
+        statements = StatementList.new(IO.read(content))
+      when TokenList
+        statements = StatementList.new(content)
+      when StatementList
+        statements = content
+      else
+        if content.respond_to? :read
+          statements = StatementList.new(content.read)
+        else
+          raise ArgumentError, "Invalid argument for SourceParser::parse: #{content.inspect}:#{content.class}"
+        end
+      end
+      
+      top_level_parse(statements)
+    end
+    
+    private
+      def top_level_parse(statements)
+        statements.each do |stmt|
+          find_handlers(stmt).each do |handler| 
+            handler.new(self, stmt).process
+          end
+        end
+      end
+    
+      def find_handlers(stmt)
+        CodeObjectHandler.subclasses.find_all {|sub| sub.handles? stmt.tokens }
+      end
+  end
+  
+  class StatementList < Array
+    include RubyToken
+
+    # The following list of tokens will require a block to be opened 
+    # if used at the beginning of a statement.
+    @@open_block_tokens = [TkCLASS, TkDEF, TkMODULE, TkUNTIL,
+                           TkIF, TkUNLESS, TkWHILE, TkFOR, TkCASE]
+
+    ##
+    # Creates a new statement list
+    #
+    # @param [TokenList, String] content the tokens to create the list from
+    def initialize(content)
+      if content.is_a? TokenList
+        @tokens = content
+      elsif content.is_a? String
+        parse_tokens(content)
+      else 
+        raise ArgumentError, "Invalid content for StatementList: #{content.inspect}:#{content.class}"
+      end
+
+      parse_statements
+    end
+
+    private
+      def parse_tokens(content)
+        @tokens = TokenList.new
+        lex = RubyLex.new(content)
+        while tk = lex.token do @tokens << tk end
+      end
+
+      def parse_statements
+        while stmt = next_statement do self << stmt end
+      end
+
+      # MUST REFACTOR THIS CODE
+      # WARNING WARNING WARNING             WARNING
+      # MUST REFACTOR THIS CODE                |
+      # OR CHILDREN WILL DIE                   V
+      # WARNING WARNING WARNING             WARNING
+      # THIS IS MEANT TO BE UGLY.
+      def next_statement
+        statement, block, comments = TokenList.new, nil, nil
+        stmt_number, level = 0, 0
+        new_statement, open_block = true, false
+        last_tk, before_last_tk = nil, nil
+        open_parens = 0
+
+        while tk = @tokens.shift
+          #p tk.class
+          open_parens += 1 if [TkLPAREN, TkLBRACK].include? tk.class
+          open_parens -= 1 if [TkRPAREN, TkRBRACK].include?(tk.class) if open_parens > 0
+          
+#          raise block.to_s + " TOKEN #{tk.inspect}" if open_parens < 0
+
+          # Get the initial comments
+          if statement.empty?
+            # Two new-lines in a row will destroy any comment blocks
+            if tk.class == TkCOMMENT && last_tk.class == TkNL && 
+              (before_last_tk && (before_last_tk.class == TkNL || before_last_tk.class == TkSPACE))
+              comments = nil
+            elsif tk.class == TkCOMMENT
+              # Remove the "#" and up to 1 space before the text
+              # Since, of course, the convention is to have "# text"
+              # and not "#text", which I deem ugly (you heard it here first)
+              comments ||= []
+              comments << (tk.text[/^#+\s{0,1}(\s*[^\s#].+)/, 1] || "") 
+              comments.pop if comments.size == 1 && comments.first =~ /^\s*$/
+            end
+          end
+                    
+          # Ignore any initial comments or whitespace
+          unless statement.empty? && [TkSPACE, TkNL, TkCOMMENT].include?(tk.class)
+            # Decrease if end or '}' is seen
+            level -= 1 if [TkEND, TkRBRACE].include?(tk.class)
+
+            # If the level is greater than 0, add the code to the block text
+            # otherwise it's part of the statement text
+            if stmt_number > 0
+              block ||= TokenList.new
+              block << tk
+            elsif stmt_number == 0 && tk.class != TkNL && tk.class != TkCOMMENT
+              statement << tk 
+            end
+
+#            puts "#{tk.line_no} #{level} #{tk} \t#{tk.text} #{tk.lex_state}" 
+
+            # Increase level if we have a 'do' or block opening
+            if tk.class == TkLBRACE
+              level += 1    
+            elsif [TkDO, TkfLBRACE, TkBEGIN].include?(tk.class)
+              #p "#{tk.line_no} #{level} #{tk} \t#{tk.text} #{tk.lex_state}" 
+              level += 1    
+              open_block = false  # Cancel our wish to open a block for the if, we're doing it now
+            end
+
+            # Vouch to open a block when this statement would otherwise end
+            open_block = true if (new_statement || (last_tk && last_tk.lex_state == EXPR_BEG)) && @@open_block_tokens.include?(tk.class)
+
+            # Check if this token creates a new statement or not
+            #puts "#{open_parens} open brackets for: #{statement.to_s}"
+            if open_parens == 0 && ([TkSEMICOLON, TkNL, TkEND_OF_SCRIPT].include?(tk.class) ||
+              (statement.first.class == TkDEF && tk.class == TkRPAREN))
+              # Make sure we don't have any running expressions
+              # This includes things like
+              #
+              # class <
+              #   Foo
+              # 
+              # if a ||
+              #    b
+              if [EXPR_END, EXPR_ARG].include? last_tk.lex_state
+                stmt_number += 1
+                new_statement = true
+                #p "NEW STATEMENT #{statement.to_s}"
+
+                # The statement started with a if/while/begin, so we must go to the next level now
+                if open_block
+                  open_block = false
+                  level += 1
+                end
+              end
+            elsif tk.class != TkSPACE
+              new_statement = false 
+            end
+
+            # Else keyword is kind of weird
+            if tk.is_a? RubyToken::TkELSE
+              new_statement = true
+              stmt_number += 1
+              open_block = false
+            end
+
+            # We're done if we've ended a statement and we're at level 0
+            break if new_statement && level == 0
+          end
+
+          before_last_tk = last_tk
+          last_tk = tk # Save last token
+        end
+
+        # Return the code block with starting token and initial comments
+        # If there is no code in the block, return nil
+        comments = comments.compact if comments
+        statement.empty? ? nil : Statement.new(statement, block, comments)
+      end
+  end
+
+  class TokenList < Array
+    def to_s
+      collect {|t| t.text }.join
+    end
+  end
+
+  class Statement 
+    attr_reader :tokens, :comments, :block
+
+    def initialize(tokens, block = nil, comments = nil)
+      @tokens = clean_tokens(tokens)
+      @block  = block
+      @comments = comments
+    end
+
+    private
+      def clean_tokens(tokens)
+        last_tk = nil
+        tokens.reject do |tk| 
+          tk.is_a?(RubyToken::TkNL) || 
+          (last_tk.is_a?(RubyToken::TkSPACE) && 
+          last_tk.class == tk.class) && last_tk = tk 
+        end
+      end
+  end  
+  
+  class NameStruct
+    attr_accessor :object, :attributes
+    def initialize(object)
+      @object, @attributes = object, { :visibility => :public, :scope => :instance }
+    end
+  end
+end

data/lib/tag_library.rb

@@ -0,0 +1,162 @@
+module YARD
+  ##
+  # Holds all the registered meta tags. If you want to extend YARD and add
+  # a new meta tag, you can do it in one of two ways.
+  #
+  # == Method #1
+  # Write your own +tagname_tag+ method that takes the raw text as a parameter.
+  # Example:
+  #   def mytag_tag(text)
+  #     Tag.parse_tag("mytag", text)
+  #   end
+  #
+  # This will allow you to use @mytag TEXT to add meta data to classes through
+  # the docstring. {Tag} has a few convenience factory methods to create 
+  #
+  # == Method #2
+  # Use {TagLibrary::define_tag!} to define a new tag by passing the tag name
+  # and the factory method to use when creating the tag. These definitions will
+  # be auto expanded into ruby code similar to what is shown in method #1. If you
+  # do not provide a factory method to use, it will default to {Tag::parse_tag}
+  # Example:
+  #   define_tag! :param, :with_types_and_name
+  #   define_tag! :author
+  #
+  # The first line will expand to the code:
+  #   def param_tag(text) Tag.parse_tag_with_types_and_name(text) end
+  #
+  # The second line will expand to:
+  #   def author_tag(text) Tag.parse_tag(text) end
+  #
+  # @see TagLibrary::define_tag!
+  module TagLibrary
+    class << self
+      ##
+      # Convenience method to define a new tag using one of {Tag}'s factory methods, or the
+      # regular {Tag::parse_tag} factory method if none is supplied.
+      #
+      # @param tag the tag name to create
+      # @param meth the {Tag} factory method to call when creating the tag
+      def self.define_tag!(tag, meth = "")
+        meth = meth.to_s
+        send_name = meth.empty? ? "" : "_" + meth
+        class_eval "def #{tag}_tag(text) Tag.parse_tag#{send_name}(#{tag.inspect}, text) end"
+      end
+      
+      define_tag! :param, :with_types_and_name
+      define_tag! :yieldparam, :with_types_and_name
+      define_tag! :yield
+      define_tag! :return, :with_types
+      define_tag! :deprecated
+      define_tag! :author
+      define_tag! :raise, :with_name
+      define_tag! :see
+      define_tag! :since
+      define_tag! :version
+    end
+  end
+  
+  class Tag
+    attr_reader :tag_name, :text, :types, :name
+    
+    class << self
+      ##
+      # Parses tag text and creates a new tag with descriptive text
+      #
+      # @param tag_name        the name of the tag to parse
+      # @param [String] text   the raw tag text
+      # @return [Tag]          a tag object with the tag_name and text values filled
+      def parse_tag(tag_name, text)
+        new(tag_name, text)
+      end
+      
+      ##
+      # Parses tag text and creates a new tag with a key name and descriptive text
+      #
+      # @param tag_name        the name of the tag to parse
+      # @param [String] text   the raw tag text
+      # @return [Tag]          a tag object with the tag_name, name and text values filled
+      def parse_tag_with_name(tag_name, text)
+        name, text = *extract_name_from_text(text)
+        new(tag_name, text, nil, name)
+      end
+      
+      ##
+      # Parses tag text and creates a new tag with formally declared types and 
+      # descriptive text
+      #
+      # @param tag_name        the name of the tag to parse
+      # @param [String] text   the raw tag text
+      # @return [Tag]          a tag object with the tag_name, types and text values filled
+      def parse_tag_with_types(tag_name, text)
+        types, text = *extract_types_from_text(text)
+        new(tag_name, text, types)
+      end
+      
+      ##
+      # Parses tag text and creates a new tag with formally declared types, a key 
+      # name and descriptive text
+      #
+      # @param tag_name        the name of the tag to parse
+      # @param [String] text   the raw tag text
+      # @return [Tag]          a tag object with the tag_name, name, types and text values filled
+      def parse_tag_with_types_and_name(tag_name, text)
+        types, text = *extract_types_from_text(text)
+        name, text = *extract_name_from_text(text)
+        new(tag_name, text, types, name)
+      end
+      
+      ##
+      # Extracts the name from raw tag text returning the name and remaining value
+      #
+      # @param [String] text the raw tag text
+      # @return [Array] an array holding the name as the first element and the 
+      #                 value as the second element
+      def extract_name_from_text(text)
+        text.strip.split(" ", 2)
+      end
+      
+      ##
+      # Extracts the type signatures from the raw tag text
+      #
+      # @param [String] text the raw tag text
+      # @return [Array] an array holding the value as the first element and
+      #                 the array of types as the second element
+      def extract_types_from_text(text)
+        types, text = [], text.strip
+        if text =~ /^\s*\[(.+?)\]\s*(.*)/
+          text, types = $2, $1.split(",").collect {|e| e.strip }
+        end
+        [types, text]
+      end
+    end
+    
+    ##
+    # Creates a new tag object with a tag name and text. Optionally, formally declared types
+    # and a key name can be specified.
+    #
+    # Types are mainly for meta tags that rely on type information, such as +param+, +return+, etc.
+    # 
+    # Key names are for tags that declare meta data for a specific key or name, such as +param+,
+    # +raise+, etc.
+    #
+    # @param tag_name                the tag name to create the tag for
+    # @param [String] text           the descriptive text for this tag
+    # @param [Array<String>] types   optional type list of formally declared types
+    #                                for the tag
+    # @param [String] name           optional key name which the tag refers to
+    def initialize(tag_name, text, types = nil, name = nil)
+      @tag_name, @text, @types, @name = tag_name.to_s, text, types, name
+    end
+    
+    ##
+    # Convenience method to access the first type specified. This should mainly
+    # be used for tags that only specify one type.
+    #
+    # @see #types
+    # @return (String) the first of the list of specified types 
+    def type
+      types.first
+    end
+  end
+end

data/lib/tag_type.rb

@@ -0,0 +1,155 @@
+module YARD
+  ## 
+  # Represents a tag and its respective tag name and value. 
+  #
+  # @abstract   Override this class to using the class name format of
+  #             'tagnameTag' to add a new tag to the registered tag library
+  class BaseTag
+    class << self
+      ##
+      # Returns the tag name that the class responds to. If the class name
+      # does not accurately represent the tag name, this method should be overrided
+      # by subclasses to return the correct tag name.
+      # 
+      # @return [String] the tag name the class represents
+      def tag_name
+        @tag_name ||= self.to_s.split("::").last.gsub(/^Base.+|Tag$/, '').downcase
+      end
+
+      ##
+      # Return all valid tags in the tag library
+      # 
+      # @return [Array<String>] a list of valid tags that are registered
+      #                         as being handled
+      def tag_library
+        @@tag_library || {}
+      end
+    
+      ##
+      # @override
+      def inherited(subclass)
+        @@tag_library ||= {}
+        @@tag_library[subclass.tag_name] = subclass unless subclass.tag_name.empty?
+      end
+      
+      ##
+      # Parses tag text from a doc string and returns a new tag 
+      def parse_tag(text)
+        new(text)
+      end
+    end
+
+    ## All tags have an optional field for text data 
+    attr_reader :text
+    
+    def initialize(text)
+      @text = text
+    end
+
+    ## 
+    # @see BaseTag::tag_name
+    def tag_name
+      self.class.tag_name
+    end
+  end
+  
+  ##
+  # Similar to the {BaseTag}, this class allows for a tag to
+  # formally specify type data if it depends on a specific object type.
+  #
+  class BaseTypeTag < BaseTag
+    ##
+    # Attribute reader for all specified types
+    #
+    # @return [String] the object types that the tag formally specifies
+    attr_reader :types
+    
+    ##
+    # Extracts the type signatures from the raw tag text
+    #
+    # @param [String] text the raw tag text
+    # @return [Array] an array holding the value as the first element and
+    #                 the array of types as the second element
+    def self.extract_types_and_text(text)
+      types, value = [], text.strip
+      if text =~ /^\s*\[(.+?)\]\s*(.*)/
+        value = $2
+        types = $1.split(",").collect {|e| e.strip }
+      end
+      [value, types]
+    end
+    
+    ##
+    # Extracts the name from raw tag text returning the name and remaining value
+    #
+    # @param [String] text the raw tag text
+    # @return [Array] an array holding the name as the first element and the 
+    #                 value as the second element
+    def self.extract_name_and_text(text)
+      text.strip.split(" ", 2)
+    end
+    
+    ## 
+    # Create a new object with formally specified types
+    #
+    # @param [String] text the text data
+    # @param [Array<String>] types the types that are formally specified
+    #                        by the tag.
+    def initialize(text, types = [])
+      super(text)
+      @types = types || []
+    end
+    
+    ##
+    # Convenience method to access the first type specified. This should mainly
+    # be used for tags that only specify one type.
+    #
+    # @return (String) the first of the list of specified types 
+    def type
+      types.first
+    end
+  end
+  
+  class ParamTag < BaseTypeTag
+    attr_reader :name
+    
+    ##
+    # Parses a typed tag's value section and returns a new {ParamTag} object
+    #
+    # @param [String] text the raw tag value to parse type specifications from
+    # @return [ParamTag] the new typed tag object with the type values 
+    #                    parsed from raw text
+    def self.parse_tag(text)
+      desc, types = *extract_types_and_text(text)
+      name, desc = *extract_name_and_text(desc)
+      new(name, desc, types)
+    end
+    
+    ##
+    # Create a new +param+ tag with a description and declared types
+    #
+    # @param [String] name the parameter name specified by the tag
+    # @param [String] text a description of the parameter 
+    # @param [Array<String>] types the optional types that the parameter accepts
+    def initialize(name, text, types = [])
+      super(text, types)
+      @name = name
+    end
+  end
+
+  class ReturnTag < BaseTypeTag
+    def self.parse_tag(text)
+      new *extract_types_and_text(text) 
+    end
+    
+    def initialize(text, types = []) super end
+  end
+  
+  class DeprecatedTag < BaseTag
+    def initialize(text) super end
+  end
+  
+  class AuthorTag < BaseTag
+    def initialize(text) super end
+  end  
+end