yard 0.2.1 → 0.2.2

data/lib/yard/parser/source_parser.rb

@@ -0,0 +1,109 @@
+require 'stringio'
+
+module YARD
+  module Parser
+    class LoadOrderError < Exception; end
+    
+    # Responsible for parsing a source file into the namespace
+    class SourceParser 
+      class << self
+        def parse(paths = "lib/**/*.rb", level = log.level)
+          if paths.is_a?(Array)
+            files = paths.map {|p| Dir[p] }.flatten
+          else
+            files = Dir[File.join(Dir.pwd, paths)]
+          end
+
+          log.enter_level(level) do
+            parse_in_order(*files.uniq)
+          end
+        end
+      
+        def parse_string(content)
+          new.parse(StringIO.new(content))
+        end
+
+        private
+        
+        def parse_in_order(*files)
+          while file = files.shift
+            begin
+              if file.is_a?(Array) && file.last.is_a?(Continuation)
+                log.debug("Re-processing #{file.first}")
+                file.last.call
+              elsif file.is_a?(String)
+                log.debug("Processing #{file}...")
+                new(true).parse(file)
+              end
+            rescue LoadOrderError => e
+              # Out of order file. Push the context to the end and we'll call it
+              files.push([file, e.message])
+            end
+          end
+        end
+      end
+
+      attr_reader :file
+      attr_accessor :namespace, :visibility, :scope, :owner, :load_order_errors
+
+      def initialize(load_order_errors = false)
+        @file = "<STDIN>"
+        @namespace = YARD::Registry.root
+        @visibility = :public
+        @scope = :instance
+        @owner = @namespace
+        @load_order_errors = load_order_errors
+      end
+
+      ##
+      # Creates a new SourceParser that parses a file and returns
+      # analysis information about it.
+      #
+      # @param [String, TokenList, StatementList, #read] content the source file to parse
+      def parse(content = __FILE__)
+        case content
+        when String
+          @file = content
+          statements = StatementList.new(IO.read(content))
+        when TokenList
+          statements = StatementList.new(content)
+        when StatementList
+          statements = content
+        else
+          if content.respond_to? :read
+            statements = StatementList.new(content.read)
+          else
+            raise ArgumentError, "Invalid argument for SourceParser::parse: #{content.inspect}:#{content.class}"
+          end
+        end
+
+        top_level_parse(statements)
+      end
+
+      private
+        def top_level_parse(statements)
+            statements.each do |stmt|
+              find_handlers(stmt).each do |handler| 
+                begin
+                  handler.new(self, stmt).process
+                rescue LoadOrderError => loaderr
+                  raise # Pass this up
+                rescue Handlers::UndocumentableError => undocerr
+                  log.warn "in #{handler.to_s}: Undocumentable #{undocerr.message}"
+                  log.warn "\tin file '#{file}':#{stmt.tokens.first.line_no}:\n\n" + stmt.inspect + "\n"
+                rescue => e
+                  log.error "Unhandled exception in #{handler.to_s}:"
+                  log.error "#{e.class.class_name}: #{e.message}"
+                  log.error "  in `#{file}`:#{stmt.tokens.first.line_no}:\n\n#{stmt.inspect}\n"
+                  log.error "Stack trace:" + e.backtrace[0..5].map {|x| "\n\t#{x}" }.join + "\n"
+                end
+              end
+            end
+        end
+
+        def find_handlers(stmt)
+          Handlers::Base.subclasses.find_all {|sub| sub.handles? stmt.tokens }
+        end
+    end
+  end
+end

data/lib/yard/parser/statement.rb

@@ -0,0 +1,36 @@
+module YARD
+  module Parser
+    class Statement 
+      attr_reader :tokens, :comments, :block
+
+      def initialize(tokens, block = nil, comments = nil)
+        @tokens = tokens
+        @block  = block
+        @comments = comments
+      end
+      
+      def inspect(lines = nil)
+        flineno = tokens.first.line_no
+        buf = [""]
+        tokens.each do |tk|
+          if tk.is_a?(RubyToken::TkNL)
+            buf.push ""
+          else
+            buf.last << tk.text
+          end
+        end
+        buf.map {|line| "\t#{flineno}: #{line}" }.join("\n")
+      end
+
+      private
+        def clean_tokens(tokens)
+          last_tk = nil
+          tokens.reject do |tk| 
+            tk.is_a?(RubyToken::TkNL) || 
+            (last_tk.is_a?(RubyToken::TkSPACE) && 
+            last_tk.class == tk.class) && last_tk = tk 
+          end
+        end
+    end
+  end
+end

data/lib/yard/parser/statement_list.rb

@@ -0,0 +1,167 @@
+module YARD
+  module Parser
+    class StatementList < Array
+      include RubyToken
+
+      # The following list of tokens will require a block to be opened 
+      # if used at the beginning of a statement.
+      OPEN_BLOCK_TOKENS = [TkCLASS, TkDEF, TkMODULE, TkUNTIL,
+                           TkIF, TkUNLESS, TkWHILE, TkFOR, TkCASE]
+      COLON_TOKENS = [TkUNTIL, TkIF, TkUNLESS, TkWHILE, TkCASE, TkWHEN]
+
+      ##
+      # Creates a new statement list
+      #
+      # @param [TokenList, String] content the tokens to create the list from
+      def initialize(content)
+        if content.is_a? TokenList
+          @tokens = content.dup
+        elsif content.is_a? String
+          @tokens = TokenList.new(content)
+        else 
+          raise ArgumentError, "Invalid content for StatementList: #{content.inspect}:#{content.class}"
+        end
+
+        parse_statements
+      end
+
+      private
+
+      def parse_statements
+        while stmt = next_statement do self << stmt end
+      end
+
+      # MUST REFACTOR THIS CODE
+      # WARNING WARNING WARNING             WARNING
+      # MUST REFACTOR THIS CODE                |
+      # OR CHILDREN WILL DIE                   V
+      # WARNING WARNING WARNING             WARNING
+      # THIS IS MEANT TO BE UGLY.
+      def next_statement
+        statement, block, comments = TokenList.new, nil, nil
+        stmt_number, level = 0, 0
+        new_statement, open_block = true, false
+        last_tk, last_ns_tk, before_last_tk = nil, nil, nil
+        open_parens = 0
+
+        while tk = @tokens.shift
+          #p tk.class
+          # !!!!!!!!!!!!!!!!!!!! REMOVED TkfLPAREN, TkfLBRACK
+          open_parens += 1 if [TkLPAREN, TkLBRACK].include? tk.class
+          open_parens -= 1 if [TkRPAREN, TkRBRACK].include?(tk.class) 
+      
+          #if open_parens < 0 || level < 0
+          #  STDERR.puts block.to_s + " TOKEN #{tk.inspect}"
+          #  exit
+          #end
+
+          # Get the initial comments
+          if statement.empty?
+            # Two new-lines in a row will destroy any comment blocks
+            if [TkCOMMENT].include?(tk.class)  && last_tk.class == TkNL && 
+              (before_last_tk && (before_last_tk.class == TkNL || before_last_tk.class == TkSPACE))
+              comments = nil
+            elsif tk.class == TkCOMMENT
+              # Remove the "#" and up to 1 space before the text
+              # Since, of course, the convention is to have "# text"
+              # and not "#text", which I deem ugly (you heard it here first)
+              comments ||= []
+              comments << tk.text.gsub(/^#+\s{0,1}/, '') 
+              comments.pop if comments.size == 1 && comments.first =~ /^\s*$/
+            end
+          end
+                
+          # Ignore any initial comments or whitespace
+          unless statement.empty? && [TkSPACE, TkNL, TkCOMMENT].include?(tk.class)
+            # Decrease if end or '}' is seen
+            level -= 1 if [TkEND, TkRBRACE].include?(tk.class)
+        
+            # If the level is greater than 0, add the code to the block text
+            # otherwise it's part of the statement text
+            if stmt_number > 0
+              #puts "Block of #{statement}"
+              #puts "#{stmt_number} #{tk.line_no} #{level} #{open_parens} #{tk.class.class_name} \t#{tk.text.inspect} #{tk.lex_state} #{open_block.inspect}" 
+              block ||= TokenList.new
+              block << tk
+            elsif stmt_number == 0 && tk.class != TkNL && tk.class != TkSEMICOLON && tk.class != TkCOMMENT
+              statement << tk 
+            end
+
+            #puts "#{tk.line_no} #{level} #{open_parens} #{tk.class.class_name} \t#{tk.text.inspect} #{tk.lex_state} #{open_block.inspect}" 
+
+            # Increase level if we have a 'do' or block opening
+            if tk.class == TkLBRACE #|| tk.class == TkfLBRACE
+              level += 1    
+            elsif [TkDO, TkBEGIN].include?(tk.class) 
+              #p "#{tk.line_no} #{level} #{tk} \t#{tk.text} #{tk.lex_state}" 
+              level += 1    
+              open_block = false  # Cancel our wish to open a block for the if, we're doing it now
+            end
+
+            # Vouch to open a block when this statement would otherwise end
+            open_block = [level, tk.class] if (new_statement || 
+                                (last_tk && last_tk.lex_state == EXPR_BEG)) && 
+                                 OPEN_BLOCK_TOKENS.include?(tk.class)
+
+            # Check if this token creates a new statement or not
+            #puts "#{open_parens} open brackets for: #{statement.to_s}"
+            if open_parens == 0 && ((last_tk && [TkSEMICOLON, TkNL, TkEND_OF_SCRIPT].include?(tk.class)) ||
+              (open_block && open_block.last == TkDEF && tk.class == TkRPAREN))
+              
+              # Make sure we don't have any running expressions
+              # This includes things like
+              #
+              # class <
+              #   Foo
+              # 
+              # if a ||
+              #    b
+              if (last_tk && [EXPR_END, EXPR_ARG].include?(last_tk.lex_state)) || 
+                  (open_block && [TkNL, TkSEMICOLON].include?(tk.class) && last_ns_tk.class != open_block.last)
+                stmt_number += 1
+                new_statement = true
+                #p "NEW STATEMENT #{block.to_s}"
+
+                # The statement started with a if/while/begin, so we must go to the next level now
+                if open_block && open_block.first == level
+                  if tk.class == TkNL && block.nil?
+                    block = TokenList.new
+                    block << tk
+                  end
+
+                  open_block = false
+                  level += 1
+                end
+              end
+            elsif tk.class != TkSPACE
+              new_statement = false 
+            end
+
+            # Else keyword is kind of weird
+            if tk.is_a? TkELSE
+              new_statement = true
+              stmt_number += 1
+              open_block = false
+            end
+
+            # We're done if we've ended a statement and we're at level 0
+            break if new_statement && level == 0
+        
+            #raise "Unexpected end" if level < 0
+          end
+      
+          #break if new_statement && level == 0
+
+          before_last_tk = last_tk
+          last_tk = tk # Save last token
+          last_ns_tk = tk unless [TkSPACE, TkNL, TkEND_OF_SCRIPT].include? tk.class
+        end
+
+        # Return the code block with starting token and initial comments
+        # If there is no code in the block, return nil
+        comments = comments.compact if comments
+        statement.empty? ? nil : Statement.new(statement, block, comments)
+      end
+    end
+  end
+end

data/lib/yard/parser/token_list.rb

@@ -0,0 +1,58 @@
+module YARD
+  module Parser
+    class TokenList < Array
+      include RubyToken
+      
+      def initialize(content = nil)
+        self << content if content
+      end
+      
+      def to_s
+        collect {|t| t.text }.join
+      end
+      
+      # @param [TokenList, Token, String] tokens
+      #   A list of tokens. If the token is a string, it
+      #   is parsed with {RubyLex}.
+      def push(*tokens)
+        tokens.each do |tok|
+          if tok.is_a?(TokenList) || tok.is_a?(Array)
+            concat tok
+          elsif tok.is_a?(Token)
+            super tok
+          elsif tok.is_a?(String)
+            parse_content(tok)
+          else
+            raise ArgumentError, "Expecting token, list of tokens or string of code to be tokenized. Got #{tok.class}"
+          end
+        end
+        self
+      end
+      alias_method :<<, :push
+      
+      def squeeze(type = TkSPACE)
+        last = nil
+        TokenList.new(map {|t| x = t.is_a?(type) && last.is_a?(type) ? nil : t; last = t; x })
+      end
+      
+      private
+      
+      def parse_content(content)
+        lex = RubyLex.new(content)
+        while tk = lex.token do 
+          self << convert_token(lex, tk)
+        end
+      end
+      
+      def convert_token(lex, tk)
+        if TkSYMBEG === tk && next_tk = lex.token
+          sym = TkSYMBOL.new(tk.line_no, tk.char_no, nil)
+          sym.lex_state = lex.lex_state
+          sym.set_text(tk.text + next_tk.text)
+        else
+          tk 
+        end
+      end
+    end
+  end
+end

data/lib/yard/rake/yardoc_task.rb

@@ -0,0 +1,30 @@
+require 'rake'
+require 'rake/tasklib'
+
+module YARD
+  module Rake
+    
+    class YardocTask < ::Rake::TaskLib
+      attr_accessor :name
+      attr_accessor :options
+      attr_accessor :files
+
+      def initialize(name = :yardoc)
+        @name = name
+        @options = []
+        @files = []
+        
+        yield self if block_given?
+        self.options +=  ENV['OPTS'].split(/[ ,]/) if ENV['OPTS'] 
+        self.files   += ENV['FILES'].split(/[ ,]/) if ENV['FILES']
+        
+        define
+      end
+      
+      def define
+        desc "Generate YARD Documentation"
+        task(name) { YARD::CLI::Yardoc.run *(options + files) }
+      end
+    end
+  end
+end

data/lib/yard/registry.rb

@@ -0,0 +1,136 @@
+require 'singleton'
+require 'find'
+
+module YARD
+  class Registry 
+    DEFAULT_YARDOC_FILE = ".yardoc"
+    
+    include Singleton
+  
+    @objects = {}
+
+    class << self
+      attr_reader :objects
+
+      def method_missing(meth, *args, &block)
+        if instance.respond_to? meth
+          instance.send(meth, *args, &block)
+        else
+          super
+        end
+      end
+      
+      def clear
+        instance.clear 
+        objects.clear
+      end
+    end
+
+    attr_accessor :yardoc_file
+    attr_reader :proxy_types
+    
+    def load(files = [], reload = false)
+      if files.is_a?(Array)
+        if File.exists?(yardoc_file) && !reload
+          load_yardoc
+        else
+          size = namespace.size
+          YARD.parse(files)
+          save if namespace.size > size
+        end
+        true
+      elsif files.is_a?(String)
+        load_yardoc(files)
+        true
+      else
+        raise ArgumentError, "Must take a list of files to parse or the .yardoc file to load."
+      end
+    end
+    
+    def load_yardoc(file = yardoc_file)
+      return false unless File.exists?(file)
+      ns, pt = *Marshal.load(IO.read(file))
+      namespace.update(ns)
+      proxy_types.update(pt)
+    end
+    
+    def save(file = yardoc_file)
+      File.open(file, "w") {|f| Marshal.dump([@namespace, @proxy_types], f) }
+      true
+    end
+
+    def all(*types)
+      namespace.values.select do |obj| 
+        if types.empty?
+          obj != Registry.root
+        else
+          obj != Registry.root &&
+            types.any? do |type| 
+              type.is_a?(Symbol) ? obj.type == type : obj.is_a?(type)
+            end
+        end
+      end
+    end
+    
+    def paths
+      namespace.keys.map {|k| k.to_s }
+    end
+      
+    def at(path) path.to_s.empty? ? root : namespace[path] end
+    alias_method :[], :at
+    
+    def root; namespace[:root] end
+    def delete(object) namespace.delete(object.path) end
+
+    def clear
+      @namespace = SymbolHash.new
+      @namespace[:root] = CodeObjects::RootObject.new(nil, :root)
+      @proxy_types = {}
+    end
+
+    def initialize
+      @yardoc_file = DEFAULT_YARDOC_FILE
+      clear
+    end
+  
+    def register(object)
+      return if object.is_a?(CodeObjects::Proxy)
+      namespace[object.path] = object
+    end
+
+    def resolve(namespace, name, proxy_fallback = false)
+      namespace = root if namespace == :root || !namespace
+
+      newname = name.to_s.gsub(/^#{CodeObjects::ISEP}/, '')
+      if name =~ /^#{CodeObjects::NSEP}/
+        [name, newname[2..-1]].each do |n|
+          return at(n) if at(n)
+        end
+      else
+        while namespace
+          [CodeObjects::NSEP, CodeObjects::ISEP].each do |s|
+            path = newname
+            if namespace != root
+              path = [namespace.path, newname].join(s)
+            end
+            found = at(path)
+            return found if found
+          end
+          namespace = namespace.parent
+        end
+
+        # Look for ::name or #name in the root space
+        [CodeObjects::NSEP, CodeObjects::ISEP].each do |s|
+          found = at(s + newname)
+          return found if found
+        end
+      end
+      proxy_fallback ? CodeObjects::Proxy.new(namespace, name) : nil
+    end
+
+    private
+  
+    attr_accessor :namespace
+    
+  end
+end