yard 0.2.2 → 0.2.3

data/lib/yard/parser/source_parser.rb

@@ -1,13 +1,22 @@
 require 'stringio'
+require 'continuation' unless RUBY18
 
 module YARD
   module Parser
+    class UndocumentableError < Exception; end
     class LoadOrderError < Exception; end
     
     # Responsible for parsing a source file into the namespace
     class SourceParser 
       class << self
+        attr_accessor :parser_type
+        
+        def parser_type=(value)
+          @parser_type = validated_parser_type(value)
+        end
+        
         def parse(paths = "lib/**/*.rb", level = log.level)
+          log.debug("Parsing #{paths} with `#{parser_type}` parser")
           if paths.is_a?(Array)
             files = paths.map {|p| Dir[p] }.flatten
           else
@@ -19,8 +28,16 @@ module YARD
           end
         end
       
-        def parse_string(content)
-          new.parse(StringIO.new(content))
+        def parse_string(content, ptype = parser_type)
+          new(ptype).parse(StringIO.new(content))
+        end
+        
+        def tokenize(content, ptype = parser_type)
+          new(ptype).tokenize(content)
+        end
+        
+        def validated_parser_type(type)
+          RUBY18 && type == :ruby ? :ruby18 : type
         end
 
         private
@@ -33,7 +50,7 @@ module YARD
                 file.last.call
               elsif file.is_a?(String)
                 log.debug("Processing #{file}...")
-                new(true).parse(file)
+                new(parser_type, true).parse(file)
               end
             rescue LoadOrderError => e
               # Out of order file. Push the context to the end and we'll call it
@@ -43,67 +60,81 @@ module YARD
         end
       end
 
-      attr_reader :file
-      attr_accessor :namespace, :visibility, :scope, :owner, :load_order_errors
+      self.parser_type = :ruby
+      
+      attr_reader :file, :parser_type
 
-      def initialize(load_order_errors = false)
-        @file = "<STDIN>"
-        @namespace = YARD::Registry.root
-        @visibility = :public
-        @scope = :instance
-        @owner = @namespace
+      def initialize(parser_type = SourceParser.parser_type, load_order_errors = false)
         @load_order_errors = load_order_errors
+        @file = '(stdin)'
+        self.parser_type = parser_type
       end
 
       ##
       # Creates a new SourceParser that parses a file and returns
       # analysis information about it.
       #
-      # @param [String, TokenList, StatementList, #read] content the source file to parse
+      # @param [String, #read, Object] content the source file to parse
       def parse(content = __FILE__)
         case content
         when String
           @file = content
-          statements = StatementList.new(IO.read(content))
-        when TokenList
-          statements = StatementList.new(content)
-        when StatementList
-          statements = content
+          content = IO.read(content)
+          self.parser_type = parser_type_for_filename(file)
         else
-          if content.respond_to? :read
-            statements = StatementList.new(content.read)
-          else
-            raise ArgumentError, "Invalid argument for SourceParser::parse: #{content.inspect}:#{content.class}"
-          end
+          content = content.read if content.respond_to? :read
+        end
+        
+        @parser = parse_statements(content)
+        post_process
+        @parser
+      end
+      
+      def tokenize(content)
+        case parser_type
+        when :c
+          raise NotImplementedError, "no support for C/C++ files"
+        when :ruby18
+          Ruby::Legacy::TokenList.new(content)
+        when :ruby
+          Ruby::RubyParser.parse(content).tokens
+        else
+          raise ArgumentError, "invalid parser type or unrecognized file"
         end
+      end
+      
+      private
 
-        top_level_parse(statements)
+      def post_process
+        post = Handlers::Processor.new(@file, @load_order_errors, @parser_type)
+        post.process(@parser.enumerator)
       end
 
-      private
-        def top_level_parse(statements)
-            statements.each do |stmt|
-              find_handlers(stmt).each do |handler| 
-                begin
-                  handler.new(self, stmt).process
-                rescue LoadOrderError => loaderr
-                  raise # Pass this up
-                rescue Handlers::UndocumentableError => undocerr
-                  log.warn "in #{handler.to_s}: Undocumentable #{undocerr.message}"
-                  log.warn "\tin file '#{file}':#{stmt.tokens.first.line_no}:\n\n" + stmt.inspect + "\n"
-                rescue => e
-                  log.error "Unhandled exception in #{handler.to_s}:"
-                  log.error "#{e.class.class_name}: #{e.message}"
-                  log.error "  in `#{file}`:#{stmt.tokens.first.line_no}:\n\n#{stmt.inspect}\n"
-                  log.error "Stack trace:" + e.backtrace[0..5].map {|x| "\n\t#{x}" }.join + "\n"
-                end
-              end
-            end
+      def parser_type=(value)
+        @parser_type = self.class.validated_parser_type(value)
+      end
+      
+      def parser_type_for_filename(filename)
+        case (File.extname(filename)[1..-1] || "").downcase
+        when "c", "cpp", "cxx"
+          :c
+        else # when "rb", "rbx", "erb"
+          parser_type == :ruby18 ? :ruby18 : :ruby
         end
-
-        def find_handlers(stmt)
-          Handlers::Base.subclasses.find_all {|sub| sub.handles? stmt.tokens }
+      end
+      
+      def parse_statements(content)
+        case parser_type
+        when :c
+          raise NotImplementedError, "no support for C/C++ files"
+        when :ruby18
+          Ruby::Legacy::StatementList.new(content)
+        when :ruby
+          Ruby::RubyParser.parse(content, file)
+        else
+          raise ArgumentError, "invalid parser type or unrecognized file"
         end
+      end
     end
   end
 end

data/lib/yard/rake/yardoc_task.rb

@@ -8,6 +8,7 @@ module YARD
       attr_accessor :name
       attr_accessor :options
       attr_accessor :files
+      attr_accessor :before, :after
 
       def initialize(name = :yardoc)
         @name = name
@@ -23,7 +24,11 @@ module YARD
       
       def define
         desc "Generate YARD Documentation"
-        task(name) { YARD::CLI::Yardoc.run *(options + files) }
+        task(name) do
+          before.call if before.is_a?(Proc)
+          YARD::CLI::Yardoc.run *(options + files) 
+          after.call if after.is_a?(Proc)
+        end
       end
     end
   end

data/lib/yard/registry.rb

@@ -62,14 +62,14 @@ module YARD
     def all(*types)
       namespace.values.select do |obj| 
         if types.empty?
-          obj != Registry.root
+          obj != root
         else
-          obj != Registry.root &&
+          obj != root &&
             types.any? do |type| 
               type.is_a?(Symbol) ? obj.type == type : obj.is_a?(type)
             end
         end
-      end
+      end + (types.include?(:root) ? [root] : [])
     end
     
     def paths
@@ -80,7 +80,11 @@ module YARD
     alias_method :[], :at
     
     def root; namespace[:root] end
-    def delete(object) namespace.delete(object.path) end
+    
+    def delete(object) 
+      namespace.delete(object.path)
+      self.class.objects.delete(object.path)
+    end
 
     def clear
       @namespace = SymbolHash.new
@@ -94,43 +98,65 @@ module YARD
     end
   
     def register(object)
+      self.class.objects[object.path] = object
       return if object.is_a?(CodeObjects::Proxy)
       namespace[object.path] = object
     end
 
-    def resolve(namespace, name, proxy_fallback = false)
-      namespace = root if namespace == :root || !namespace
+    def resolve(namespace, name, inheritance = false, proxy_fallback = false)
+      if namespace.is_a?(CodeObjects::Proxy)
+        return proxy_fallback ? CodeObjects::Proxy.new(namespace, name) : nil
+      end
+      
+      if namespace == :root || !namespace
+        namespace = root
+      else
+        namespace = namespace.parent until namespace.is_a?(CodeObjects::NamespaceObject)
+      end
+      orignamespace = namespace
 
-      newname = name.to_s.gsub(/^#{CodeObjects::ISEP}/, '')
-      if name =~ /^#{CodeObjects::NSEP}/
-        [name, newname[2..-1]].each do |n|
+      name = name.to_s
+      if name =~ /^#{CodeObjects::NSEPQ}/
+        [name, name[2..-1]].each do |n|
           return at(n) if at(n)
         end
       else
         while namespace
-          [CodeObjects::NSEP, CodeObjects::ISEP].each do |s|
-            path = newname
-            if namespace != root
-              path = [namespace.path, newname].join(s)
+          if namespace.is_a?(CodeObjects::NamespaceObject)
+            nss = inheritance ? namespace.inheritance_tree(true) : [namespace]
+            nss.each do |ns|
+              next if ns.is_a?(CodeObjects::Proxy)
+              found = partial_resolve(ns, name)
+              return found if found
             end
-            found = at(path)
-            return found if found
           end
           namespace = namespace.parent
         end
 
         # Look for ::name or #name in the root space
-        [CodeObjects::NSEP, CodeObjects::ISEP].each do |s|
-          found = at(s + newname)
+        [CodeObjects::ISEP, CodeObjects::NSEP].each do |s|
+          found = at(s + name)
           return found if found
         end
       end
-      proxy_fallback ? CodeObjects::Proxy.new(namespace, name) : nil
+      proxy_fallback ? CodeObjects::Proxy.new(orignamespace, name) : nil
     end
 
     private
   
     attr_accessor :namespace
-    
+
+    def partial_resolve(namespace, name)
+      [CodeObjects::NSEP, CodeObjects::CSEP, ''].each do |s|
+        next if s.empty? && name =~ /^\w/
+        path = name
+        if namespace != root
+          path = [namespace.path, name].join(s)
+        end
+        found = at(path)
+        return found if found
+      end
+      nil
+    end
   end
 end

data/lib/yard/serializers/file_system_serializer.rb

@@ -21,7 +21,7 @@ module YARD
       def serialized_path(object)
         return object if object.is_a?(String)
 
-        objname = object.name.to_s
+        objname = object != YARD::Registry.root ? object.name.to_s : "top-level-namespace"
         objname += '_' + object.scope.to_s[0,1] if object.is_a?(CodeObjects::MethodObject)
         fspath = [objname + (extension.empty? ? '' : ".#{extension}")]
         if object.namespace && object.namespace.path != ""
@@ -38,11 +38,16 @@ module YARD
         # Windows disallows \ / : * ? " < > | but we will just remove any
         # non alphanumeric (plus period, underscore and dash).
         fspath.map! do |p|
-          p.gsub(/[^\w\.-]/) {|x| '_' + x[0].to_s(16).upcase }
+          p.gsub(/[^\w\.-]/) do |x|
+            encoded = '_'
+
+            x.each_byte { |b| encoded << ("%X" % b) }
+            encoded
+          end
         end
         
         File.join(fspath)
       end
     end
   end
-end
+end

data/lib/yard/tags/default_factory.rb

@@ -1,6 +1,9 @@
 module YARD
   module Tags
     class DefaultFactory
+      TYPELIST_OPENING_CHARS = '[({<'
+      TYPELIST_CLOSING_CHARS = '>})]'
+      
       ##
       # Parses tag text and creates a new tag with descriptive text
       #
@@ -30,7 +33,8 @@ module YARD
       # @param [String] text   the raw tag text
       # @return [Tag]          a tag object with the tag_name, types and text values filled
       def parse_tag_with_types(tag_name, text)
-        types, text = *extract_types_from_text(text)
+        name, types, text = *extract_types_and_name_from_text(text)
+        raise TagFormatError, "cannot specify a name before type list for '@#{tag_name}'" if name
         Tag.new(tag_name, text, types)
       end
       
@@ -42,8 +46,8 @@ module YARD
       # @param [String] text   the raw tag text
       # @return [Tag]          a tag object with the tag_name, name, types and text values filled
       def parse_tag_with_types_and_name(tag_name, text)
-        types, text = *extract_types_from_text(text)
-        name, text = *extract_name_from_text(text)
+        name, types, text = *extract_types_and_name_from_text(text)
+        name, text = *extract_name_from_text(text) unless name
         Tag.new(tag_name, text, types, name)
       end
       
@@ -56,6 +60,24 @@ module YARD
         Tag.new(tag_name, desc, nil, title)
       end
       
+      def parse_tag_with_types_name_and_default(tag_name, text)
+        # Can't allow () in a default tag, otherwise the grammar is too ambiguous when types is omitted.
+        open, close = TYPELIST_OPENING_CHARS.gsub('(', ''), TYPELIST_CLOSING_CHARS.gsub(')', '')
+        name, types, text = *extract_types_and_name_from_text(text, open, close)
+        name, text = *extract_name_from_text(text) unless name
+        if text =~ /\A\(/
+          _, default, text = *extract_types_and_name_from_text(text, '(', ')')
+          DefaultTag.new(tag_name, text, types, name, default)
+        else
+          DefaultTag.new(tag_name, text, types, name, nil)
+        end
+      end
+      
+      def parse_tag_with_options(tag_name, text)
+        name, text = *extract_name_from_text(text)
+        OptionTag.new(tag_name, name, parse_tag_with_types_name_and_default(tag_name, text))
+      end
+      
       private
       
       ##
@@ -69,17 +91,15 @@ module YARD
       end
       
       ##
-      # Extracts the type signatures from the raw tag text
+      # Extracts the type signatures and optional name from the raw tag text
       #
       # @param [String] text the raw tag text
       # @return [Array] an array holding the value as the first element and
       #                 the array of types as the second element
-      def extract_types_from_text(text)
-        types, text = [], text.strip
-        if text =~ /^\s*\[(.+?)\]\s*(.*)/
-          text, types = $2, $1.split(",").collect {|e| e.strip }
-        end
-        [types, text]
+      def extract_types_and_name_from_text(text)
+        text = text.strip
+        types, range, name = *parse_types(text)
+        [name, types, (range ? text[(range.end+1)..-1].strip : text)]
       end
       
       def extract_title_and_desc_from_raw_text(raw_text)
@@ -93,6 +113,46 @@ module YARD
         end
         [title, desc]
       end
+      
+      # Parses a [], <>, {} or () block at the beginning of a line of text into a list of
+      # comma delimited values. Returns the text be
+      # 
+      # @example
+      #   obj.parse_types('[String, Array<Hash, String>, nil]') # => [nil, ['String', 'Array<Hash, String>', 'nil'], ""]
+      #   obj.parse_types('b<String> A string') # => ['b', ['String'], 'A string']
+      # 
+      # @return [String, Array<String>] the text before the type list (or nil), followed by the type list parsed
+      #   into an array of strings, followed by the text following the type list.
+      # @return [nil] if no type list is present.
+      def extract_types_and_name_from_text(text, opening_types = TYPELIST_OPENING_CHARS, closing_types = TYPELIST_CLOSING_CHARS)
+        s, e = 0, 0
+        before = ''
+        list, level = [''], 0
+        text.split(//).each_with_index do |c, i|
+          if opening_types.include?(c)
+            list.last << c if level > 0
+            s = i if level == 0
+            level += 1
+          elsif closing_types.include?(c)
+            level -= 1 unless list.last[-1,1] == '='
+            break e = i if level == 0
+            list.last << c
+          elsif c == ',' && level == 1
+            list.push ''
+          elsif c =~ /\S/ && level == 0
+            before << c
+          elsif level >= 1
+            list.last << c
+          end
+        end
+
+        before = before.empty? ? nil : before.strip
+        if list.size == 1 && list.first == ''
+          [nil, nil, text.strip]
+        else
+          [before, list.map {|x| x.strip }, text[(e+1)..-1].strip]
+        end
+      end
     end
   end
 end