yard 0.6.4 → 0.6.5

data/lib/yard/parser/ruby/ruby_parser.rb

@@ -1,9 +1,12 @@
-require 'ripper'
+begin require 'ripper'; rescue LoadError; end
 
 module YARD
   module Parser
     module Ruby
       # Ruby 1.9 parser
+      # @attr_reader encoding_line
+      # @attr_reader shebang_line
+      # @attr_reader enumerator
       class RubyParser < Parser::Base
         def initialize(source, filename)
           @parser = RipperParser.new(source, filename)
@@ -12,12 +15,15 @@ module YARD
         def parse; @parser.parse end
         def tokenize; @parser.tokens end
         def enumerator; @parser.enumerator end
+        def shebang_line; @parser.shebang_line end
+        def encoding_line; @parser.encoding_line end
       end
       
       # Internal parser class
       # @since 0.5.6
       class RipperParser < Ripper
         attr_reader :ast, :charno, :comments, :file, :tokens
+        attr_reader :shebang_line, :encoding_line
         alias root ast
 
         def initialize(source, filename, *args)
@@ -27,12 +33,15 @@ module YARD
           @source = source
           @tokens = []
           @comments = {}
+          @comments_flags = {}
           @heredoc_tokens = []
           @map = {}
           @ns_charno = 0
           @list = []
           @charno = 0
           @groups = []
+          @shebang_line = nil
+          @encoding_line = nil
         end
 
         def parse
@@ -116,15 +125,15 @@ module YARD
         PARSER_EVENT_TABLE.each do |event, arity|
           node_class = AstNode.node_class_for(event)
           
-          if /_new\z/ =~ event and arity == 0
+          if /_new\z/ =~ event.to_s and arity == 0
             module_eval(<<-eof, __FILE__, __LINE__ + 1)
               def on_#{event}(*args)
-                #{node_class}.new(:list, args, listchar: charno...charno, listline: lineno..lineno)
+                #{node_class}.new(:list, args, :listchar => charno...charno, :listline => lineno..lineno)
               end
             eof
-          elsif /_add(_.+)?\z/ =~ event
+          elsif /_add(_.+)?\z/ =~ event.to_s
             module_eval(<<-eof, __FILE__, __LINE__ + 1)
-              undef on_#{event} if instance_method(:on_#{event})
+              begin; undef on_#{event}; rescue NameError; end
               def on_#{event}(list, item)
                 list.push(item)
                 list
@@ -132,16 +141,16 @@ module YARD
             eof
           elsif MAPPINGS.has_key?(event)
             module_eval(<<-eof, __FILE__, __LINE__ + 1)
-              undef on_#{event} if instance_method(:on_#{event})
+              begin; undef on_#{event}; rescue NameError; end
               def on_#{event}(*args)
                 visit_event #{node_class}.new(:#{event}, args)
               end
             eof
           else
             module_eval(<<-eof, __FILE__, __LINE__ + 1)
-              undef on_#{event} if instance_method(:on_#{event})
+              begin; undef on_#{event}; rescue NameError; end
               def on_#{event}(*args)
-                #{node_class}.new(:#{event}, args, listline: lineno..lineno, listchar: charno...charno)
+                #{node_class}.new(:#{event}, args, :listline => lineno..lineno, :listchar => charno...charno)
               end
             eof
           end
@@ -150,17 +159,18 @@ module YARD
         SCANNER_EVENTS.each do |event|
           ast_token = AST_TOKENS.include?(event)
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
-            undef on_#{event} if instance_method(:on_#{event})
+            begin; undef on_#{event}; rescue NameError; end
             def on_#{event}(tok)
               visit_ns_token(:#{event}, tok, #{ast_token.inspect})
             end
           eof
         end
         
-        REV_MAPPINGS.select {|k| k.is_a?(Symbol) }.each do |event, value|
+        REV_MAPPINGS.select {|k,v| k.is_a?(Symbol) }.each do |pair|
+          event, value = *pair
           ast_token = AST_TOKENS.include?(event)
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
-            undef on_#{event} if instance_method(:on_#{event})
+            begin; undef on_#{event}; rescue NameError; end
             def on_#{event}(tok)
               (@map[:#{event}] ||= []) << [lineno, charno]
               visit_ns_token(:#{event}, tok, #{ast_token.inspect})
@@ -170,7 +180,7 @@ module YARD
         
         [:kw, :op].each do |event|
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
-            undef on_#{event} if instance_method(:on_#{event})
+            begin; undef on_#{event}; rescue NameError; end
             def on_#{event}(tok)
               unless @last_ns_token == [:kw, "def"] ||
                   (@tokens.last && @tokens.last[0] == :symbeg)
@@ -183,7 +193,7 @@ module YARD
 
         [:sp, :nl, :ignored_nl].each do |event|
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
-            undef on_#{event} if instance_method(:on_#{event})
+            begin; undef on_#{event}; rescue NameError; end
             def on_#{event}(tok)
               add_token(:#{event}, tok)
               @charno += tok.length
@@ -192,7 +202,8 @@ module YARD
         end
         
         def visit_event(node)
-          lstart, sstart = *@map[MAPPINGS[node.type]].pop
+          map = @map[MAPPINGS[node.type]]
+          lstart, sstart = *(map ? map.pop : [lineno, lineno])
           node.source_range = Range.new(sstart, @ns_charno - 1)
           node.line_range = Range.new(lstart, lineno)
           node
@@ -213,7 +224,7 @@ module YARD
           @charno += data.length
           @ns_charno = charno
           if ast_token
-            AstNode.new(token, [data], line: lineno..lineno, char: ch..charno-1, token: true)
+            AstNode.new(token, [data], :line => lineno..lineno, :char => ch..charno-1, :token => true)
           end
         end
         
@@ -280,7 +291,7 @@ module YARD
           ll, lc = *@map[:aref].pop
           sr = args.first.source_range.first..lc
           lr = args.first.line_range.first..ll
-          AstNode.new(:aref, args, char: sr, line: lr)
+          AstNode.new(:aref, args, :char => sr, :line => lr)
         end
         
         def on_rbracket(tok)
@@ -301,31 +312,34 @@ module YARD
         end
         
         def on_const_path_ref(*args)
-          klass = AstNode.node_class_for(:const_path_ref)
-          klass.new(:const_path_ref, args, listline: lineno..lineno, listchar: charno..charno)
+          ReferenceNode.new(:const_path_ref, args, :listline => lineno..lineno, :listchar => charno..charno)
         end
         
         [:if_mod, :unless_mod, :while_mod].each do |kw|
           node_class = AstNode.node_class_for(kw)
           module_eval(<<-eof, __FILE__, __LINE__ + 1)
-            undef on_#{kw} if instance_method(:on_#{kw})
+            begin; undef on_#{kw}; rescue NameError; end
             def on_#{kw}(*args)
               sr = args.last.source_range.first..args.first.source_range.last
               lr = args.last.line_range.first..args.first.line_range.last
-              #{node_class}.new(:#{kw}, args, line: lr, char: sr)
+              #{node_class}.new(:#{kw}, args, :line => lr, :char => sr)
             end
           eof
         end
         
         def on_qwords_new
-          visit_event AstNode.new(:qwords_literal, [])
+          visit_event LiteralNode.new(:qwords_literal, [])
         end
         
         def on_string_literal(*args)
-          node = visit_event_arr(AstNode.new(:string_literal, args))
-          if @source[charno, 1] !~ /["']/
-            nsr = node.source_range
-            node.source_range = Range.new(nsr.first, nsr.last + 1) 
+          node = visit_event_arr(LiteralNode.new(:string_literal, args))
+          if args.size == 1
+            r = args[0].source_range
+            if node.source_range != Range.new(r.first - 1, r.last + 1)
+              klass = AstNode.node_class_for(node[0].type)
+              r = Range.new(node.source_range.first + 1, node.source_range.last - 1)
+              node[0] = klass.new(node[0].type, [@source[r]], :line => node.line_range, :char => r)
+            end
           end
           node
         end
@@ -335,7 +349,7 @@ module YARD
         end
         
         def on_string_content(*args)
-          AstNode.new(:string_content, args, listline: lineno..lineno, listchar: charno..charno)
+          AstNode.new(:string_content, args, :listline => lineno..lineno, :listchar => charno..charno)
         end
         
         def on_rescue(exc, *args)
@@ -344,7 +358,7 @@ module YARD
         end
 
         def on_void_stmt
-          AstNode.new(:void_stmt, [], line: lineno..lineno, char: charno...charno)
+          AstNode.new(:void_stmt, [], :line => lineno..lineno, :char => charno...charno)
         end
 
         def on_params(*args)
@@ -353,18 +367,18 @@ module YARD
               if arg.first.class == Array
                 arg.map! do |sub_arg|
                   if sub_arg.class == Array
-                    AstNode.new(:default_arg, sub_arg, listline: lineno..lineno, listchar: charno..charno)
+                    AstNode.new(:default_arg, sub_arg, :listline => lineno..lineno, :listchar => charno..charno)
                   else
                     sub_arg
                   end
                 end
               end
-              AstNode.new(:list, arg, listline: lineno..lineno, listchar: charno..charno)
+              AstNode.new(:list, arg, :listline => lineno..lineno, :listchar => charno..charno)
             else
               arg
             end
           end
-          ParameterNode.new(:params, args, listline: lineno..lineno, listchar: charno..charno)
+          ParameterNode.new(:params, args, :listline => lineno..lineno, :listchar => charno..charno)
         end
         
         def on_label(data)
@@ -372,11 +386,26 @@ module YARD
           ch = charno
           @charno += data.length
           @ns_charno = charno
-          AstNode.new(:label, [data[0...-1]], line: lineno..lineno, char: ch..charno-1, token: true)
+          AstNode.new(:label, [data[0...-1]], :line => lineno..lineno, :char => ch..charno-1, :token => true)
         end
 
         def on_comment(comment)
+          not_comment = false
+          if @last_ns_token.nil? || @last_ns_token.size == 0
+            if comment =~ SourceParser::SHEBANG_LINE && !@encoding_line
+              @shebang_line = comment
+              not_comment = true
+            elsif comment =~ SourceParser::ENCODING_LINE
+              @encoding_line = comment
+              not_comment = true
+            end
+          end
+
           visit_ns_token(:comment, comment)
+          if not_comment
+            @last_ns_token = nil
+            return
+          end
           case comment
           when /\A# @group\s+(.+)\s*\Z/
             @groups.unshift [lineno, $1]
@@ -385,16 +414,20 @@ module YARD
             @groups.unshift [lineno, nil]
             return
           end
-
-          comment = comment.gsub(/^\#{1,2}\s{0,1}/, '').chomp
-          append_comment = @comments[lineno - 1]
           
+          comment = comment.gsub(/^(\#{1,2})\s{0,1}/, '').chomp
+          append_comment = @comments[lineno - 1]
+          hash_flag = $1 == '##' ? true : false
+
           if append_comment && @comments_last_column == column
             @comments.delete(lineno - 1)
+            @comments_flags[lineno] = @comments_flags[lineno - 1]
+            @comments_flags.delete(lineno - 1)
             comment = append_comment + "\n" + comment
           end
           
           @comments[lineno] = comment
+          @comments_flags[lineno] = hash_flag if !append_comment
           @comments_last_column = column
         end
         
@@ -424,9 +457,11 @@ module YARD
             (node.line - 2).upto(node.line) do |line|
               comment = @comments[line]
               if comment && !comment.empty?
+                node.docstring_hash_flag = @comments_flags[line]
                 node.docstring = comment
                 node.docstring_range = ((line - comment.count("\n"))..line)
-                comments.delete(line)
+                @comments.delete(line)
+                @comments_flags.delete(line)
                 break
               end
             end
@@ -447,7 +482,7 @@ module YARD
             freeze_tree(child)
           end
         end
-      end
+      end if defined?(::Ripper)
     end
   end
 end

data/lib/yard/parser/source_parser.rb

@@ -1,8 +1,6 @@
 require 'stringio'
 
-begin
-  require 'continuation'
-rescue LoadError; end
+begin require 'continuation'; rescue LoadError; end
 
 module YARD
   module Parser
@@ -34,6 +32,9 @@ module YARD
     # @see Handlers::Base
     # @see CodeObjects::Base
     class SourceParser 
+      SHEBANG_LINE  = /\A\s*#!\S+/
+      ENCODING_LINE = /\A(?:\s*#*!.*\r?\n)?\s*#+.*coding\s*[:=]{1,2}\s*(\S+)/i
+      
       class << self
         # @return [Symbol] the default parser type (defaults to :ruby)
         attr_reader :parser_type
@@ -133,13 +134,13 @@ module YARD
         end
         
         # Returns the validated parser type. Basically, enforces that :ruby
-        # type is never set from Ruby 1.8
+        # type is never set if the Ripper library is not available
         # 
         # @param [Symbol] type the parser type to set
         # @return [Symbol] the validated parser type
         # @private
         def validated_parser_type(type)
-          RUBY18 && type == :ruby ? :ruby18 : type
+          !defined?(::Ripper) && type == :ruby ? :ruby18 : type
         end
         
         private
@@ -169,11 +170,11 @@ module YARD
         end
       end
 
-      self.parser_type = :ruby
-      
-      register_parser_type :ruby,   Ruby::RubyParser if RUBY19
+      register_parser_type :ruby,   Ruby::RubyParser
       register_parser_type :ruby18, Ruby::Legacy::RubyParser
       register_parser_type :c,      CParser, ['c', 'cc', 'cxx', 'cpp']
+
+      self.parser_type = :ruby
       
       # The filename being parsed by the parser.
       attr_reader :file
@@ -238,8 +239,8 @@ module YARD
       # Searches for encoding line and forces encoding
       # @since 0.5.3
       def convert_encoding(content)
-        return content if RUBY18
-        if content =~ /\A(?:\s*#*!.*\r?\n)?\s*#+.*coding\s*[:=]{1,2}\s*(\S+)/i
+        return content unless content.respond_to?(:force_encoding)
+        if content =~ ENCODING_LINE
           content.force_encoding($1)
         else
           content

data/lib/yard/registry.rb

@@ -9,15 +9,27 @@ module YARD
   # objects are done on the singleton Registry instance using the {Registry.at} 
   # or {Registry.resolve} methods.
   # 
-  # The registry is saved to a "yardoc" file, which can be loaded back to 
-  # perform any lookups.
+  # == Saving / Loading a Registry 
+  # The registry is saved to a "yardoc file" (actually a directory), which can 
+  # be loaded back to perform any lookups. See {Registry.load!} and 
+  # {Registry.save} for information on saving and loading of a yardoc file.
   # 
-  # This class is a singleton class. Any method called on the class will be
-  # delegated to the instance.
+  # == Threading Notes
+  # The registry class is a singleton class that is accessed directly in many
+  # places across YARD. To mitigate threading issues, YARD (0.6.5+) makes
+  # the Registry thread local. This means all access to a registry for a specific
+  # object set must occur in the originating thread.
+  # 
+  # @example Loading a Registry
+  #   Registry.load!('/path/to/yardocfile') # loads all objects into memory
+  #   Registry.at('YARD::CodeObjects::Base').docstring
+  #   # => "+Base+ is the superclass of all code objects ..."
+  # @example Performing a Search on a Registry
+  #   Registry.resolve(P('YARD::CodeObjects::Base'), '#docstring')
+  #   # => <
   module Registry
     DEFAULT_YARDOC_FILE = ".yardoc"
     LOCAL_YARDOC_INDEX = File.expand_path('~/.yard/gem_index')
-    @yardoc_file = DEFAULT_YARDOC_FILE
 
     extend Enumerable
 
@@ -58,6 +70,10 @@ module YARD
       # @return [String] the yardoc filename
       # @see DEFAULT_YARDOC_FILE
       attr_accessor :yardoc_file
+      def yardoc_file=(v) Thread.current[:__yard_yardoc_file__] = v end
+      def yardoc_file
+        Thread.current[:__yard_yardoc_file__] ||= DEFAULT_YARDOC_FILE 
+      end
     
       # @group Loading Data from Disk
     
@@ -81,9 +97,9 @@ module YARD
           if File.exists?(yardoc_file) && !reparse
             load_yardoc
           else
-            size = @store.keys.size
+            size = thread_local_store.keys.size
             YARD.parse(files)
-            save if @store.keys.size > size
+            save if thread_local_store.keys.size > size
           end
         elsif files.is_a?(String)
           load_yardoc(files)
@@ -99,7 +115,7 @@ module YARD
       # @return [Registry] the registry object (for chaining)
       def load_yardoc(file = yardoc_file)
         clear
-        @store.load(file)
+        thread_local_store.load(file)
         self
       end
     
@@ -113,7 +129,7 @@ module YARD
       # @since 0.5.1
       def load!(file = yardoc_file)
         clear
-        @store.load!(file)
+        thread_local_store.load!(file)
         self
       end
     
@@ -127,7 +143,7 @@ module YARD
       # @return [Registry] the registry object (for chaining)
       # @since 0.5.1
       def load_all
-        @store.load_all
+        thread_local_store.load_all
         self
       end
     
@@ -138,13 +154,13 @@ module YARD
       # @param [String] file the yardoc file to save to
       # @return [Boolean] true if the file was saved
       def save(merge = false, file = yardoc_file)
-        @store.save(merge, file)
+        thread_local_store.save(merge, file)
       end
     
       # Deletes the yardoc file from disk
       # @return [void]
       def delete_from_disk
-        @store.destroy
+        thread_local_store.destroy
       end
     
       # @group Adding and Deleting Objects from the Registry
@@ -155,20 +171,20 @@ module YARD
       # @return [CodeObjects::Base] the registered object
       def register(object)
         return if object.is_a?(CodeObjects::Proxy)
-        @store[object.path] = object
+        thread_local_store[object.path] = object
       end
     
       # Deletes an object from the registry
       # @param [CodeObjects::Base] object the object to remove
       # @return [void] 
       def delete(object) 
-        @store.delete(object.path)
+        thread_local_store.delete(object.path)
       end
 
       # Clears the registry
       # @return [void] 
       def clear
-        @store = RegistryStore.new
+        self.thread_local_store = RegistryStore.new
       end
 
       # @group Accessing Objects in the Registry
@@ -191,7 +207,7 @@ module YARD
       # @return [Array<CodeObjects::Base>] the list of objects found
       # @see CodeObjects::Base#type
       def all(*types)
-        @store.values.select do |obj| 
+        thread_local_store.values.select do |obj| 
           if types.empty?
             obj != root
           else
@@ -207,7 +223,7 @@ module YARD
       # @param [Boolean] reload whether to load entire database
       # @return [Array<String>] all of the paths in the registry.
       def paths(reload = false)
-        @store.keys(reload).map {|k| k.to_s }
+        thread_local_store.keys(reload).map {|k| k.to_s }
       end
     
       # Returns the object at a specific path.
@@ -215,12 +231,12 @@ module YARD
       #   returns the {root} object.
       # @return [CodeObjects::Base] the object at path
       # @return [nil] if no object is found
-      def at(path) path ? @store[path] : nil end
+      def at(path) path ? thread_local_store[path] : nil end
       alias_method :[], :at
     
       # The root namespace object.
       # @return [CodeObjects::RootObject] the root object in the namespace
-      def root; @store[:root] end
+      def root; thread_local_store[:root] end
     
       # Attempts to find an object by name starting at +namespace+, performing
       # a lookup similar to Ruby's method of resolving a constant in a namespace.
@@ -285,7 +301,7 @@ module YARD
     
       # @return [Hash{String => String}] a set of checksums for files
       def checksums
-        @store.checksums
+        thread_local_store.checksums
       end
     
       # @param [String] data data to checksum
@@ -294,13 +310,25 @@ module YARD
         Digest::SHA1.hexdigest(data)
       end
     
-      # @group Managing Internal State (Testing Only)
+      # @group Managing Internal State (Advanced / Testing Only)
+       
+      # Whether or not the Registry storage should load everything into a
+      # single object database (for disk efficiency), or spread them out 
+      # (for load time efficiency).
+      # 
+      # @note Setting this attribute to nil will offload the decision to
+      #   the {RegistryStore storage adapter}.
+      # @return [Boolean, nil] if this value is set to nil, the storage
+      #   adapter will decide how to store the data.
+      attr_accessor :single_object_db
+      def single_object_db=(v) Thread.current[:__yard_single_db__] = v end
+      def single_object_db; Thread.current[:__yard_single_db__] end
 
       # The assumed types of a list of paths. This method is used by CodeObjects::Base
       # @return [{String => Symbol}] a set of unresolved paths and their assumed type
       # @private
       def proxy_types
-        @store.proxy_types
+        thread_local_store.proxy_types
       end
       
       # @group Legacy Methods
@@ -355,8 +383,18 @@ module YARD
           File.exist?(path) ? path : nil
         end
       end
+      
+      # @group Threading support
+       
+      # @since 0.6.5
+      def thread_local_store
+        Thread.current[:__yard_registry__] ||= clear
+      end
+      
+      # @since 0.6.5
+      def thread_local_store=(value)
+        Thread.current[:__yard_registry__] = value
+      end
     end
-    
-    clear
   end
 end