yard 0.9.4 → 0.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 994e4c83b93e9b6c520138f2d8a221b25ab95ca4
-  data.tar.gz: 82668951125bfa84bb79b636c130af43a6daf772
+  metadata.gz: f93ca569b327f662d35892e83a4799a91868da56
+  data.tar.gz: c185c90879b8b862d40bb3eee2da95e67311e568
 SHA512:
-  metadata.gz: f41d5b06ef5224e1f4ebdc2278323df57e8c70ccd13105827905494c1e4ea81e170c20d9fe797e362dd45e045f4a17c700818e0cba2503fbf8c9178ffdac6392
-  data.tar.gz: 99e47e1a3ea86721d389ee3215d6adf643e0e1becc851f2ea0e4c1f7e26224d9d61e30530533c776c094d5e9035b5e43f45241e72e9f76cf59ca362526730f07
+  metadata.gz: 1f0cf4b6767842dcd90879cd6039e510767af156c17f6722b46028d239bca925dc854b5399009a27615761a50875523f7e2aa27f89b4e6f5f7d9d101d48f9ef2
+  data.tar.gz: 85d0822f567fb4caad941758e54f7805bceae24dbe1f84a31c4c0ae79658ddafb152c1646e3d65201847a69288133d7227a442825fbf0560b8399ff26714a6e6

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+# 0.9.5 - July 22nd, 2016
+
+- `yard doc` will now generate `.yardoc/processing` and `.yardoc/complete` files
+  to allow other tools to properly detect when YARD is in the middle of parsing
+  source files, and when it has completed writing the database.
+- Added support for on-demand generation of LibraryVersion objects using the
+  `:disk` source type. LibraryVersion objects pointing to a .yardoc database
+  directory will now auto-generate if there is a `source_path` attached.
+- Added warning for macros attached to non-method objects.
+- Fixed a few more parsing errors.
+
 # 0.9.4 - July 21st, 2016
 
 - Minor Ruby file parsing and CSS bug fixes.

data/lib/yard/cli/gems.rb

@@ -35,9 +35,10 @@ module YARD
             next unless yfile
             next unless File.directory?(spec.full_gem_path)
             Registry.clear
-            Dir.chdir(spec.full_gem_path)
-            log.info "Building yardoc index for gem: #{spec.full_name}"
-            Yardoc.run('--no-stats', '-n', '-b', yfile)
+            Dir.chdir(spec.full_gem_path) do
+              log.info "Building yardoc index for gem: #{spec.full_name}"
+              Yardoc.run('--no-stats', '-n', '-b', yfile)
+            end
           end
         end
       end

data/lib/yard/cli/server.rb

@@ -86,7 +86,7 @@ module YARD
             libver = create_library_version_if_yardopts_exist(library, pwd)
 
             # Check default location
-            yfile = File.join(pwd, '.yardoc')
+            yfile = File.join(pwd, Registry::DEFAULT_YARDOC_FILE)
             libver ||= YARD::Server::LibraryVersion.new(library, nil, yfile)
           end
 
@@ -213,8 +213,9 @@ module YARD
           add_libraries([File.basename(Dir.pwd), nil])
 
           # Generate doc for first time
+          # This is not necessary but makes for a better first-run experience
           libver = libraries.empty? ? nil : libraries.values.first.first
-          if libver and !File.exist?(libver.yardoc_file)
+          if libver and !libver.ready?
             generate_doc_for_first_time(libver)
           end
         else

data/lib/yard/cli/yardoc.rb

@@ -246,8 +246,15 @@ module YARD
           Registry.load
           checksums = Registry.checksums.dup
         end
-        YARD.parse(files, excluded)
-        Registry.save(use_cache) if save_yardoc
+
+        if save_yardoc
+          Registry.lock_for_writing do
+            YARD.parse(files, excluded)
+            Registry.save(use_cache)
+          end
+        else
+          YARD.parse(files, excluded)
+        end
 
         if generate
           run_generate(checksums)

data/lib/yard/handlers/ruby/decorator_handler_methods.rb

@@ -94,6 +94,10 @@ module YARD::Handlers::Ruby::DecoratorHandlerMethods
       name = node.jump(:ident, :string_content, :const).source
     end
 
+    if name.nil?
+      raise YARD::Parser::UndocumentableError, 'statement, cannot determine method name'
+    end
+
     method = YARD::CodeObjects::Proxy.new(
       namespace,
       (scope == :instance ? '#' : '.') + name.to_s,

data/lib/yard/parser/ruby/ruby_parser.rb

@@ -47,6 +47,7 @@ module YARD
           @encoding_line = nil
           @frozen_string_line = nil
           @file_encoding = nil
+          @newline = true
         end
 
         def parse

data/lib/yard/registry.rb

@@ -199,6 +199,20 @@ module YARD
         self.thread_local_store = RegistryStore.new
       end
 
+      # Creates a pessmistic transactional lock on the database for writing.
+      # Use with {YARD.parse} to ensure the database is not written multiple
+      # times.
+      #
+      # @see locked_for_writing?
+      def lock_for_writing(file = yardoc_file, &block)
+        thread_local_store.lock_for_writing(file, &block)
+      end
+
+      # (see Serializers::YardocSerializer#locked_for_writing?)
+      def locked_for_writing?(file = yardoc_file)
+        thread_local_store.locked_for_writing?(file)
+      end
+
       # @group Accessing Objects in the Registry
 
       # Iterates over {all} with no arguments
@@ -375,8 +389,11 @@ module YARD
       def global_yardoc_file(spec, for_writing = false)
         path = spec.doc_dir
         yfile = spec.doc_dir(DEFAULT_YARDOC_FILE)
-        if for_writing && File.writable?(path)
-          return yfile
+        if for_writing
+          if File.writable?(path) ||
+              (!File.directory?(path) && File.writable?(File.dirname(path)))
+            return yfile
+          end
         elsif !for_writing && File.exist?(yfile)
           return yfile
         end

data/lib/yard/registry_store.rb

@@ -187,9 +187,22 @@ module YARD
       write_proxy_types
       write_object_types
       write_checksums
+      write_complete_lock
       true
     end
 
+    # (see Serializers::YardocSerializer#lock_for_writing)
+    # @param file [String] if supplied, the path to the database
+    def lock_for_writing(file = nil, &block)
+      Serializers::YardocSerializer.new(file || @file).lock_for_writing(&block)
+    end
+
+    # (see Serializers::YardocSerializer#locked_for_writing?)
+    # @param file [String] if supplied, the path to the database
+    def locked_for_writing?(file = nil)
+      Serializers::YardocSerializer.new(file || @file).locked_for_writing?
+    end
+
     # Deletes the .yardoc database on disk
     #
     # @param [Boolean] force if force is not set to true, the file/directory
@@ -315,5 +328,9 @@ module YARD
         @checksums.each {|k, v| f.puts("#{k} #{v}") }
       end
     end
+
+    def write_complete_lock
+      File.open!(@serializer.complete_lock_path, 'w') {}
+    end
   end
 end

data/lib/yard/serializers/yardoc_serializer.rb

@@ -36,6 +36,29 @@ module YARD
       def proxy_types_path; File.join(basepath, 'proxy_types') end
       def checksums_path; File.join(basepath, 'checksums') end
       def object_types_path; File.join(basepath, 'object_types') end
+      def complete_lock_path; File.join(basepath, 'complete') end
+      def processing_path; File.join(basepath, 'processing') end
+
+      def complete?
+        File.exist?(complete_lock_path) && !locked_for_writing?
+      end
+
+      # Creates a pessmistic transactional lock on the database for writing.
+      # Use with {YARD.parse} to ensure the database is not written multiple
+      # times.
+      #
+      # @see #locked_for_writing?
+      def lock_for_writing(&block)
+        File.open!(processing_path, 'w') {}
+        yield
+      ensure
+        File.unlink(processing_path) if File.exist?(processing_path)
+      end
+
+      # @return [Boolean] whether the database is currently locked for writing
+      def locked_for_writing?
+        File.exist?(processing_path)
+      end
 
       def serialized_path(object)
         path = case object

data/lib/yard/server/commands/library_command.rb

@@ -106,7 +106,7 @@ module YARD
         end
 
         def load_yardoc
-          raise LibraryNotPreparedError unless library.yardoc_file
+          raise LibraryNotPreparedError unless library.ready?
           if Thread.current[:__yard_last_yardoc__] == library.yardoc_file
             log.debug "Reusing yardoc file: #{library.yardoc_file}"
             return

data/lib/yard/server/library_version.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'thread'
 
 module YARD
   module Server
@@ -144,6 +145,13 @@ module YARD
       alias == eql?
       alias equal? eql?
 
+      # @return [Boolean] whether the library has been completely processed
+      #   and is ready to be served
+      def ready?
+        return false if yardoc_file.nil?
+        serializer.complete?
+      end
+
       # @note You should not directly override this method. Instead, implement
       #   +load_yardoc_from_SOURCENAME+ when implementing loading for a specific
       #   source type. See the {LibraryVersion} documentation for "Implementing
@@ -160,7 +168,7 @@ module YARD
       #   begin preparing the library for subsequent requests, although this
       #   is not necessary.
       def prepare!
-        return if yardoc_file
+        return if ready?
         meth = "load_yardoc_from_#{source}"
         send(meth) if respond_to?(meth, true)
       end
@@ -170,16 +178,31 @@ module YARD
       # @return [nil] if there is no installed gem for the library
       def gemspec
         ver = version ? "= #{version}" : ">= 0"
-        Gem.source_index.find_name(name, ver).first
+        Gem.source_index.find_name(name, ver).last
       end
 
       protected
 
+      @@chdir_mutex = Mutex.new
+
       # Called when a library of source type "disk" is to be prepared. In this
-      # case, the {#yardoc_file} should already be set, so nothing needs to be
-      # done.
+      # case, the {#yardoc_file} should already be set, but the library may not
+      # be prepared. Run preparation if not done.
+      #
+      # @raise [LibraryNotPreparedError] if the yardoc file has not been
+      #   prepared.
       def load_yardoc_from_disk
-        nil
+        return if ready?
+
+        @@chdir_mutex.synchronize do
+          Dir.chdir(source_path_for_disk) do
+            Thread.new do
+              CLI::Yardoc.run('--no-stats', '-n', '-b', yardoc_file)
+            end
+          end
+        end
+
+        raise LibraryNotPreparedError
       end
 
       # Called when a library of source type "gem" is to be prepared. In this
@@ -192,18 +215,18 @@ module YARD
         require 'rubygems'
         ver = version ? "= #{version}" : ">= 0"
         self.yardoc_file = Registry.yardoc_file_for_gem(name, ver)
-        unless yardoc_file && File.directory?(yardoc_file)
+        return if ready?
+
+        @@chdir_mutex.synchronize do
           Thread.new do
             # Build gem docs on demand
             log.debug "Building gem docs for #{to_s(false)}"
             CLI::Gems.run(name, ver)
             self.yardoc_file = Registry.yardoc_file_for_gem(name, ver)
-            FileUtils.touch(File.join(yardoc_file, 'complete'))
           end
         end
-        unless yardoc_file && File.exist?(File.join(yardoc_file, 'complete'))
-          raise LibraryNotPreparedError
-        end
+
+        raise LibraryNotPreparedError
       end
 
       # @return [String] the source path for a disk source
@@ -222,6 +245,11 @@ module YARD
         meth = "source_path_for_#{source}"
         send(meth) if respond_to?(meth, true)
       end
+
+      def serializer
+        return if yardoc_file.nil?
+        @serializer ||= Serializers::YardocSerializer.new(yardoc_file)
+      end
     end
   end
 end

data/lib/yard/tags/directives.rb

@@ -299,8 +299,14 @@ module YARD
       def find_or_create
         if new? || attach?
           if handler && attach?
-            obj = object ? object :
-              P("#{handler.namespace}.#{handler.caller_method}")
+            if object && object.is_a?(CodeObjects::NamespaceObject)
+              log.warn "Attaching macros to non-methods is unsupported, ignoring: " +
+                "#{object.path} (#{handler.parser.file}:#{handler.statement.line})"
+              obj = nil
+            else
+              obj = object ? object :
+                P("#{handler.namespace}.#{handler.caller_method}")
+            end
           else
             obj = nil
           end

data/lib/yard/version.rb

@@ -1,3 +1,3 @@
 module YARD
-  VERSION = '0.9.4'
+  VERSION = '0.9.5'
 end

data/spec/cli/gems_spec.rb

@@ -19,11 +19,11 @@ describe YARD::CLI::Gems do
 
   def build_specs(*specs)
     specs.each do |themock|
-      expect(Registry).to receive(:yardoc_file_for_gem).with(themock.name, "= #{themock.version}").and_return(themock.yardoc_file)
-      expect(File).to receive(:directory?).with(themock.yardoc_file).and_return(@rebuild)
-      expect(File).to receive(:directory?).with(themock.full_gem_path).and_return(true)
-      expect(Registry).to receive(:yardoc_file_for_gem).with(themock.name, "= #{themock.version}", true).and_return(themock.yardoc_file)
-      expect(Dir).to receive(:chdir).with(themock.full_gem_path)
+      allow(Registry).to receive(:yardoc_file_for_gem).with(themock.name, "= #{themock.version}").and_return(themock.yardoc_file)
+      allow(File).to receive(:directory?).with(themock.yardoc_file).and_return(@rebuild)
+      allow(File).to receive(:directory?).with(themock.full_gem_path).and_return(true)
+      allow(Registry).to receive(:yardoc_file_for_gem).with(themock.name, "= #{themock.version}", true).and_return(themock.yardoc_file)
+      expect(Dir).to receive(:chdir).with(themock.full_gem_path).and_yield
     end
     expect(Registry).to receive(:clear).exactly(specs.size).times
     expect(CLI::Yardoc).to receive(:run).exactly(specs.size).times

data/spec/cli/server_spec.rb

@@ -83,7 +83,9 @@ describe YARD::CLI::Server do
   context 'when .yardopts file exists' do
     before :each do
       Registry.yardoc_file = Registry::DEFAULT_YARDOC_FILE
+      allow(File).to receive(:exist?).with(/\.yardoc\/complete$/).and_return(false)
       allow(Dir).to receive(:pwd).and_return('/path/to/bar')
+      allow(Dir).to receive(:chdir).and_yield
       @name = 'bar'
     end
 
@@ -95,6 +97,7 @@ describe YARD::CLI::Server do
     end
 
     it "uses the yardoc db location specified by .yardopts" do
+      allow(File).to receive(:exist?).with(/\/foo\/complete$/).and_return(false)
       mock_file '/path/to/bar/.yardopts', '--db foo'
       @libraries[@name] = [Server::LibraryVersion.new(@name, nil, File.expand_path('/path/to/bar/foo'))]
       @libraries.values[0][0].source_path = File.expand_path('/path/to/bar')
@@ -111,6 +114,7 @@ describe YARD::CLI::Server do
 
   context "when .yardopts file doesn't exist" do
     before :each do
+      allow(File).to receive(:exist?).with(/\.yardoc\/complete$/).and_return(false)
       allow(File).to receive(:exist?).with(/^(.*[\\\/])?\.yardopts$/).and_return(false)
     end
 
@@ -143,6 +147,7 @@ describe YARD::CLI::Server do
 
   describe "General options" do
     before do
+      allow(File).to receive(:exist?).with(/\.yardoc\/complete$/).and_return(false)
       allow(File).to receive(:exist?).with(/\.yardopts$/).and_return(false)
     end
 

data/spec/cli/yardoc_spec.rb

@@ -12,6 +12,7 @@ describe YARD::CLI::Yardoc do
     allow(YARD).to receive(:parse)
     allow(Registry).to receive(:load)
     allow(Registry).to receive(:save)
+    allow(File).to receive(:open!)
   end
 
   describe "Defaults" do
@@ -803,5 +804,15 @@ describe YARD::CLI::Yardoc do
       expect(@yardoc).not_to receive(:parse_arguments)
       @yardoc.run(nil)
     end
+
+    it "creates processing lock if saving" do
+      expect(Registry).to receive(:lock_for_writing).and_yield
+      @yardoc.run
+    end
+
+    it "does not create processing lock if not saving" do
+      expect(Registry).not_to receive(:lock_for_writing)
+      @yardoc.run('--no-save')
+    end
   end
 end

data/spec/handlers/dsl_handler_spec.rb

@@ -194,6 +194,10 @@ describe "YARD::Handlers::Ruby::#{LEGACY_PARSER ? "Legacy::" : ""}DSLHandler" do
     eof
   end
 
+  it "warns on unparseable DSL statements" do
+    undoc_error "private(*foo(bar))"
+  end if HAVE_RIPPER
+
   # @note Currently unsupported behavior. Overriding a macro on an alias will
   #   not work until macro lookups can be done by caller_method directly.
   # @todo optimize MacroObject lookup to work by caller name, not macro name.

data/spec/parser/ruby/ruby_parser_spec.rb

@@ -403,6 +403,18 @@ eof
         expect(Registry.at("Foo::Bar#foo").docstring).to eq "Docstring2"
       end
 
+      it "supports #{type} statements at start of source" do
+        Registry.clear
+        YARD.parse_string <<-eof
+          #{type} condition?
+            class Foo; def bar; #{type} true; end end end
+          end
+        eof
+
+        expect(log.io.string).to eq ""
+        expect(Registry.at('Foo#bar')).not_to eq nil
+      end
+
       it "can handle complex non-modifier '#{type}' statements" do
         Registry.clear
         YARD.parse_string <<-eof
@@ -414,6 +426,7 @@ eof
           end
         eof
 
+        expect(log.io.string).to eq ""
         expect(Registry.at('Foo#initialize')).not_to eq nil
       end
 

data/spec/registry_spec.rb

@@ -66,6 +66,14 @@ describe YARD::Registry do
       expect(Registry.yardoc_file_for_gem('foo', '>= 0', true)).to eq '/path/to/foo/doc/.yardoc'
     end
 
+    it "returns new global .yardoc path for gem if for_writing=true and parent dir is writable (but dir does not exist)" do
+      expect(File).to receive(:writable?).with(@gem.doc_dir).and_return(false)
+      expect(File).to receive(:directory?).with(@gem.doc_dir).and_return(false)
+      expect(File).to receive(:writable?).with(File.dirname(@gem.doc_dir)).and_return(true)
+      expect(Gem.source_index).to receive(:find_name).with('foo', '>= 0').and_return([@gem])
+      expect(Registry.yardoc_file_for_gem('foo', '>= 0', true)).to eq '/path/to/foo/doc/.yardoc'
+    end
+
     it "returns local .yardoc path for gem if for_writing=true and dir is not writable" do
       expect(File).to receive(:writable?).with(@gem.doc_dir).and_return(false)
       expect(File).to receive(:writable?).with(@gem.full_gem_path).and_return(false)

data/spec/registry_store_spec.rb

@@ -132,6 +132,7 @@ describe YARD::RegistryStore do
       Registry.single_object_db = nil
       add_items(100)
       saves_to_singledb
+      expect(@serializer.complete?).to eq true
     end
 
     it "saves as single object db if single_object_db is nil and there are more than 3000 objects" do

data/spec/serializers/yardoc_serializer_spec.rb

@@ -43,4 +43,35 @@ describe YARD::Serializers::YardocSerializer do
       @serializer.serialize(data)
     end
   end
+
+  describe "#lock_for_writing" do
+    it "creates a lock file during writing and cleans up" do
+      expect(File).to receive(:open!).with(@serializer.processing_path, 'w')
+      expect(File).to receive(:exist?).with(@serializer.processing_path).exactly(2).times.and_return(true)
+      expect(File).to receive(:unlink).with(@serializer.processing_path)
+      @serializer.lock_for_writing do
+        expect(@serializer.locked_for_writing?).to eq true
+      end
+    end
+  end
+
+  describe "#complete?" do
+    it "returns false if complete file does not exist" do
+      allow(File).to receive(:exist?).with(@serializer.complete_lock_path).and_return(false)
+      allow(File).to receive(:exist?).with(@serializer.processing_path).and_return(false)
+      expect(@serializer.complete?).to eq false
+    end
+
+    it "returns false if processing file exists" do
+      allow(File).to receive(:exist?).with(@serializer.complete_lock_path).and_return(true)
+      allow(File).to receive(:exist?).with(@serializer.processing_path).and_return(true)
+      expect(@serializer.complete?).to eq false
+    end
+
+    it "returns true if complete file exists with no processing file" do
+      allow(File).to receive(:exist?).with(@serializer.complete_lock_path).and_return(true)
+      allow(File).to receive(:exist?).with(@serializer.processing_path).and_return(false)
+      expect(@serializer.complete?).to eq true
+    end
+  end
 end

data/spec/tags/directives_spec.rb

@@ -78,6 +78,7 @@ describe YARD::Tags::MacroDirective do
                    :caller_method => 'foo',
                    :scope => :instance, :visibility => :public,
                    :namespace => P('Foo::Bar'),
+                   :parser => OpenStruct.new(:file => "(stdin)", :line => 1),
                    :statement => OpenStruct.new(:source => 'foo :a, :b, :c'))
   end
 
@@ -145,6 +146,15 @@ describe YARD::Tags::MacroDirective do
       expect(doc).to eq 'expanded_data'
     end
 
+    it "does not attach macros to class/modules but creates macro" do
+      YARD::Registry.clear
+      YARD.parse_string "module Foo; end"
+      tag_parse("@!macro [attach] attached4\n  $1 $2 $3", YARD::Registry.at('Foo'), handler)
+      macro = CodeObjects::MacroObject.find('attached4')
+      expect(macro.method_object).to eq nil
+      expect(log.io.string).to match(/Attaching macros to non-methods is unsupported/)
+    end
+
     it "does not attempt to expand macro values if handler = nil" do
       tag_parse("@!macro [attach] xyz\n  $1 $2 $3")
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yard
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.9.5
 platform: ruby
 authors:
 - Loren Segal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-21 00:00:00.000000000 Z
+date: 2016-07-22 00:00:00.000000000 Z
 dependencies: []
 description: |2
       YARD is a documentation generation tool for the Ruby programming language.