solargraph 0.39.0 → 0.40.0

data/lib/solargraph/yard_map/rdoc_to_yard.rb

@@ -11,22 +11,27 @@ module Solargraph
       extend ApiMap::SourceToYard
 
       # @param spec [Gem::Specification]
-      def self.run spec
+      # @param cache_dir [String]
+      # @return [void]
+      def self.run spec, cache_dir: nil
         Dir.mktmpdir do |tmpdir|
-          rdir = File.join(tmpdir, 'rdoc')
-          Dir.chdir spec.full_gem_path do
+          rdir = File.join(tmpdir, 'sg_tmp_rdoc')
+          FileUtils.cp_r Dir.glob(File.join(spec.full_gem_path, '*')), tmpdir
+          Dir.chdir tmpdir do
             pins = []
             pins.push Solargraph::Pin::ROOT_PIN
             name_hash = {}
-            cmd = "rdoc -q -N -r -o #{rdir}"
+
+            argv = ['-q', '-r', '-N', '-o', rdir]
             spec.load_paths.each do |path|
-              cmd += " -i #{path}"
+              argv.concat ['-i', path]
             end
-            `#{cmd}`
-            store = RDoc::Store.new(rdir)
-            store.load_all
+            rdoc = RDoc::RDoc.new
+            rdoc.document argv
+            # @type [RDoc::Store]
+            store = rdoc.store
+            store.path = rdir
             store.cache[:modules].each do |mod|
-              # store.load_class(mod)
               # @type [RDoc::NormalClass]
               mod = store.find_class_or_module(mod)
               closure = pins.select { |pin| pin.path == mod.full_name.split('::')[0..-2].join('::') }.first || pins.first
@@ -46,7 +51,7 @@ module Solargraph
                   closure: namepin
                 )
               end
-              # @param inc [RDoc::Extend]
+              # @param ext [RDoc::Extend]
               mod.extends.each do |ext|
                 pins.push Solargraph::Pin::Reference::Extend.new(
                   location: locate(ext),
@@ -58,13 +63,13 @@ module Solargraph
               name_hash[mod.full_name] = namepin
               # @param met [RDoc::AnyMethod]
               mod.each_method do |met|
-                pin = Solargraph::SourceMap.load_string("def Object.tmp#{met.param_seq};end").first_pin('Object.tmp') || Solargraph::Pin::BaseMethod.new
+                pin = Solargraph::SourceMap.load_string("def Object.tmp#{met.param_seq};end").first_pin('Object.tmp') || Solargraph::Pin::Method.new
                 pins.push Solargraph::Pin::Method.new(
                   name: met.name,
                   closure: namepin,
                   comments: commentary(met.comment),
                   scope: met.type.to_sym,
-                  args: pin.parameters,
+                  parameters: pin.parameters,
                   visibility: met.visibility,
                   location: locate(met)
                 )
@@ -85,11 +90,13 @@ module Solargraph
             code_object_map.values.each do |co|
               YARD::Registry.register(co)
             end
-            cache_dir = File.join(Solargraph::YardMap::CoreDocs.cache_dir, 'gems', "#{spec.name}-#{spec.version}", "yardoc")
+            cache_dir ||= File.join(Solargraph::YardMap::CoreDocs.cache_dir, 'gems', "#{spec.name}-#{spec.version}", "yardoc")
             FileUtils.remove_entry_secure cache_dir if File.exist?(cache_dir)
             FileUtils.mkdir_p cache_dir
             # @todo Should merge be true?
             YARD::Registry.save true, cache_dir
+            # Clear the serialized cache if it exists
+            FileUtils.safe_unlink File.join(CoreDocs.cache_dir, 'gems', "#{spec.name}-#{spec.version}.ser")
           end
         end
       end
@@ -98,7 +105,11 @@ module Solargraph
         mod.full_name.split('::')[0..-2].join('::')
       end
 
+      # @param cmnt [RDoc::Comment]
+      # @return [String]
       def self.commentary cmnt
+        return cmnt if cmnt.is_a?(String)
+        return cmnt.text if cmnt.is_a?(RDoc::Comment)
         result = []
         cmnt.parts.each do |part|
           result.push RDoc::Markup::ToHtml.new({}).to_html(part.text) if part.respond_to?(:text)
@@ -119,9 +130,9 @@ module Solargraph
 
       def self.find_file obj
         if obj.respond_to?(:in_files) && !obj.in_files.empty?
-          [obj.in_files.first.to_s.sub(/^file /, ''), obj.in_files.first.line]
+          [obj.in_files.first.to_s.sub(/^file /, ''), obj.line]
         else
-          [obj.file_name, obj.line]
+          [obj.file, obj.line]
         end
       end
     end

data/lib/solargraph/yard_map/stdlib_fills.rb

@@ -0,0 +1,43 @@
+module Solargraph
+  class YardMap
+    module StdlibFills
+      Override = Pin::Reference::Override
+
+      LIBS = {
+        'benchmark' => [
+          Override.method_return('Benchmark.measure', 'Benchmark::Tms')
+        ],
+
+        'pathname' => [
+          Override.method_return('Pathname#join', 'Pathname'),
+          Override.method_return('Pathname#basename', 'Pathname'),
+          Override.method_return('Pathname#dirname', 'Pathname'),
+          Override.method_return('Pathname#cleanpath', 'Pathname'),
+          Override.method_return('Pathname#children', 'Array<Pathname>'),
+          Override.method_return('Pathname#entries', 'Array<Pathname>')
+        ],
+
+        'set' => [
+          Override.method_return('Enumerable#to_set', 'Set'),
+          Override.method_return('Set#add', 'self'),
+          Override.method_return('Set#add?', 'self, nil'),
+          Override.method_return('Set#classify', 'Hash'),
+          Override.from_comment('Set#each', '@yieldparam_single_parameter'),
+        ],
+
+        'tempfile' => [
+          Override.from_comment('Tempfile.open', %(
+@yieldparam [Tempfile]
+@return [Tempfile]
+          ))
+        ]
+      }.freeze
+
+      # @param path [String]
+      # @return [Array<Pin::Reference::Override>]
+      def self.get path
+        LIBS[path] || []
+      end
+    end
+  end
+end

data/lib/solargraph/yard_map/to_method.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Solargraph
+  class YardMap
+    class ToMethod
+      module InnerMethods
+        module_function
+
+        # @param code_object [YARD::CodeObjects::Base]
+        # @return [Array<Solargraph::Pin::Parameter>]
+        def get_parameters code_object, location, comments
+          return [] unless code_object.is_a?(YARD::CodeObjects::MethodObject)
+          # HACK: Skip `nil` and `self` parameters that are sometimes emitted
+          # for methods defined in C
+          # See https://github.com/castwide/solargraph/issues/345
+          code_object.parameters.select { |a| a[0] && a[0] != 'self' }.map do |a|
+            Solargraph::Pin::Parameter.new(
+              location: location,
+              closure: self,
+              comments: comments,
+              name: arg_name(a),
+              presence: nil,
+              decl: arg_type(a),
+              asgn_code: a[1]
+            )
+          end
+        end
+
+        # @param a [Array]
+        # @return [String]
+        def arg_name a
+          a[0].gsub(/[^a-z0-9_]/i, '')
+        end
+
+        # @param a [Array]
+        # @return [::Symbol]
+        def arg_type a
+          if a[0].start_with?('**')
+            :kwrestarg
+          elsif a[0].start_with?('*')
+            :restarg
+          elsif a[0].start_with?('&')
+            :blockarg
+          elsif a[0].end_with?(':')
+            a[1] ? :kwoptarg : :kwarg
+          elsif a[1]
+            :optarg
+          else
+            :arg
+          end
+        end
+      end
+      private_constant :InnerMethods
+
+      # include YardMixin
+      # extend YardMixin
+      extend Helpers
+
+      def make code_object, name = nil, scope = nil, visibility = nil, closure = nil, spec = nil
+        closure ||= Solargraph::Pin::Namespace.new(
+          name: code_object.namespace.to_s,
+          gates: [code_object.namespace.to_s]
+        )
+        location = object_location(code_object, spec)
+        comments = code_object.docstring ? code_object.docstring.all.to_s : ''
+        Pin::Method.new(
+          location: location,
+          closure: closure,
+          name: name || code_object.name.to_s,
+          comments: comments,
+          scope: scope || code_object.scope,
+          visibility: visibility || code_object.visibility,
+          parameters: InnerMethods.get_parameters(code_object, location, comments),
+          explicit: code_object.is_explicit?
+        )
+      end
+    end
+  end
+end

data/lib/solargraph/yard_map.rb

@@ -3,17 +3,22 @@
 require 'yard'
 require 'yard-solargraph'
 require 'rubygems/package'
+require 'set'
 
 module Solargraph
   # The YardMap provides access to YARD documentation for the Ruby core, the
   # stdlib, and gems.
   #
   class YardMap
-    autoload :Cache,    'solargraph/yard_map/cache'
-    autoload :CoreDocs, 'solargraph/yard_map/core_docs'
-    autoload :CoreGen,  'solargraph/yard_map/core_gen'
-    autoload :Mapper,   'solargraph/yard_map/mapper'
-    autoload :RdocToYard, 'solargraph/yard_map/rdoc_to_yard'
+    autoload :Cache,       'solargraph/yard_map/cache'
+    autoload :CoreDocs,    'solargraph/yard_map/core_docs'
+    autoload :CoreGen,     'solargraph/yard_map/core_gen'
+    autoload :Mapper,      'solargraph/yard_map/mapper'
+    autoload :RdocToYard,  'solargraph/yard_map/rdoc_to_yard'
+    autoload :CoreFills,   'solargraph/yard_map/core_fills'
+    autoload :StdlibFills, 'solargraph/yard_map/stdlib_fills'
+    autoload :Helpers,     'solargraph/yard_map/helpers'
+    autoload :ToMethod,    'solargraph/yard_map/to_method'
 
     CoreDocs.require_minimum
 
@@ -49,6 +54,8 @@ module Solargraph
     def initialize(required: [], gemset: {}, with_dependencies: true)
       # HACK: YardMap needs its own copy of this array
       @required = required.clone
+      # HACK: Hardcoded YAML handling
+      @required.push 'psych' if @required.include?('yaml')
       @with_dependencies = with_dependencies
       @gem_paths = {}
       @stdlib_namespaces = []
@@ -56,6 +63,7 @@ module Solargraph
       @source_gems = []
       process_requires
       yardocs.uniq!
+      @pin_select_cache = {}
     end
 
     # @return [Array<Solargraph::Pin::Base>]
@@ -72,6 +80,8 @@ module Solargraph
     # @param new_gemset [Hash{String => String}]
     # @return [Boolean]
     def change new_requires, new_gemset, source_gems = []
+      # HACK: Hardcoded YAML handling
+      new_requires.push 'psych' if new_requires.include?('yaml')
       if new_requires.uniq.sort == required.uniq.sort && new_gemset == gemset && @source_gems.uniq.sort == source_gems.uniq.sort
         false
       else
@@ -80,10 +90,22 @@ module Solargraph
         @gemset = new_gemset
         @source_gems = source_gems
         process_requires
+        @rebindable_method_names = nil
+        @pin_class_hash = nil
+        @pin_select_cache = {}
         true
       end
     end
 
+    # @return [Set<String>]
+    def rebindable_method_names
+      @rebindable_method_names ||= pins_by_class(Pin::Method)
+        .select { |pin| pin.comments && pin.comments.include?('@yieldself') }
+        .map(&:name)
+        .concat(['instance_eval', 'instance_exec', 'class_eval', 'class_exec', 'module_eval', 'module_exec'])
+        .to_set
+    end
+
     # @return [Array<String>]
     def yardocs
       @yardocs ||= []
@@ -102,7 +124,7 @@ module Solargraph
       else
         YARD::Registry.load! y
       end
-    rescue Exception => e
+    rescue StandardError => e
       Solargraph::Logging.logger.warn "Error loading yardoc '#{y}' #{e.class} #{e.message}"
       yardocs.delete y
       nil
@@ -110,58 +132,14 @@ module Solargraph
 
     # @return [Array<Solargraph::Pin::Base>]
     def core_pins
-      @@core_pins ||= begin
-        yd = CoreDocs.yardoc_file
-        ser = File.join(File.dirname(yd), 'core.ser')
-        result = []
-        if File.file?(ser)
-          file = File.open(ser, 'rb')
-          dump = file.read
-          file.close
-          result.concat Marshal.load(dump)
-        else
-          load_yardoc CoreDocs.yardoc_file
-          result.concat Mapper.new(YARD::Registry.all).map
-          # HACK: Assume core methods with a single `args` parameter accept restarg
-          result.select { |pin| pin.is_a?(Solargraph::Pin::BaseMethod )}.each do |pin|
-            if pin.parameters.length == 1 && pin.parameters.first.name == 'args' && pin.parameters.first.decl == :arg
-              # @todo Smelly instance variable access
-              pin.parameters.first.instance_variable_set(:@decl, :restarg)
-            end
-          end
-          # HACK: Set missing parameters on `==` methods, e.g., `Symbol#==`
-          result.select { |pin| pin.name == '==' && pin.parameters.empty? }.each do |pin|
-            pin.parameters.push Pin::Parameter.new(decl: :arg, name: 'obj2')
-          end
-          dump = Marshal.dump(result)
-          file = File.open(ser, 'wb')
-          file.write dump
-          file.close
-        end
-        # HACK: Add Errno exception classes
-        errno = result.select{ |pin| pin.path == 'Errno' }.first
-        Errno.constants.each do |const|
-          result.push Solargraph::Pin::Namespace.new(type: :class, name: const.to_s, closure: errno)
-          result.push Solargraph::Pin::Reference::Superclass.new(closure: result.last, name: 'SystemCallError')
-        end
-        CoreFills::OVERRIDES.each do |ovr|
-          pin = result.select { |p| p.path == ovr.name }.first
-          next if pin.nil?
-          (ovr.tags.map(&:tag_name) + ovr.delete).uniq.each do |tag|
-            pin.docstring.delete_tags tag.to_sym
-          end
-          ovr.tags.each do |tag|
-            pin.docstring.add_tag(tag)
-          end
-        end
-        result
-      end
+      # Using a class variable to reduce loads
+      @@core_pins ||= load_core_pins
     end
 
     # @param path [String]
     # @return [Pin::Base]
     def path_pin path
-      pins.select{ |p| p.path == path }.first
+      pins.select { |p| p.path == path }.first
     end
 
     # Get the location of a file referenced by a require path.
@@ -181,6 +159,10 @@ module Solargraph
       nil
     end
 
+    def stdlib_pins
+      @stdlib_pins ||= []
+    end
+
     private
 
     # @return [YardMap::Cache]
@@ -188,6 +170,16 @@ module Solargraph
       @cache ||= YardMap::Cache.new
     end
 
+    # @return [Hash]
+    def pin_class_hash
+      @pin_class_hash ||= pins.to_set.classify(&:class).transform_values(&:to_a)
+    end
+
+    # @return [Array<Pin::Base>]
+    def pins_by_class klass
+      @pin_select_cache[klass] ||= pin_class_hash.select { |key, _| key <= klass }.values.flatten
+    end
+
     # @param ns [YARD::CodeObjects::NamespaceObject]
     # @return [Array<YARD::CodeObjects::Base>]
     def recurse_namespace_object ns
@@ -201,12 +193,13 @@ module Solargraph
 
     # @return [void]
     def process_requires
-      pins.clear
+      pins.replace core_pins
       unresolved_requires.clear
-      # stdnames = {}
+      stdlib_pins.clear
+      environ = Convention.for_global(self)
       done = []
       from_std = []
-      required.each do |r|
+      (required + environ.requires).each do |r|
         next if r.nil? || r.empty? || done.include?(r)
         done.push r
         cached = cache.get_path_pins(r)
@@ -231,37 +224,17 @@ module Solargraph
             yardocs.unshift yd
             result.concat process_yardoc yd, spec
             result.concat add_gem_dependencies(spec) if with_dependencies?
-            stdlib_fill r, result
           end
         rescue Gem::LoadError => e
           base = r.split('/').first
           next if from_std.include?(base)
           from_std.push base
-          stdtmp = []
-          ser = File.join(File.dirname(CoreDocs.yardoc_stdlib_file), "#{base}.ser")
-          if File.file?(ser)
-            Solargraph.logger.info "Loading #{base} stdlib from cache"
-            file = File.open(ser, 'rb')
-            dump = file.read
-            file.close
-            stdtmp.concat Marshal.load(dump)
-          else
-            if stdlib_paths[base]
-              Solargraph.logger.info "Loading #{base} stdlib from yardoc"
-              stdtmp.concat Mapper.new(stdlib_paths[base]).map
-              next if stdtmp.empty?
-              dump = Marshal.dump(stdtmp)
-              file = File.open(ser, 'wb')
-              file.write dump
-              file.close
-            end
-          end
+          stdtmp = load_stdlib_pins(base)
           if stdtmp.empty?
             unresolved_requires.push r
           else
-            stdlib_fill base, stdtmp
+            stdlib_pins.concat stdtmp
             result.concat stdtmp
-            # stdnames[r] = stdtmp
           end
         end
         result.delete_if(&:nil?)
@@ -270,8 +243,13 @@ module Solargraph
           pins.concat result
         end
       end
-      # pins.concat process_stdlib(stdnames)
-      pins.concat core_pins
+      if required.include?('yaml') && required.include?('psych')
+        # HACK: Hardcoded YAML handling
+        # @todo Why can't this be handled with an override or a virtual pin?
+        pin = path_pin('YAML')
+        pin.instance_variable_set(:@return_type, ComplexType.parse('Module<Psych>')) unless pin.nil?
+      end
+      pins.concat environ.pins
     end
 
     # @param spec [Gem::Specification]
@@ -316,7 +294,12 @@ module Solargraph
           file = File.open(ser, 'rb')
           dump = file.read
           file.close
-          return Marshal.load(dump)
+          begin
+            return Marshal.load(dump)
+          rescue StandardError => e
+            Solargraph.logger.warn "Error loading pin cache: [#{e.class}] #{e.message}"
+            File.unlink ser
+          end
         end
       end
       if !size.nil? && size > 20_000_000
@@ -364,20 +347,85 @@ module Solargraph
       spec
     end
 
-    # @param path [String]
-    # @param pins [Array<Pin::Base>]
-    # @return [void]
-    def stdlib_fill path, pins
-      StdlibFills.get(path).each do |ovr|
-        pin = pins.select { |p| p.path == ovr.name }.first
-        next if pin.nil?
-        (ovr.tags.map(&:tag_name) + ovr.delete).uniq.each do |tag|
-          pin.docstring.delete_tags tag.to_sym
+    def load_core_pins
+      yd = CoreDocs.yardoc_file
+      ser = File.join(File.dirname(yd), 'core.ser')
+      result = if File.file?(ser)
+        file = File.open(ser, 'rb')
+        dump = file.read
+        file.close
+        begin
+          Marshal.load(dump)
+        rescue StandardError => e
+          Solargraph.logger.warn "Error loading core pin cache: [#{e.class}] #{e.message}"
+          File.unlink ser
+          read_core_and_save_cache(yd, ser)
         end
-        ovr.tags.each do |tag|
-          pin.docstring.add_tag(tag)
+      else
+        read_core_and_save_cache(yd, ser)
+      end
+      ApiMap::Store.new(result + CoreFills::ALL).pins.reject { |pin| pin.is_a?(Pin::Reference::Override) }
+    end
+
+    def read_core_and_save_cache yd, ser
+      result = []
+      load_yardoc yd
+      result.concat Mapper.new(YARD::Registry.all).map
+      # HACK: Assume core methods with a single `args` parameter accept restarg
+      result.select { |pin| pin.is_a?(Solargraph::Pin::Method )}.each do |pin|
+        if pin.parameters.length == 1 && pin.parameters.first.name == 'args' && pin.parameters.first.decl == :arg
+          # @todo Smelly instance variable access
+          pin.parameters.first.instance_variable_set(:@decl, :restarg)
         end
       end
+      # HACK: Set missing parameters on `==` methods, e.g., `Symbol#==`
+      result.select { |pin| pin.name == '==' && pin.parameters.empty? }.each do |pin|
+        pin.parameters.push Pin::Parameter.new(decl: :arg, name: 'obj2')
+      end
+      dump = Marshal.dump(result)
+      file = File.open(ser, 'wb')
+      file.write dump
+      file.close
+      result
+    end
+
+    def load_stdlib_pins base
+      ser = File.join(File.dirname(CoreDocs.yardoc_stdlib_file), "#{base}.ser")
+      result = if File.file?(ser)
+        Solargraph.logger.info "Loading #{base} stdlib from cache"
+        file = File.open(ser, 'rb')
+        dump = file.read
+        file.close
+        begin
+          Marshal.load(dump)
+        rescue StandardError => e
+          Solargraph.logger.warn "Error loading #{base} stdlib pin cache: [#{e.class}] #{e.message}"
+          File.unlink ser
+          read_stdlib_and_save_cache(base, ser)
+        end
+      else
+        read_stdlib_and_save_cache(base, ser)
+      end
+      fills = StdlibFills.get(base)
+      unless fills.empty?
+        result = ApiMap::Store.new(result + fills).pins.reject { |pin| pin.is_a?(Pin::Reference::Override) }
+      end
+      result
+    end
+
+    def read_stdlib_and_save_cache base, ser
+      result = []
+      if stdlib_paths[base]
+        Solargraph.logger.info "Loading #{base} stdlib from yardoc"
+        result.concat Mapper.new(stdlib_paths[base]).map
+        unless result.empty?
+          dump = Marshal.dump(result)
+          file = File.open(ser, 'wb')
+          file.write dump
+          file.close
+        end
+      end
+      result
     end
   end
 end

data/lib/solargraph.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+Encoding.default_external = 'UTF-8'
+
+require 'solargraph/compat'
 require 'solargraph/version'
 
 # The top-level namespace for the Solargraph code mapping, documentation,
@@ -24,15 +27,13 @@ module Solargraph
   autoload :YardMap,          'solargraph/yard_map'
   autoload :Pin,              'solargraph/pin'
   autoload :ServerMethods,    'solargraph/server_methods'
-  autoload :CoreFills,        'solargraph/core_fills'
-  autoload :StdlibFills,      'solargraph/stdlib_fills'
   autoload :LanguageServer,   'solargraph/language_server'
   autoload :Workspace,        'solargraph/workspace'
   autoload :Page,             'solargraph/page'
   autoload :Library,          'solargraph/library'
   autoload :Diagnostics,      'solargraph/diagnostics'
   autoload :ComplexType,      'solargraph/complex_type'
-  autoload :Bundle,           'solargraph/bundle'
+  autoload :Bench,            'solargraph/bench'
   autoload :Logging,          'solargraph/logging'
   autoload :TypeChecker,      'solargraph/type_checker'
   autoload :Environ,          'solargraph/environ'
@@ -53,7 +54,7 @@ module Solargraph
   end
 
   # A helper method that runs Bundler.with_unbundled_env or falls back to
-  # Bundler.with_clean env for earlier versions of Bundler.
+  # Bundler.with_clean_env for earlier versions of Bundler.
   #
   def self.with_clean_env &block
     meth = if Bundler.respond_to?(:with_unbundled_env)

data/solargraph.gemspec

@@ -17,18 +17,18 @@ Gem::Specification.new do |s|
   s.license     = 'MIT'
   s.executables = ['solargraph']
 
-  s.required_ruby_version = '>= 2.1'
+  s.required_ruby_version = '>= 2.4'
 
   s.add_runtime_dependency 'backport', '~> 1.1'
   s.add_runtime_dependency 'benchmark'
   s.add_runtime_dependency 'bundler', '>= 1.17.2'
   s.add_runtime_dependency 'e2mmap'
   s.add_runtime_dependency 'jaro_winkler', '~> 1.5'
-  s.add_runtime_dependency 'maruku', '~> 0.7', '>= 0.7.3'
-  s.add_runtime_dependency 'nokogiri', '~> 1.9', '>= 1.9.1'
+  s.add_runtime_dependency 'kramdown', '~> 2.3'
+  s.add_runtime_dependency 'kramdown-parser-gfm', '~> 1.1'
   s.add_runtime_dependency 'parser', '~> 2.3'
-  s.add_runtime_dependency 'reverse_markdown', '~> 1.0', '>= 1.0.5'
-  s.add_runtime_dependency 'rubocop', '~> 0.52'
+  s.add_runtime_dependency 'reverse_markdown', '>= 1.0.5', '< 3'
+  s.add_runtime_dependency 'rubocop', '>= 0.52'
   s.add_runtime_dependency 'thor', '~> 1.0'
   s.add_runtime_dependency 'tilt', '~> 2.0'
   s.add_runtime_dependency 'yard', '~> 0.9', '>= 0.9.24'