sord 0.10.0 → 3.0.0

data/lib/sord.rb

@@ -1,6 +1,6 @@
 # typed: strong
 require 'sord/version'
-require 'sord/rbi_generator'
+require 'sord/generator'
 require 'sord/parlour_plugin'
 require 'yard'
 require 'sorbet-runtime'

data/lib/sord/generator.rb

@@ -0,0 +1,592 @@
+# typed: true
+require 'yard'
+require 'sord/type_converter'
+require 'sord/logging'
+require 'parlour'
+require 'rainbow'
+
+module Sord
+  # Converts the current working directory's YARD registry into an type
+  # signature file.
+  class Generator    
+    VALID_MODES = [:rbi, :rbs]
+
+    # @return [Integer] The number of objects this generator has processed so 
+    #   far.
+    def object_count
+      @namespace_count + @method_count
+    end
+
+    # @return [Array<Array(String, YARD::CodeObjects::Base, Integer)>] The 
+    #   errors encountered by by the generator. Each element is of the form
+    #   [message, item, line].
+    attr_reader :warnings
+
+    # Create a new generator.
+    # @param [Hash] options
+    # @option options [Symbol] mode
+    # @option options [Integer] break_params
+    # @option options [Boolean] replace_errors_with_untyped
+    # @option options [Boolean] replace_unresolved_with_untyped
+    # @option options [Boolean] comments
+    # @option options [Parlour::Generator] generator
+    # @option options [Parlour::TypedObject] root
+    # @return [void]
+    def initialize(options)
+      @mode = options[:mode].to_sym rescue options[:mode]
+      raise "invalid mode #{@mode}, expected one of: #{VALID_MODES.join(', ')}" \
+        unless VALID_MODES.include?(@mode)
+
+      @parlour = options[:parlour] || \
+        case @mode
+        when :rbi
+          Parlour::RbiGenerator.new
+        when :rbs
+          Parlour::RbsGenerator.new
+        end
+      @current_object = options[:root] || @parlour.root
+
+      @namespace_count = 0
+      @method_count = 0
+      @warnings = []
+
+      @replace_errors_with_untyped = options[:replace_errors_with_untyped]
+      @replace_unresolved_with_untyped = options[:replace_unresolved_with_untyped]
+      @keep_original_comments = options[:keep_original_comments]
+      @skip_constants = options[:skip_constants]
+      @use_original_initialize_return = options[:use_original_initialize_return]
+
+      # Hook the logger so that messages are added as comments
+      Logging.add_hook do |type, msg, item|
+        @current_object.add_comment_to_next_child("sord #{type} - #{msg}")
+      end if options[:sord_comments]
+
+      # Hook the logger so that warnings are collected
+      Logging.add_hook do |type, msg, item|
+        # TODO: is it possible to get line numbers here?
+        warnings << [msg, item, 0] if type == :warn
+      end
+    end
+
+    # Increment the namespace counter.
+    # @return [void]
+    def count_namespace
+      @namespace_count += 1
+    end
+
+    # Increment the method counter.
+    # @return [void]
+    def count_method
+      @method_count += 1
+    end
+
+    # Given a YARD CodeObject, add lines defining its mixins (that is, extends
+    # and includes) to the current file. Returns the number of mixins.
+    # @param [YARD::CodeObjects::Base] item
+    # @return [Integer]
+    def add_mixins(item)
+      item.instance_mixins.reverse_each do |i|
+        @current_object.create_include(i.path.to_s)
+      end
+      item.class_mixins.reverse_each do |e|
+        @current_object.create_extend(e.path.to_s)
+      end
+
+      item.instance_mixins.length + item.class_mixins.length
+    end
+
+    # Given a YARD NamespaceObject, add lines defining constants.
+    # @param [YARD::CodeObjects::NamespaceObject] item
+    # @return [void]
+    def add_constants(item)
+      inserted_constant_names = Set.new
+
+      item.constants(included: false).each do |constant|      
+        # Take a constant (like "A::B::CONSTANT"), split it on each '::', and
+        # set the constant name to the last string in the array.
+        constant_name = constant.to_s.split('::').last
+        if inserted_constant_names.include?(constant_name) && @mode == :rbs
+          Logging.warn("RBS doesn't support duplicate constants, but '#{constant_name}' was duplicated - dropping future occurences", constant)
+          next
+        end
+        inserted_constant_names << constant_name
+        
+        # Add the constant to the current object being generated.
+        case @mode
+        when :rbi
+          @current_object.create_constant(constant_name, value: "T.let(#{constant.value}, T.untyped)") do |c|
+            c.add_comments(constant.docstring.all.split("\n"))
+          end
+        when :rbs
+          @current_object.create_constant(constant_name, type: Parlour::Types::Untyped.new) do |c|
+            c.add_comments(constant.docstring.all.split("\n"))
+          end
+        end
+      end
+    end
+
+    # Adds comments to an object based on a docstring.
+    # @param [YARD::CodeObjects::NamespaceObject] item
+    # @param [Parlour::TypedObject] typed_object
+    # @return [void]
+    def add_comments(item, typed_object)
+      if @keep_original_comments
+        typed_object.add_comments(item.docstring.all.split("\n"))
+      else
+        parser = YARD::Docstring.parser
+        parser.parse(item.docstring.all)
+
+        docs_array = parser.text.split("\n")
+
+        # Add @param tags if there are any with names and descriptions.
+        params = parser.tags.select { |tag| tag.tag_name == 'param' && tag.is_a?(YARD::Tags::Tag) && !tag.name.nil? }
+        # Add a blank line if there's anything before the params.
+        docs_array << '' if docs_array.length.positive? && params.length.positive?
+        params.each do |param|
+          docs_array << '' if docs_array.last != '' && docs_array.length.positive?
+          # Output params in the form of:
+          # _@param_ `foo` — Lorem ipsum.
+          # _@param_ `foo`
+          if param.text.nil? || param.text == ''
+            docs_array << "_@param_ `#{param.name}`"
+          else
+            docs_array << "_@param_ `#{param.name}` — #{param.text.gsub("\n", " ")}"
+          end
+        end
+
+        # Add @return tags (there could possibly be more than one, despite this not being supported)
+        returns = parser.tags.select { |tag| tag.tag_name == 'return' && tag.is_a?(YARD::Tags::Tag) && !tag.text.nil? && tag.text.strip != '' }
+        # Add a blank line if there's anything before the returns.
+        docs_array << '' if docs_array.length.positive? && returns.length.positive?
+        returns.each do |retn|
+          docs_array << '' if docs_array.last != '' && docs_array.length.positive?
+          # Output returns in the form of:
+          # _@return_ — Lorem ipsum.
+          docs_array << "_@return_ — #{retn.text}"
+        end
+
+        # Iterate through the @example tags for a given YARD doc and output them in Markdown codeblocks.
+        examples = parser.tags.select { |tag| tag.tag_name == 'example' && tag.is_a?(YARD::Tags::Tag) }
+        examples.each do |example|
+          # Only add a blank line if there's anything before the example.
+          docs_array << '' if docs_array.length.positive?
+          # Include the example's 'name' if there is one.
+          docs_array << example.name unless example.name.nil? || example.name == ""
+          docs_array << "```ruby"
+          docs_array.concat(example.text.split("\n"))
+          docs_array << "```"
+        end if examples.length.positive?
+
+        # Add @note and @deprecated tags.
+        notice_tags = parser.tags.select { |tag| ['note', 'deprecated'].include?(tag.tag_name) && tag.is_a?(YARD::Tags::Tag) }
+        # Add a blank line if there's anything before the params.
+        docs_array << '' if docs_array.last != '' && docs_array.length.positive? && notice_tags.length.positive?
+        notice_tags.each do |notice_tag|
+          docs_array << '' if docs_array.last != ''
+          # Output note/deprecated/see in the form of:
+          # _@note_ — Lorem ipsum.
+          # _@note_
+          if notice_tag.text.nil?
+            docs_array << "_@#{notice_tag.tag_name}_"
+          else
+            docs_array << "_@#{notice_tag.tag_name}_ — #{notice_tag.text}"
+          end
+        end
+
+        # Add @see tags.
+        see_tags = parser.tags.select { |tag| tag.tag_name == 'see' && tag.is_a?(YARD::Tags::Tag) }
+        # Add a blank line if there's anything before the params.
+        docs_array << '' if docs_array.last != '' && docs_array.length.positive? && see_tags.length.positive?
+        see_tags.each do |see_tag|
+          docs_array << '' if docs_array.last != ''
+          # Output note/deprecated/see in the form of:
+          # _@see_ `B` — Lorem ipsum.
+          # _@see_ `B`
+          if see_tag.text.nil?
+            docs_array << "_@see_ `#{see_tag.name}`"
+          else
+            docs_array << "_@see_ `#{see_tag.name}` — #{see_tag.text}"
+          end
+        end
+
+        # fix: yard text may contains multiple line. should deal \n.
+        # else generate text will be multiple line and only first line is commented
+        docs_array = docs_array.flat_map {|line| line.empty? ? [""] : line.split("\n")}
+        typed_object.add_comments(docs_array)
+      end
+    end
+
+    # Given a YARD NamespaceObject, add lines defining its methods and their
+    # signatures to the current file.
+    # @param [YARD::CodeObjects::NamespaceObject] item
+    # @return [void]
+    def add_methods(item)
+      item.meths(inherited: false).each do |meth|
+        count_method
+
+        # If the method is an alias, skip it so we don't define it as a
+        # separate method.
+        if meth.is_alias?
+          next
+        end
+
+        # If the method is an attribute, it'll be handled by add_attributes
+        if meth.is_attribute?
+          next
+        end
+      
+        # Sort parameters
+        meth.parameters.reverse.sort! { |pair1, pair2| sort_params(pair1, pair2) }
+        # This is better than iterating over YARD's "@param" tags directly 
+        # because it includes parameters without documentation
+        # (The gsubs allow for better splat-argument compatibility)
+        parameter_names_and_defaults_to_tags = meth.parameters.map do |name, default|
+          [[name, default], meth.tags('param')
+            .find { |p| p.name&.gsub('*', '')&.gsub(':', '') == name.gsub('*', '').gsub(':', '') }]
+        end.to_h
+
+        parameter_types = parameter_names_and_defaults_to_tags.map do |name_and_default, tag|
+          name = name_and_default.first
+
+          if tag
+            TypeConverter.yard_to_parlour(tag.types, meth, @replace_errors_with_untyped, @replace_unresolved_with_untyped)
+          elsif name.start_with? '&'
+            # Find yieldparams and yieldreturn
+            yieldparams = meth.tags('yieldparam')
+            yieldreturn = meth.tag('yieldreturn')&.types
+            yieldreturn = nil if yieldreturn&.length == 1 &&
+              yieldreturn&.first&.downcase == 'void'
+
+            # Create strings
+            params = yieldparams.map do |param|
+              Parlour::Types::Proc::Parameter.new(
+                param.name.gsub('*', ''),
+                TypeConverter.yard_to_parlour(param.types, meth, @replace_errors_with_untyped, @replace_unresolved_with_untyped)
+              )
+            end
+            returns = TypeConverter.yard_to_parlour(yieldreturn, meth, @replace_errors_with_untyped, @replace_unresolved_with_untyped)
+
+            # Create proc types, if possible
+            if yieldparams.empty? && yieldreturn.nil?
+              Parlour::Types::Untyped.new
+            else
+              Parlour::Types::Proc.new(params, yieldreturn.nil? ? nil : returns)
+            end
+          elsif meth.path.end_with? '='
+            # Look for the matching getter method
+            getter_path = meth.path[0...-1]
+            getter = item.meths.find { |m| m.path == getter_path }
+
+            unless getter
+              if parameter_names_and_defaults_to_tags.length == 1 \
+                && meth.tags('param').length == 1 \
+                && meth.tag('param').types
+  
+                Logging.infer("argument name in single @param inferred as #{parameter_names_and_defaults_to_tags.first.first.first.inspect}", meth)
+                next TypeConverter.yard_to_parlour(meth.tag('param').types, meth, @replace_errors_with_untyped, @replace_unresolved_with_untyped)
+              else  
+                Logging.omit("no YARD type given for #{name.inspect}, using untyped", meth)
+                next Parlour::Types::Untyped.new
+              end
+            end
+
+            return_types = getter.tags('return').flat_map(&:types)
+            unless return_types.any?
+              Logging.omit("no YARD type given for #{name.inspect}, using untyped", meth)
+              next Parlour::Types::Untyped.new
+            end
+            inferred_type = TypeConverter.yard_to_parlour(
+              return_types, meth, @replace_errors_with_untyped, @replace_unresolved_with_untyped)
+            
+            Logging.infer("inferred type of parameter #{name.inspect} as #{inferred_type.describe} using getter's return type", meth)
+            inferred_type
+          else
+            # Is this the only argument, and was a @param specified without an
+            # argument name? If so, infer it
+            if parameter_names_and_defaults_to_tags.length == 1 \
+              && meth.tags('param').length == 1 \
+              && meth.tag('param').types
+
+              Logging.infer("argument name in single @param inferred as #{parameter_names_and_defaults_to_tags.first.first.first.inspect}", meth)
+              TypeConverter.yard_to_parlour(meth.tag('param').types, meth, @replace_errors_with_untyped, @replace_unresolved_with_untyped)
+            else
+              Logging.omit("no YARD type given for #{name.inspect}, using untyped", meth)
+              Parlour::Types::Untyped.new
+            end
+          end
+        end
+
+        return_tags = meth.tags('return')
+        returns = if meth.name == :initialize && !@use_original_initialize_return
+          nil
+        elsif return_tags.length == 0
+          Logging.omit("no YARD return type given, using untyped", meth)
+          Parlour::Types::Untyped.new
+        elsif return_tags.length == 1 && %w{void nil}.include?(return_tags&.first&.types&.first&.downcase)
+          nil
+        else
+          TypeConverter.yard_to_parlour(meth.tag('return').types, meth, @replace_errors_with_untyped, @replace_unresolved_with_untyped)
+        end
+
+        rbs_block = nil
+        parlour_params = parameter_names_and_defaults_to_tags
+          .zip(parameter_types)
+          .map do |((name, default), _), type|
+            # If the default is "nil" but the type is not nilable, then it 
+            # should become nilable
+            # (T.untyped can include nil, so don't alter that)
+            type = Parlour::Types::Nilable.new(type) \
+              if default == 'nil' && !type.is_a?(Parlour::Types::Nilable) && !type.is_a?(Parlour::Types::Untyped)
+
+            case @mode
+            when :rbi
+              Parlour::RbiGenerator::Parameter.new(
+                name.to_s,
+                type: type,
+                default: default
+              )
+            when :rbs
+              if name.start_with?('&')
+                rbs_block = type
+                nil
+              else
+                Parlour::RbsGenerator::Parameter.new(
+                  name.to_s,
+                  type: type,
+                  required: default.nil?
+                )
+              end
+            end
+          end
+          .compact
+
+        case @mode
+        when :rbi
+          @current_object.create_method(
+            meth.name.to_s, 
+            parameters: parlour_params,
+            returns: returns,
+            class_method: meth.scope == :class
+          ) do |m|
+            add_comments(meth, m)
+          end
+        when :rbs
+          @current_object.create_method(
+            meth.name.to_s,
+            [Parlour::RbsGenerator::MethodSignature.new(
+              parlour_params, returns, block: rbs_block && !rbs_block.is_a?(Parlour::Types::Untyped) \
+                ? Parlour::RbsGenerator::Block.new(rbs_block, false)
+                : nil
+            )],
+            class_method: meth.scope == :class
+          ) do |m|
+            add_comments(meth, m)
+          end
+        end  
+      end
+    end
+
+    # Given a YARD NamespaceObject, add lines defining either its class
+    # and instance attributes and their signatures to the current file.
+    # @param [YARD::CodeObjects::NamespaceObject] item
+    # @return [void]
+    def add_attributes(item)
+      [:class, :instance].each do |attr_loc|
+        # Grab attributes for the current location (class or instance)
+        attrs = item.attributes[attr_loc]
+        attrs.each do |name, attribute|
+          reader = attribute[:read]
+          writer = attribute[:write]
+
+          unless reader || writer
+            Logging.warn("attribute is not readable or writable somehow, skipping", attribute)
+            next
+          end
+
+          # Get all given types
+          yard_types = []
+          if reader
+            yard_types += reader.tags('return').flat_map(&:types).compact.reject { |x| x.downcase == 'void' } +
+              reader.tags('param').flat_map(&:types)
+          end
+          if writer
+            yard_types += writer.tags('return').flat_map(&:types).compact.reject { |x| x.downcase == 'void' } +
+              writer.tags('param').flat_map(&:types)
+          end
+
+          # Use untyped if not types specified anywhere, otherwise try to
+          # compute Parlour type given all these types
+          if yard_types.empty?
+            Logging.omit("no YARD type given for #{name.inspect}, using untyped", reader || writer)
+            parlour_type = Parlour::Types::Untyped.new
+          elsif yard_types.all? { |x| x == 'nil' }
+            # Nil attributes are extremely unusual, so just use untyped
+            parlour_type = Parlour::Types::Untyped.new
+          else
+            parlour_type = TypeConverter.yard_to_parlour(
+              yard_types, reader || writer, @replace_errors_with_untyped, @replace_unresolved_with_untyped)
+          end
+
+          # Generate attribute
+          if reader && writer
+            kind = :accessor
+          elsif reader
+            kind = :reader
+          elsif writer
+            kind = :writer
+          end
+
+          case @mode
+          when :rbi
+            @current_object.create_attribute(
+              name.to_s,
+              kind: kind,
+              type: parlour_type,
+              class_attribute: (attr_loc == :class)
+            ) do |m|
+              add_comments(reader || writer, m)
+            end
+          when :rbs
+            if attr_loc == :class
+              Logging.warn("RBS doesn't support class attributes, dropping", reader || writer)
+            else
+              @current_object.create_attribute(
+                name.to_s,
+                kind: kind,
+                type: parlour_type,
+              ) do |m|
+                add_comments(reader || writer, m)
+              end
+            end
+          end
+        end
+      end
+    end
+
+    # Given a YARD NamespaceObject, add lines defining its mixins, methods
+    # and children to the file.
+    # @param [YARD::CodeObjects::NamespaceObject] item
+    # @return [void]
+    def add_namespace(item)
+      count_namespace
+
+      superclass = nil
+      superclass = item.superclass.path.to_s if item.type == :class && item.superclass.to_s != "Object"
+
+      parent = @current_object
+      @current_object = item.type == :class \
+        ? parent.create_class(item.name.to_s, superclass: superclass)
+        : parent.create_module(item.name.to_s)
+      @current_object.add_comments(item.docstring.all.split("\n"))
+
+      add_mixins(item)
+      add_methods(item)
+      add_attributes(item)
+      add_constants(item) unless @skip_constants
+
+      item.children.select { |x| [:class, :module].include?(x.type) }
+        .each { |child| add_namespace(child) }
+
+      @current_object = parent
+    end
+
+    # Populates the generator with the contents of the YARD registry. You 
+    # must load the YARD registry first!
+    # @return [void]
+    def populate
+      # Generate top-level modules, which recurses to all modules
+      YARD::Registry.root.children
+        .select { |x| [:class, :module].include?(x.type) }
+        .each { |child| add_namespace(child) }
+      end
+
+    # Populates the generator with the contents of the YARD registry, then
+    # uses the loaded Parlour::Generator to generate the file. You must
+    # load the YARD registry first!
+    # @return [void]
+    def generate
+      populate
+      @parlour.send(@mode)
+    end
+
+    # Loads the YARD registry, populates the file, and prints any relevant
+    # final logs.
+    # @return [void]
+    def run
+      # Get YARD ready
+      YARD::Registry.load!
+
+      # Populate the type information file
+      populate
+
+      if object_count.zero?
+        Logging.warn("No objects processed.")
+        Logging.warn("Have you definitely generated the YARD documentation for this project?")
+        Logging.warn("Run `yard` to generate docs.")
+      end
+
+      Logging.done("Processed #{object_count} objects (#{@namespace_count} namespaces and #{@method_count} methods)")
+
+      Logging.hooks.clear
+
+      unless warnings.empty?
+        Logging.warn("There were #{warnings.length} important warnings in the output file, listed below.")
+        if @replace_errors_with_untyped
+          Logging.warn("The types which caused them have been replaced with untyped.")
+        else
+          Logging.warn("The types which caused them have been replaced with SORD_ERROR_ constants.")
+        end
+        Logging.warn("Please edit the file to fix these errors.")
+        Logging.warn("Alternatively, edit your YARD documentation so that your types are valid and re-run Sord.")
+        warnings.each do |(msg, item, _)|
+          puts "        (#{Rainbow(item&.path).bold}) #{msg}"
+        end
+      end
+    rescue
+      Logging.error($!)
+      $@.each do |line|
+        puts "         #{line}"
+      end
+    end
+
+    # Given two pairs of arrays representing method parameters, in the form
+    # of ["variable_name", "default_value"], sort the parameters so they're
+    # valid for Sorbet. Sorbet requires that, e.g. required kwargs go before
+    # optional kwargs.
+    #
+    # @param [Array] pair1
+    # @param [Array] pair2
+    # @return Integer
+    def sort_params(pair1, pair2)
+      pair1_type, pair2_type = [pair1, pair2].map do |pair|
+        if pair[0].start_with?('&')
+          :blk
+        elsif pair[0].start_with?('**')
+          :doublesplat
+        elsif pair[0].start_with?('*')
+          :splat
+        elsif !pair[0].end_with?(':') && pair[1].nil?
+          :required_ordered_param
+        elsif !pair[0].end_with?(':') && !pair[1].nil?
+          :optional_ordered_param
+        elsif pair[0].end_with?(':') && pair[1].nil?
+          :required_kwarg
+        elsif pair[0].end_with?(':') && !pair[1].nil?
+          :optional_kwarg
+        end
+      end
+
+      pair_type_order = {
+        required_ordered_param: 1,
+        optional_ordered_param: 2,
+        splat: 3,
+        required_kwarg: 4,
+        optional_kwarg: 5,
+        doublesplat: 6,
+        blk: 7
+      }
+
+      return pair_type_order[pair1_type] <=> pair_type_order[pair2_type]
+    end
+  end
+end