hyper-pro-gem 0.0.1

data/yard-0.9.44/lib/yard/parser/ruby/token_resolver.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+module YARD
+  module Parser
+    module Ruby
+      # Supports {#each} enumeration over a source's tokens, yielding
+      # the token and a possible {CodeObjects::Base} associated with the
+      # constant or identifier token.
+      class TokenResolver
+        include Enumerable
+        include CodeObjects::NamespaceMapper
+
+        # Creates a token resolver for given source.
+        #
+        # @param source [String] the source code to tokenize
+        # @param namespace [CodeObjects::Base] the object/namespace to resolve from
+        def initialize(source, namespace = Registry.root)
+          @tokens = RubyParser.parse(source, '(tokenize)').tokens
+          raise ParserSyntaxError if @tokens.empty? && !source.empty?
+          @default_namespace = namespace
+        end
+
+        # Iterates over each token, yielding the token and a possible code
+        # object that is associated with the token.
+        #
+        # @yieldparam token [Array(Symbol,String,Array(Integer,Integer))] the
+        #   current token object being iterated
+        # @yieldparam object [CodeObjects::Base, nil] the fully qualified code
+        #   object associated with the current token, or nil if there is no object
+        #   for the yielded token.
+        # @example Yielding code objects
+        #   r = TokenResolver.new("A::B::C")
+        #   r.each do |tok, obj|
+        #     if obj
+        #       puts "#{tok[0]} -> #{obj.path.inspect}"
+        #     else
+        #       puts "No object: #{tok.inspect}"
+        #     end
+        #   end
+        #
+        #   # Prints:
+        #   # :const -> "A"
+        #   # No object: [:op, "::"]
+        #   # :const -> "A::B"
+        #   # No object: [:op, "::"]
+        #   # :const -> "A::B::C"
+        def each
+          @states = []
+          push_state
+          @tokens.each do |token|
+            yield_obj = false
+
+            if skip_group && [:const, :ident, :op, :period].include?(token[0])
+              yield token, nil
+              next
+            else
+              self.skip_group = false
+            end
+
+            case token[0]
+            when :const
+              lookup(token[0], token[1])
+              yield_obj = true
+              self.last_sep = nil
+            when :ident
+              lookup(token[0], token[1])
+              yield_obj = true
+              self.last_sep = nil
+            when :op, :period
+              self.last_sep = token[1]
+              unless CodeObjects.types_for_separator(token[1])
+                self.object = nil
+                self.last_sep = nil
+              end
+            when :lparen
+              push_state
+            when :rparen
+              pop_state
+            else
+              self.object = nil
+            end
+
+            yield token, (yield_obj ? object : nil)
+
+            if next_object
+              self.object = next_object
+              self.next_object = nil
+            end
+            self.skip_group = true if yield_obj && object.nil?
+          end
+        end
+
+        def self.state_attr(*attrs)
+          attrs.each do |attr|
+            define_method(attr) { @states.last[attr.to_sym] }
+            define_method("#{attr}=") {|v| @states.last[attr.to_sym] = v }
+            protected attr, :"#{attr}="
+          end
+        end
+
+        private
+
+        def push_state
+          @states.push :object => nil, :skip_group => false, :last_sep => nil
+        end
+
+        def pop_state
+          @states.pop
+        end
+
+        state_attr :object, :next_object, :skip_group, :last_sep
+
+        def lookup(toktype, name)
+          types = object_resolved_types
+          return self.object = nil if types.empty?
+
+          if toktype == :const
+            types.any? do |type|
+              prefix = type ? type.path : ""
+              prefix += last_sep.to_s if separators.include?(last_sep.to_s)
+              self.object = Registry.resolve(@default_namespace, "#{prefix}#{name}", true)
+            end
+          else # ident
+            types.any? do |type|
+              obj = Registry.resolve(type, name, true)
+              if obj.nil? && name == "new"
+                obj = Registry.resolve(object, "#initialize", true)
+                self.next_object = object if obj.nil?
+              end
+              self.object = obj
+            end
+          end
+        end
+
+        def object_resolved_types(obj = object)
+          return [obj] unless obj.is_a?(CodeObjects::MethodObject)
+
+          resolved_types = []
+          tags = obj.tags(:return)
+          tags += obj.tags(:overload).map {|o| o.tags(:return) }.flatten
+          tags.each do |tag|
+            next if tag.types.nil?
+            tag.types.each do |type|
+              type = type.sub(/<.+>/, '')
+              if type == "self"
+                resolved_types << obj.parent
+              else
+                type_obj = Registry.resolve(obj, type, true)
+                resolved_types << type_obj if type_obj
+              end
+            end
+          end
+
+          resolved_types
+        end
+      end
+    end
+  end
+end

data/yard-0.9.44/lib/yard/parser/source_parser.rb

@@ -0,0 +1,526 @@
+# frozen_string_literal: true
+require 'stringio'
+
+module YARD
+  module Parser
+    # Raised when an object is recognized but cannot be documented. This
+    # generally occurs when the Ruby syntax used to declare an object is
+    # too dynamic in nature.
+    class UndocumentableError < RuntimeError; end
+
+    # Raised when the parser sees a Ruby syntax error
+    class ParserSyntaxError < UndocumentableError; end
+
+    # Responsible for parsing a list of files in order. The
+    # {#parse} method of this class can be called from the
+    # {SourceParser#globals} globals state list to re-enter
+    # parsing for the remainder of files in the list recursively.
+    #
+    # @see Processor#parse_remaining_files
+    class OrderedParser
+      # @return [Array<String>] the list of remaining files to parse
+      attr_accessor :files
+
+      # Creates a new OrderedParser with the global state and a list
+      # of files to parse.
+      #
+      # @note OrderedParser sets itself as the +ordered_parser+ key on
+      #   global_state for later use in {Handlers::Processor}.
+      # @param [OpenStruct] global_state a structure containing all global
+      #   state during parsing
+      # @param [Array<String>] files the list of files to parse
+      def initialize(global_state, files)
+        @global_state = global_state
+        @files = files.dup
+        @global_state.ordered_parser = self
+      end
+
+      # Parses the remainder of the {#files} list.
+      #
+      # @see Processor#parse_remaining_files
+      def parse
+        until files.empty?
+          file = files.shift
+          log.capture("Parsing #{file}") do
+            SourceParser.new(SourceParser.parser_type, @global_state).parse(file)
+          end
+        end
+      end
+    end
+
+    # Responsible for parsing a source file into the namespace. Parsing
+    # also invokes handlers to process the parsed statements and generate
+    # any code objects that may be recognized.
+    #
+    # == Custom Parsers
+    # SourceParser allows custom parsers to be registered and called when
+    # a certain filetype is recognized. To register a parser and hook it
+    # up to a set of file extensions, call {register_parser_type}
+    #
+    # @see register_parser_type
+    # @see Handlers::Base
+    # @see CodeObjects::Base
+    class SourceParser
+      SHEBANG_LINE  = /\A\s*#!\S+/
+      ENCODING_LINE = %r{\A(?:\s*#*!.*\r?\n)?\s*(?:#+|/\*+|//+).*coding\s*[:=]{1,2}\s*([a-z\d_\-]+)}i
+      FROZEN_STRING_LINE = /frozen(-|_)string(-|_)literal:\s+(true|false)/i
+
+      # The default glob of files to be parsed.
+      # @since 0.9.0
+      DEFAULT_PATH_GLOB = ["{lib,app}/**/*.{rb,rbs}", "sig/**/*.rbs", "ext/**/*.{c,cc,cxx,cpp,rb}"]
+
+      # Byte order marks for various encodings
+      # @since 0.7.0
+      ENCODING_BYTE_ORDER_MARKS = {
+        'utf-8' => String.new("\xEF\xBB\xBF"),
+        # Not yet supported
+        # 'utf-16be' => "\xFE\xFF",
+        # 'utf-16le' => "\xFF\xFE",
+        # 'utf-32be' => "\x00\x00\xFF\xFE",
+        # 'utf-32le' => "\xFF\xFE",
+      }
+
+      class << self
+        # @return [Symbol] the default parser type (defaults to :ruby)
+        attr_reader :parser_type
+
+        def parser_type=(value)
+          @parser_type = validated_parser_type(value)
+        end
+
+        # Parses a path or set of paths
+        #
+        # @param [String, Array<String>] paths a path, glob, or list of paths to
+        #   parse
+        # @param [Array<String, Regexp>] excluded a list of excluded path matchers
+        # @param [Fixnum] level the logger level to use during parsing. See
+        #   {YARD::Logger}
+        # @return [void]
+        def parse(paths = DEFAULT_PATH_GLOB, excluded = [], level = log.level)
+          log.debug("Parsing #{paths.inspect} with `#{parser_type}` parser")
+          excluded = excluded.map do |path|
+            case path
+            when Regexp; path
+            else Regexp.new(path.to_s, Regexp::IGNORECASE)
+            end
+          end
+          files = [paths].flatten.
+            map {|p| File.directory?(p) ? "#{p}/**/*.{rb,rbs,c,cc,cxx,cpp}" : p }.
+            map {|p| p.include?("*") ? Dir[p].sort_by {|d| [d.length, d] } : p }.flatten.
+            reject {|p| !File.file?(p) || excluded.any? {|re| p =~ re } }.
+            map {|p| p.encoding == Encoding.default_external ? p : p.dup.force_encoding(Encoding.default_external) }
+
+          log.enter_level(level) do
+            parse_in_order(*files.uniq)
+          end
+        end
+
+        # Parses a string +content+
+        #
+        # @param [String] content the block of code to parse
+        # @param [Symbol] ptype the parser type to use. See {parser_type}.
+        # @return the parser object that was used to parse +content+
+        def parse_string(content, ptype = parser_type)
+          new(ptype).parse(StringIO.new(content))
+        end
+
+        # Tokenizes but does not parse the block of code
+        #
+        # @param [String] content the block of code to tokenize
+        # @param [Symbol] ptype the parser type to use. See {parser_type}.
+        # @return [Array] a list of tokens
+        def tokenize(content, ptype = parser_type)
+          new(ptype).tokenize(content)
+        end
+
+        # Registers a new parser type.
+        #
+        # @example Registering a parser for "java" files
+        #   SourceParser.register_parser_type :java, JavaParser, 'java'
+        # @param [Symbol] type a symbolic name for the parser type
+        # @param [Base] parser_klass a class that implements parsing and tokenization
+        # @param [Array<String>, String, Regexp] extensions a list of extensions or a
+        #   regex to match against the file extension
+        # @return [void]
+        # @see Parser::Base
+        def register_parser_type(type, parser_klass, extensions = nil)
+          unless Base > parser_klass
+            raise ArgumentError, "expecting parser_klass to be a subclass of YARD::Parser::Base"
+          end
+          parser_type_extensions[type.to_sym] = extensions if extensions
+          parser_types[type.to_sym] = parser_klass
+        end
+
+        # @return [Hash{Symbol=>Object}] a list of registered parser types
+        # @private
+        # @since 0.5.6
+        def parser_types; @@parser_types ||= {} end
+        def parser_types=(value) @@parser_types = value end
+
+        # @return [Hash] a list of registered parser type extensions
+        # @private
+        # @since 0.5.6
+        def parser_type_extensions; @@parser_type_extensions ||= {} end
+        def parser_type_extensions=(value) @@parser_type_extensions = value end
+
+        # Finds a parser type that is registered for the extension. If no
+        # type is found, the default Ruby type is returned.
+        #
+        # @return [Symbol] the parser type to be used for the extension
+        # @since 0.5.6
+        def parser_type_for_extension(extension)
+          type = parser_type_extensions.find do |_t, exts|
+            [exts].flatten.any? {|ext| ext === extension }
+          end
+          validated_parser_type(type ? type.first : :ruby)
+        end
+
+        # Returns the validated parser type. Basically, enforces that :ruby
+        # type is never set if the Ripper library is not available
+        #
+        # @param [Symbol] type the parser type to set
+        # @return [Symbol] the validated parser type
+        # @private
+        def validated_parser_type(type)
+          !defined?(::Ripper) && type == :ruby ? :ruby18 : type
+        end
+
+        # @group Parser Callbacks
+
+        # Registers a callback to be called before a list of files is parsed
+        # via {parse}. The block passed to this method will be called on
+        # subsequent parse calls.
+        #
+        # @example Installing a simple callback
+        #   SourceParser.before_parse_list do |files, globals|
+        #     puts "Starting to parse..."
+        #   end
+        #   YARD.parse('lib/**/*.rb')
+        #   # prints "Starting to parse..."
+        #
+        # @example Setting global state
+        #   SourceParser.before_parse_list do |files, globals|
+        #     globals.method_count = 0
+        #   end
+        #   SourceParser.after_parse_list do |files, globals|
+        #     puts "Found #{globals.method_count} methods"
+        #   end
+        #   class MyCountHandler < Handlers::Ruby::Base
+        #     handles :def, :defs
+        #     process { globals.method_count += 1 }
+        #   end
+        #   YARD.parse
+        #   # Prints: "Found 37 methods"
+        #
+        # @example Using a global callback to cancel parsing
+        #   SourceParser.before_parse_list do |files, globals|
+        #     return false if files.include?('foo.rb')
+        #   end
+        #
+        #   YARD.parse(['foo.rb', 'bar.rb']) # callback cancels this method
+        #   YARD.parse('bar.rb') # parses normally
+        #
+        # @yield [files, globals] the yielded block is called once before
+        #   parsing all files
+        # @yieldparam [Array<String>] files the list of files that will be parsed.
+        # @yieldparam [OpenStruct] globals a global structure to store arbitrary
+        #   state for post processing (see {Handlers::Processor#globals})
+        # @yieldreturn [Boolean] if the block returns +false+, parsing is
+        #   cancelled.
+        # @return [Proc] the yielded block
+        # @see after_parse_list
+        # @see before_parse_file
+        # @since 0.7.0
+        def before_parse_list(&block)
+          before_parse_list_callbacks << block
+        end
+
+        # Registers a callback to be called after a list of files is parsed
+        # via {parse}. The block passed to this method will be called on
+        # subsequent parse calls.
+        #
+        # @example Printing results after parsing occurs
+        #   SourceParser.after_parse_list do
+        #     puts "Finished parsing!"
+        #   end
+        #   YARD.parse
+        #   # Prints "Finished parsing!" after parsing files
+        # @yield [files, globals] the yielded block is called once before
+        #   parsing all files
+        # @yieldparam [Array<String>] files the list of files that will be parsed.
+        # @yieldparam [OpenStruct] globals a global structure to store arbitrary
+        #   state for post processing (see {Handlers::Processor#globals})
+        # @yieldreturn [void] the return value for the block is ignored.
+        # @return [Proc] the yielded block
+        # @see before_parse_list
+        # @see before_parse_file
+        # @since 0.7.0
+        def after_parse_list(&block)
+          after_parse_list_callbacks << block
+        end
+
+        # Registers a callback to be called before an individual file is parsed.
+        # The block passed to this method will be called on subsequent parse
+        # calls.
+        #
+        # To register a callback that is called before the entire list of files
+        # is processed, see {before_parse_list}.
+        #
+        # @example Installing a simple callback
+        #   SourceParser.before_parse_file do |parser|
+        #     puts "I'm parsing #{parser.file}"
+        #   end
+        #   YARD.parse('lib/**/*.rb')
+        #   # prints:
+        #   "I'm parsing lib/foo.rb"
+        #   "I'm parsing lib/foo_bar.rb"
+        #   "I'm parsing lib/last_file.rb"
+        #
+        # @example Cancel parsing of any test_*.rb files
+        #   SourceParser.before_parse_file do |parser|
+        #     return false if parser.file =~ /^test_.+\.rb$/
+        #   end
+        #
+        # @yield [parser] the yielded block is called once before each
+        #   file that is parsed. This might happen many times for a single
+        #   codebase.
+        # @yieldparam [SourceParser] parser the parser object that will {#parse}
+        #   the file.
+        # @yieldreturn [Boolean] if the block returns +false+, parsing for
+        #   the file is cancelled.
+        # @return [Proc] the yielded block
+        # @see after_parse_file
+        # @see before_parse_list
+        # @since 0.7.0
+        def before_parse_file(&block)
+          before_parse_file_callbacks << block
+        end
+
+        # Registers a callback to be called after an individual file is parsed.
+        # The block passed to this method will be called on subsequent parse
+        # calls.
+        #
+        # To register a callback that is called after the entire list of files
+        # is processed, see {after_parse_list}.
+        #
+        # @example Printing the length of each file after it is parsed
+        #   SourceParser.after_parse_file do |parser|
+        #     puts "#{parser.file} is #{parser.contents.size} characters"
+        #   end
+        #   YARD.parse('lib/**/*.rb')
+        #   # prints:
+        #   "lib/foo.rb is 1240 characters"
+        #   "lib/foo_bar.rb is 248 characters"
+        #
+        # @yield [parser] the yielded block is called once after each file
+        #   that is parsed. This might happen many times for a single codebase.
+        # @yieldparam [SourceParser] parser the parser object that parsed
+        #   the file.
+        # @yieldreturn [void] the return value for the block is ignored.
+        # @return [Proc] the yielded block
+        # @see before_parse_file
+        # @see after_parse_list
+        # @since 0.7.0
+        def after_parse_file(&block)
+          after_parse_file_callbacks << block
+        end
+
+        # @return [Array<Proc>] the list of callbacks to be called before
+        #   parsing a list of files. Should only be used for testing.
+        # @since 0.7.0
+        def before_parse_list_callbacks
+          @before_parse_list_callbacks ||= []
+        end
+
+        # @return [Array<Proc>] the list of callbacks to be called after
+        #   parsing a list of files. Should only be used for testing.
+        # @since 0.7.0
+        def after_parse_list_callbacks
+          @after_parse_list_callbacks ||= []
+        end
+
+        # @return [Array<Proc>] the list of callbacks to be called before
+        #   parsing a file. Should only be used for testing.
+        # @since 0.7.0
+        def before_parse_file_callbacks
+          @before_parse_file_callbacks ||= []
+        end
+
+        # @return [Array<Proc>] the list of callbacks to be called after
+        #   parsing a file. Should only be used for testing.
+        # @since 0.7.0
+        def after_parse_file_callbacks
+          @after_parse_file_callbacks ||= []
+        end
+
+        # @endgroup
+
+        private
+
+        # Parses a list of files in a queue.
+        #
+        # @param [Array<String>] files a list of files to queue for parsing
+        # @return [void]
+        def parse_in_order(*files)
+          global_state = OpenStruct.new
+
+          return if before_parse_list_callbacks.any? do |cb|
+            cb.call(files, global_state) == false
+          end
+
+          OrderedParser.new(global_state, files).parse
+
+          after_parse_list_callbacks.each do |cb|
+            cb.call(files, global_state)
+          end
+        end
+      end
+
+      register_parser_type :ruby,   Ruby::RubyParser
+      register_parser_type :ruby18, Ruby::Legacy::RubyParser
+      register_parser_type :c,      C::CParser, ['c', 'cc', 'cxx', 'cpp']
+      register_parser_type :rbs,    RBS::RbsParser, ['rbs']
+
+      self.parser_type = :ruby
+
+      # @return [String] the filename being parsed by the parser.
+      attr_accessor :file
+
+      # @return [Symbol] the parser type associated with the parser instance.
+      #   This should be set by the {#initialize constructor}.
+      attr_reader :parser_type
+
+      # @return [OpenStruct] an open struct containing arbitrary global state
+      #   shared between files and handlers.
+      # @since 0.7.0
+      attr_reader :globals
+
+      # @return [String] the contents of the file to be parsed
+      # @since 0.7.0
+      attr_reader :contents
+
+      # @overload initialize(parser_type = SourceParser.parser_type, globals = nil)
+      #   Creates a new parser object for code parsing with a specific parser type.
+      #
+      #   @param [Symbol] parser_type the parser type to use
+      #   @param [OpenStruct] globals global state to be re-used across separate source files
+      def initialize(parser_type = SourceParser.parser_type, globals1 = nil, globals2 = nil)
+        globals = [true, false].include?(globals1) ? globals2 : globals1
+        @file = '(stdin)'
+        @globals = globals || OpenStruct.new
+        self.parser_type = parser_type
+      end
+
+      # The main parser method. This should not be called directly. Instead,
+      # use the class methods {parse} and {parse_string}.
+      #
+      # @param [String, #read, Object] content the source file to parse
+      # @return [Object, nil] the parser object used to parse the source
+      def parse(content = __FILE__)
+        case content
+        when String
+          @file = File.cleanpath(content)
+          content = convert_encoding(String.new(File.read_binary(file)))
+          checksum = Registry.checksum_for(content)
+          return if Registry.checksums[file] == checksum
+
+          if Registry.checksums.key?(file)
+            log.info "File '#{file}' was modified, re-processing..."
+          end
+          Registry.checksums[@file] = checksum
+          self.parser_type = parser_type_for_filename(file)
+        else
+          content = content.read if content.respond_to? :read
+        end
+
+        @contents = content
+        @parser = parser_class.new(content, file)
+
+        self.class.before_parse_file_callbacks.each do |cb|
+          return @parser if cb.call(self) == false
+        end
+
+        @parser.parse
+        post_process
+
+        self.class.after_parse_file_callbacks.each do |cb|
+          cb.call(self)
+        end
+
+        @parser
+      rescue ArgumentError, NotImplementedError => e
+        log.warn("Cannot parse `#{file}': #{e.message}")
+        log.backtrace(e, :warn)
+      rescue ParserSyntaxError => e
+        log.warn(e.message.capitalize)
+        log.backtrace(e, :warn)
+      end
+
+      # Tokenizes but does not parse the block of code using the current {#parser_type}
+      #
+      # @param [String] content the block of code to tokenize
+      # @return [Array] a list of tokens
+      def tokenize(content)
+        @parser = parser_class.new(content, file)
+        @parser.tokenize
+      end
+
+      private
+
+      # Searches for encoding line and forces encoding
+      # @since 0.5.3
+      def convert_encoding(content)
+        return content unless content.respond_to?(:force_encoding)
+        if content =~ ENCODING_LINE
+          content.force_encoding($1)
+        else
+          content.force_encoding('binary')
+          ENCODING_BYTE_ORDER_MARKS.each do |encoding, bom|
+            bom.force_encoding('binary')
+            if content.start_with?(bom)
+              return content.sub(bom, '').force_encoding(encoding)
+            end
+          end
+          content.force_encoding('utf-8') # UTF-8 is default encoding
+          content
+        end
+      end
+
+      # Runs a {Handlers::Processor} object to post process the parsed statements.
+      # @return [void]
+      def post_process
+        return unless @parser.respond_to?(:enumerator)
+
+        enumerator = @parser.enumerator
+        if enumerator
+          post = Handlers::Processor.new(self)
+          post.process(enumerator)
+        end
+      end
+
+      def parser_type=(value)
+        @parser_type = self.class.validated_parser_type(value)
+      end
+
+      # Guesses the parser type to use depending on the file extension.
+      #
+      # @param [String] filename the filename to use to guess the parser type
+      # @return [Symbol] a parser type that matches the filename
+      def parser_type_for_filename(filename)
+        ext = (File.extname(filename)[1..-1] || "").downcase
+        type = self.class.parser_type_for_extension(ext)
+        parser_type == :ruby18 && type == :ruby ? :ruby18 : type
+      end
+
+      # @since 0.5.6
+      def parser_class
+        klass = self.class.parser_types[parser_type]
+        unless klass
+          raise ArgumentError, "invalid parser type '#{parser_type}' or unrecognized file", caller[1..-1]
+        end
+
+        klass
+      end
+    end
+  end
+end