docrb-parser 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bfa4a19237733b6e0ace1d9af8584eeeb1dc671ec3a08c000d2b5f99adfcfa90
+  data.tar.gz: 02e8182ff82e529c89c99655411a7980a56c89a511263923616468c1bbbc0f65
+SHA512:
+  metadata.gz: '048771cd5044806674962f019c76fa4fedbd5f2778ff461b3b7fd4eef77076bf93b4562283b5667318d0f68d99d7c816dd77b6a803c26fe8859f6f8358075ea7'
+  data.tar.gz: ba979ca0cb853ec2a3ef2542957f887f0b5b9e92355e380e1006bfd6883e06506b65ed49c3ed867116f7f11bfc9e32470b595a140e2917d669f5b407667e579c

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,75 @@
+AllCops:
+  TargetRubyVersion: 3.2
+  SuggestExtensions: false
+  NewCops: enable
+  Exclude:
+    - spec/fixtures/*.rb
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120
+  Exclude:
+    - spec/**/**
+
+Naming/VariableNumber:
+  Exclude:
+    - spec/**/**
+
+Layout/FirstHashElementIndentation:
+  EnforcedStyle: consistent
+
+Layout/EndAlignment:
+  EnforcedStyleAlignWith: start_of_line
+
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+
+Style/Documentation:
+  Enabled: false
+
+Layout/CaseIndentation:
+  EnforcedStyle: end
+
+Layout/FirstArgumentIndentation:
+  EnforcedStyle: consistent_relative_to_receiver
+
+Layout/ArgumentAlignment:
+  EnforcedStyle: with_fixed_indentation
+
+Style/EmptyCaseCondition:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Naming/PredicateName:
+  Enabled: false
+
+Naming/FileName:
+  Exclude:
+    - lib/docrb-parser.rb

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/docrb-parser.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "lib/docrb/parser/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "docrb-parser"
+  spec.version = Docrb::Parser::VERSION
+  spec.authors = ["Victor Gama"]
+  spec.email = ["hey@vito.io"]
+
+  spec.summary = "Docrb's Ruby Parser"
+  spec.description = <<~DESC
+    docrb-parser is responsible for parsing Ruby sources into a structured#{" "}
+    format for usage by docrb and docrb-html.
+  DESC
+  spec.homepage = "https://github.com/heyvito/docrb"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.2"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "#{spec.homepage}/tree/trunk/lib/docrb-parser"
+  spec.metadata["changelog_uri"] = spec.homepage
+  spec.metadata["rubygems_mfa_required"] = "true"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "prism", "~> 0.13"
+end

data/lib/docrb/core_extensions.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class Object
+  def own_methods = methods.sort - Object.methods
+  def object_id_hex = "0x#{object_id.to_s(16).rjust(16, "0")}"
+
+  def self.docrb_inspect(&)
+    return if @__inspect__installed__
+
+    @__inspect__installed__ = true
+    define_method(:to_s) { "<#{self.class.name}:#{object_id_hex} #{instance_exec(&)}>" }
+    define_method(:inspect) { to_s }
+  end
+
+  def try(method, *, **, &)
+    return nil unless respond_to? method
+
+    send(method, *, **, &)
+  end
+
+  def attr_list(*names)
+    names.map { "#{_1}: #{send(_1).inspect}" }.join(", ")
+  end
+
+  def self.docrb_inspect_attrs(*)
+    @inspectable_attrs = superclass.instance_variable_get(:@inspectable_attrs).dup || [] if @inspectable_attrs.nil?
+    @inspectable_attrs.append(*)
+    docrb_inspect { attr_list(*self.class.instance_variable_get(:@inspectable_attrs)) }
+  end
+
+  def self.visible_attr_reader(*)
+    attr_reader(*)
+
+    docrb_inspect_attrs(*)
+  end
+
+  def self.visible_attr_accessor(*)
+    attr_accessor(*)
+
+    docrb_inspect_attrs(*)
+  end
+end
+
+class Array
+  def first!
+    first or raise("#first! called on empty array")
+  end
+
+  alias old_first first
+
+  def first(*, **, &)
+    return old_first(*, **) unless block_given?
+
+    lazy.map(&).filter(&:itself).first
+  end
+end
+
+module Kernel
+  def then! = nil? ? nil : yield(self)
+end

data/lib/docrb/parser/attribute.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Docrb
+  class Parser
+    class Attribute
+      visible_attr_reader :name, :location
+      visible_attr_accessor :writer_visibility, :reader_visibility, :type
+      attr_accessor :parent, :doc
+
+      def initialize(parser, parent, node, name, type)
+        @object_id = parser.make_id(self)
+        @name = name
+        @parent = parent
+        @location = parser.location(node.location)
+        @type = type
+        (parent.current_visibility_modifier || :public).tap do |vis|
+          @writer_visibility = vis
+          @reader_visibility = vis
+        end
+      end
+
+      def id = @object_id
+    end
+  end
+end

data/lib/docrb/parser/call.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Docrb
+  class Parser
+    class Call
+      visible_attr_reader :name, :arguments, :parent, :location
+
+      def initialize(parser, parent, node)
+        @object_id = parser.make_id(self)
+        @name = node.name.to_sym
+        @arguments = []
+        @parent = parent
+        @location = parser.location(node.location)
+        node.arguments&.arguments&.each do |arg|
+          @arguments << case arg.type
+          when :constant_path_node, :constant_read_node
+            parser.unfurl_constant_path(arg)
+          else
+            arg
+          end
+        end
+      end
+
+      def id = @object_id
+    end
+  end
+end

data/lib/docrb/parser/class.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Docrb
+  class Parser
+    class Class < Container
+      visible_attr_accessor :inherits, :singleton
+      attr_accessor :node
+
+      def kind = :class
+
+      def initialize(parser, parent, node)
+        @default_constructor_visibility = :public
+
+        # WARNING: super WILL CALL methods that may require ivars to already be
+        # defined. Define those ivars before this point.
+        super
+
+        @inherits = if node&.type == :class_node && !node.superclass.nil?
+          reference(parser.unfurl_constant_path(node.superclass))
+        end
+
+        update_constructor_visibility!
+        adjust_split_attributes! :class
+        adjust_split_attributes! :instance
+      end
+
+      def unowned_classes
+        super.tap do |arr|
+          arr.merge_unowned(*@inherits.dereference!.all_classes) if @inherits&.fulfilled?
+        end
+      end
+
+      def unowned_modules
+        super.tap do |arr|
+          arr.merge_unowned(*@inherits.dereference!.unowned_modules) if @inherits&.fulfilled?
+        end
+      end
+
+      def unowned_instance_methods
+        super.tap do |arr|
+          arr.merge_unowned(*@inherits.dereference!.all_instance_methods) if @inherits&.fulfilled?
+        end
+      end
+
+      def unowned_class_methods
+        super.tap do |arr|
+          arr.merge_unowned(*@inherits.dereference!.unowned_class_methods) if @inherits&.fulfilled?
+        end
+      end
+
+      def unowned_class_attributes
+        super.tap do |arr|
+          arr.merge_unowned(*@inherits.dereference!.unowned_class_attributes) if @inherits&.fulfilled?
+        end
+      end
+
+      def unowned_instance_attributes
+        super.tap do |arr|
+          arr.merge_unowned(*@inherits.dereference!.unowned_instance_attributes) if @inherits&.fulfilled?
+        end
+      end
+
+      def is_inherited?(obj, parent)
+        return false if parent.nil? || !parent.fulfilled?
+        return true if parent.resolved.id == obj.parent.id
+
+        is_inherited?(obj, parent.dereference!.inherits)
+      end
+
+      def source_of(obj)
+        return :inherited if is_inherited?(obj, inherits)
+
+        super
+      end
+
+      def singleton! = tap { @singleton = true }
+
+      def singleton? = @singleton || false
+
+      def handle_parsed_node(parser, node) = parser.unhandled_node! node
+
+      def merge_singleton_class(other)
+        raise ArgumentError, "Cannot merge non-singleton class #{other.name} into #{name}" unless other.singleton?
+
+        class_methods.append(*other.instance_methods)
+        class_attributes.append(*other.instance_attributes)
+        return if other.location.try(:virtual?)
+
+        @defined_by << other.location
+        @location = other.location if @location.nil? || @location.try(:virtual)
+      end
+    end
+  end
+end

data/lib/docrb/parser/comment.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Docrb
+  class Parser
+    class Comment
+      attr_reader :comments
+
+      def initialize(parser, location)
+        @location = location
+        @file_path = location.file_path
+        @parser = parser
+        @comments = nil
+        locate
+      end
+
+      def locate
+        return if @location.virtual?
+
+        lines = @parser.lines_for(@file_path, @location.ast)
+
+        # NOTE: Regarding -2, -1 since `lines` is zero-indexed, and another -1
+        # to get the line before the current location
+        offset = @location.line_start - 2
+        comments = []
+
+        until offset.zero?
+          line = lines[offset]
+          break if line.nil?
+
+          break unless line.strip.start_with? "#"
+
+          comments << line
+          offset -= 1
+        end
+
+        @comments = comments.reverse.map(&:strip)
+      end
+    end
+  end
+end

data/lib/docrb/parser/comment_parser.rb

@@ -0,0 +1,290 @@
+# frozen_string_literal: true
+
+module Docrb
+  class Parser
+    # CommentParser implements a small parser for matching comment's contents to
+    # relevant references and annotations.
+    class CommentParser
+      NEWLINE = "\n"
+      POUND = "#"
+      SPACE = " "
+      DASH = "-"
+      COLON = ":"
+
+      attr_accessor :objects, :current_object, :cursor, :visibility
+
+      def self.parse(data)
+        new(data)
+          .tap(&:parse)
+          .then do |parser|
+          { meta: { visibility: parser.visibility }.compact, value: parser.objects }
+        end
+      end
+
+      def initialize(data)
+        @objects = []
+        @current_object = []
+        @data = data
+          .split(NEWLINE)
+          .map(&:rstrip)
+          .map { _1.gsub(/^\s*#\s?/, "") }
+          .join(NEWLINE)
+          .each_grapheme_cluster
+          .to_a
+        @data_len = @data.length
+        @visibility = nil
+
+        @cursor = 0
+      end
+
+      def at_end? = (cursor >= @data_len)
+
+      def will_end? = (cursor + 1 >= @data_len)
+
+      def at_start? = cursor.zero?
+
+      def peek = at_end? ? nil : @data[cursor]
+
+      def peek_next = will_end? ? nil : @data[cursor + 1]
+
+      def peek_prev = at_start? ? nil : @data[cursor - 1]
+
+      def advance = at_end? ? nil : peek.tap { self.cursor += 1 }
+
+      def match?(*args) = args.any? { _1 == peek }
+
+      def consume_spaces = (advance while match?(SPACE) && !at_end?)
+
+      def extract_while = (current_object << advance while yield && !at_end?)
+
+      def extract_until
+        until at_end?
+          break if yield
+
+          current_object << advance
+        end
+      end
+
+      def parse
+        parse_one until at_end?
+        flush_current_object
+        detect_field_list
+        process_code_examples
+        process_text_blocks
+        process_visibility
+        objects.map! { normalize_tree(_1) }
+        true
+      end
+
+      def flush_current_object
+        data = current_object.join.rstrip
+        return if data.empty?
+
+        objects << data
+        current_object.clear
+      end
+
+      def parse_one
+        extract_until { match? NEWLINE }
+        advance # Consume newline
+        if match? NEWLINE
+          advance # consume newline
+          flush_current_object
+          return
+        end
+        current_object << peek_prev if peek_prev == NEWLINE
+      end
+
+      FIELD_LIST_HEADING = /^([a-z][a-z_0-9]*:?)\s+-\s+(.*)/
+
+      def detect_field_list
+        objects.each.with_index do |obj, idx|
+          definitions = obj.split("\n").reject { _1.start_with? SPACE }
+
+          if (definitions.length == 1 && definitions.first =~ FIELD_LIST_HEADING) ||
+             (definitions.first =~ FIELD_LIST_HEADING && definitions[1] =~ FIELD_LIST_HEADING)
+            return process_field_list(obj, idx)
+          end
+        end
+      end
+
+      def process_field_list(obj, at)
+        lines = obj.lines
+        result = {}
+        last_key = nil
+        lines.each do |line|
+          if (match = FIELD_LIST_HEADING.match(line))
+            last_key = match[1]
+            contents = match[2]
+            result[last_key] = contents
+          elsif last_key
+            result[last_key] = "#{result[last_key]} #{line.lstrip}"
+          end
+        end
+        objects[at] = { type: :fields, value: result }
+      end
+
+      def process_text_blocks
+        objects.each.with_index do |obj, idx|
+          next objects[idx] = process_text_block(obj) if obj.is_a? String
+
+          case obj[:type]
+          when :fields
+            obj[:value].transform_values! { process_text_block(_1) }
+          when :code_example then next
+          else
+            raise NotImplementedError, "Can't process text block for type #{obj[:type]}"
+          end
+        end
+      end
+
+      def span(text) = { type: :span, value: text }
+
+      def process_text_block(text)
+        objs = [span(text)]
+        changed = true
+        while changed
+          changed = false
+          objs.each.with_index do |obj, idx|
+            next unless obj[:type] == :span
+
+            value = obj[:value]
+            changes = extract_method_reference(value) ||
+                      extract_symbol(value) ||
+                      extract_camelcase_identifier(value)
+            next unless changes
+
+            changes => { start_idx:, end_idx:, object: }
+            objs.delete_at(idx)
+            left = value[0...start_idx]
+            right = value[end_idx...]
+
+            new_items = [
+              (span(left) unless left.empty?),
+              object,
+              (span(right) unless right.empty?)
+            ]
+            objs.insert(idx, *new_items.compact)
+            changed = true
+            break
+          end
+        end
+        objs.length == 1 ? objs.first : objs
+      end
+
+      # rubocop:disable Layout/LineLength
+      COMMENT_METHOD_REF_REGEXP = /(?:([A-Z][a-zA-Z0-9_]*::)*([A-Z][a-zA-Z0-9_]*))?(::|\.|#)([A-Za-z_][a-zA-Z0-9_@]*[!?]?)(?:\([a-zA-Z0-9=_,\s*]+\))?/
+      # rubocop:enable Layout/LineLength
+
+      def extract_method_reference(text)
+        match = COMMENT_METHOD_REF_REGEXP.match(text) or return nil
+        value, class_path, target, invocation, name = match.to_a
+        class_path&.gsub!(/::$/, "")
+
+        {
+          start_idx: match.begin(0),
+          end_idx: match.end(0),
+          object: {
+            type: invocation == POUND ? :method_ref : :class_path_ref,
+            class_path:,
+            target:,
+            name:,
+            value:
+          }
+        }
+      end
+
+      COMMENT_SYMBOL_REGEXP = /:(!|[@$][a-z_][a-z0-9_]*|[a-z_][a-z0-9_]*|[a-z_][a-z0-9_]*[?!]?)/i
+
+      def extract_symbol(text)
+        match = COMMENT_SYMBOL_REGEXP.match(text) or return nil
+
+        {
+          start_idx: match.begin(0),
+          end_idx: match.end(0),
+          object: {
+            type: :symbol,
+            value: match[0]
+          }
+        }
+      end
+
+      CAMELCASE_IDENTIFIER_REGEXP = /[A-Z][a-z]+(?:[A-Z][a-z]+)+/
+
+      def extract_camelcase_identifier(text)
+        match = CAMELCASE_IDENTIFIER_REGEXP.match(text) or return nil
+
+        {
+          start_idx: match.begin(0),
+          end_idx: match.end(0),
+          object: {
+            type: :identifier,
+            value: match[0]
+          }
+        }
+      end
+
+      VISIBILITY_INDICATOR_REGEXP = /^\s*(public|private|internal|deprecated|protected):\s+/i
+
+      def process_visibility(obj = nil)
+        obj ||= objects.first
+        case obj
+        when Array then process_visibility(obj.first)
+        when Hash
+          return if obj[:type] == :fields
+          return process_visibility(obj[:value]) unless obj[:type] == :span
+
+          value = obj[:value]
+          match = VISIBILITY_INDICATOR_REGEXP.match(value) or return nil
+          obj[:value] = value[match.end(0)...]
+          @visibility = match[1]
+          nil
+        end
+      end
+
+      def process_code_examples
+        changed = true
+        while changed
+          start_at = nil
+          changed = false
+          objects.each.with_index do |obj, idx|
+            is_code = (obj.is_a?(String) && obj.start_with?("  "))
+            next start_at = idx if is_code && start_at.nil?
+
+            if !is_code && start_at
+              join_code_example_lines(start_at, idx)
+              start_at = nil
+              changed = true
+              break
+            end
+          end
+          join_code_example_lines(start_at, objects.length) unless start_at.nil?
+        end
+      end
+
+      def join_code_example_lines(start_at, end_at)
+        lines = objects[start_at...end_at]
+          .map { _1.split("\n") }
+          .map { |el| el.map { "#{_1[2...]}\n" } }
+          .flatten
+        objects.slice!(start_at...end_at)
+        objects.insert(start_at, {
+          type: :code_example,
+          source: lines.join("\n")
+        })
+      end
+
+      def normalize_tree(obj)
+        if obj.is_a?(Array)
+          { type: :block, value: obj }
+        elsif obj.is_a?(Hash) && obj[:type] == :span
+          { type: :block, value: [obj] }
+        elsif obj.is_a?(Hash) && obj[:type] == :fields
+          obj.tap { |f| f[:value].transform_values! { normalize_tree(_1) } }
+        else
+          obj
+        end
+      end
+    end
+  end
+end