boondoc 0.1.0

data/lib/boondoc/frontend/visitor.rb

@@ -0,0 +1,359 @@
+# frozen_string_literal: true
+
+module Boondoc
+  module Frontend
+    # Walks a Prism tree and registers namespaces, methods, and constants.
+    #
+    # Method bodies are never entered, so only definitions made at class, module
+    # or top level are documented.
+    class Visitor < ::Prism::Visitor # rubocop:disable Metrics/ClassLength -- one handler per Ruby construct
+      # The state that applies while visiting one namespace body.
+      Frame = Struct.new(:namespace, :visibility, :singleton, :module_function)
+
+      CALLS = {
+        attr: :on_attr, attr_reader: :on_attr, attr_writer: :on_attr, attr_accessor: :on_attr,
+        public: :on_visibility, protected: :on_visibility, private: :on_visibility,
+        private_class_method: :on_class_visibility, public_class_method: :on_class_visibility,
+        module_function: :on_module_function, private_constant: :on_private_constant,
+        alias_method: :on_alias_method, define_method: :on_define_method,
+        include: :on_mixin, extend: :on_mixin, prepend: :on_mixin
+      }.freeze
+      ALWAYS_PRIVATE = %w[initialize_copy initialize_clone initialize_dup respond_to_missing?].freeze
+      ATTR_READERS_ONLY = %i[attr attr_reader].freeze
+      ANONYMOUS = { 'Struct' => :new, 'Data' => :define, 'Class' => :new, 'Module' => :new }.freeze
+
+      # @param registry [Model::Registry]
+      # @param file [String] file name recorded on each object
+      # @param comments [Comments]
+      def initialize(registry, file, comments)
+        super()
+        @registry = registry
+        @file = file
+        @comments = comments
+        @frames = [Frame.new(registry.root, :public, false, false)]
+        @visibility_override = nil
+      end
+
+      def visit_module_node(node)
+        open_namespace(node, :module, nil)
+      end
+
+      def visit_class_node(node)
+        open_namespace(node, :class, node.superclass&.slice)
+      end
+
+      def visit_singleton_class_node(node)
+        return unless node.expression.is_a?(::Prism::SelfNode)
+
+        with_frame(frame.namespace, singleton: true) { visit(node.body) }
+      end
+
+      def visit_def_node(node)
+        explicit = !node.receiver.nil?
+        scope = def_scope(node)
+        return unless scope
+
+        add_method(node.name.to_s, scope, line(node), parameters: node.parameters&.slice, explicit: explicit)
+      end
+
+      def visit_call_node(node)
+        handler = CALLS[node.name]
+        return super unless handler && (node.receiver.nil? || node.receiver.is_a?(::Prism::SelfNode))
+
+        send(handler, node)
+      end
+
+      def visit_alias_method_node(node)
+        return unless node.new_name.is_a?(::Prism::SymbolNode) && node.old_name.is_a?(::Prism::SymbolNode)
+
+        add_alias(node.new_name.unescaped, node.old_name.unescaped, line(node))
+      end
+
+      def visit_constant_write_node(node)
+        return if @comments.nodoc?(line(node))
+
+        value = node.value
+        return define_anonymous(node, value) if anonymous_namespace?(value)
+
+        constant = Model::ConstantObject.new(node.name.to_s, frame.namespace, value: value.slice)
+        @registry.register(constant)
+        document(constant, line(node))
+      end
+
+      private
+
+      def frame
+        @frames.last
+      end
+
+      def line(node)
+        node.location.start_line
+      end
+
+      # The scope of methods defined by calls such as `attr_reader` and `alias_method`.
+      def current_scope
+        frame.singleton ? :class : :instance
+      end
+
+      def find_method(scope, name)
+        @registry[Model::MethodObject.path_for(frame.namespace, scope, name)]
+      end
+
+      def with_frame(namespace, singleton: false)
+        @frames.push(Frame.new(namespace, :public, singleton, false))
+        yield
+      ensure
+        @frames.pop
+      end
+
+      def with_visibility(visibility)
+        previous = @visibility_override
+        @visibility_override = visibility
+        yield
+      ensure
+        @visibility_override = previous
+      end
+
+      # --- namespaces ---------------------------------------------------------
+
+      def open_namespace(node, kind, superclass)
+        return if @comments.nodoc?(line(node))
+
+        namespace = ensure_namespace(node.constant_path.slice, kind)
+        namespace.superclass ||= superclass
+        document(namespace, line(node))
+        with_frame(namespace) { visit(node.body) }
+      end
+
+      def ensure_namespace(name, kind)
+        parts = name.delete_prefix('::').split('::')
+        parent = namespace_base(name, parts)
+        parts.each_with_index do |part, index|
+          last = index == parts.size - 1
+          parent = child_namespace(parent, part, last ? kind : :module, placeholder: !last)
+        end
+        parent
+      end
+
+      # Where a (possibly qualified) name is looked up from: the root for `::Foo`,
+      # the namespace that already holds the first segment for `Foo::Bar`, and
+      # the current namespace otherwise.
+      def namespace_base(name, parts)
+        return @registry.root if name.start_with?('::')
+        return frame.namespace if parts.size == 1
+
+        @registry.resolve(frame.namespace, parts.first)&.namespace || frame.namespace
+      end
+
+      def child_namespace(parent, name, kind, placeholder:)
+        path = parent.root? ? name : "#{parent.path}::#{name}"
+        existing = @registry[path]
+        if existing.is_a?(Model::NamespaceObject)
+          existing.kind = kind unless placeholder
+          return existing
+        end
+
+        @registry.register(Model::NamespaceObject.new(name, parent, kind: kind))
+      end
+
+      def anonymous_namespace?(value)
+        value.is_a?(::Prism::CallNode) && value.receiver &&
+          ANONYMOUS[value.receiver.slice.delete_prefix('::')] == value.name
+      end
+
+      # `Foo = Struct.new(:a)`, `Foo = Data.define(:a)`, `Foo = Class.new(Base)`
+      def define_anonymous(node, value)
+        receiver = value.receiver.slice.delete_prefix('::')
+        namespace = ensure_namespace(node.name.to_s, receiver == 'Module' ? :module : :class)
+        namespace.superclass ||= anonymous_superclass(receiver, value)
+        document(namespace, line(node))
+        with_frame(namespace) do
+          define_members(receiver, value, line(node))
+          visit(value.block.body) if value.block.is_a?(::Prism::BlockNode)
+        end
+      end
+
+      def anonymous_superclass(receiver, value)
+        return receiver if %w[Struct Data].include?(receiver)
+        return nil unless receiver == 'Class'
+
+        first = arguments(value).first
+        first.slice if first.is_a?(::Prism::ConstantReadNode) || first.is_a?(::Prism::ConstantPathNode)
+      end
+
+      def define_members(receiver, value, line)
+        return unless %w[Struct Data].include?(receiver)
+
+        arguments(value).grep(::Prism::SymbolNode).each do |member|
+          add_method(member.unescaped, :instance, line, attribute: :reader, documented: false)
+          next if receiver == 'Data'
+
+          add_method("#{member.unescaped}=", :instance, line, attribute: :writer, documented: false)
+        end
+      end
+
+      # --- methods ------------------------------------------------------------
+
+      def def_scope(node)
+        case node.receiver
+        when nil then frame.singleton || frame.module_function ? :class : :instance
+        when ::Prism::SelfNode then :class
+        else :class if [frame.namespace.name, frame.namespace.path].include?(node.receiver.slice)
+        end
+      end
+
+      def add_method(name, scope, line, explicit: false, documented: true, **)
+        return if @comments.nodoc?(line)
+
+        meth = Model::MethodObject.new(name, frame.namespace, scope: scope, **)
+        meth.visibility = method_visibility(name, scope, explicit)
+        previous = @registry[meth.path]
+        @registry.register(meth)
+        meth.add_file(@file, line)
+        document(meth, line) if documented
+        meth.docstring = previous.docstring if previous && meth.docstring.empty?
+        meth
+      end
+
+      def method_visibility(name, scope, explicit)
+        return @visibility_override if @visibility_override
+        return :private if scope == :instance && ALWAYS_PRIVATE.include?(name)
+        return :public if explicit || frame.module_function
+
+        frame.visibility
+      end
+
+      def add_alias(new_name, old_name, line)
+        scope = current_scope
+        original = find_method(scope, old_name)
+        meth = add_method(new_name, scope, line, parameters: original&.parameters, alias_for: old_name)
+        return unless meth && original
+
+        # An alias copies the original's visibility, unless wrapped as in `private alias_method ...`.
+        meth.visibility = original.visibility unless @visibility_override
+        meth.docstring = original.docstring if meth.docstring.empty?
+      end
+
+      # --- call handlers ------------------------------------------------------
+
+      def on_attr(node)
+        scope = current_scope
+        names(node).each do |name|
+          add_method(name, scope, line(node), attribute: :reader) unless node.name == :attr_writer
+          next if ATTR_READERS_ONLY.include?(node.name)
+
+          add_method("#{name}=", scope, line(node), attribute: :writer)
+        end
+      end
+
+      def on_visibility(node)
+        args = arguments(node)
+        if args.empty?
+          frame.visibility = node.name
+          frame.module_function = false
+          return
+        end
+
+        apply_visibility(args, node.name, current_scope)
+      end
+
+      def on_class_visibility(node)
+        visibility = node.name == :private_class_method ? :private : :public
+        apply_visibility(arguments(node), visibility, :class)
+      end
+
+      def apply_visibility(args, visibility, scope)
+        args.each do |arg|
+          case arg
+          when ::Prism::ArrayNode then apply_visibility(arg.elements, visibility, scope)
+          when ::Prism::DefNode, ::Prism::CallNode then with_visibility(visibility) { visit(arg) }
+          else change_visibility(literal(arg), visibility, scope)
+          end
+        end
+      end
+
+      def change_visibility(name, visibility, scope)
+        meth = name && find_method(scope, name)
+        meth.visibility = visibility if meth
+      end
+
+      def on_module_function(node)
+        args = arguments(node)
+        return frame.module_function = true if args.empty?
+
+        args.each do |arg|
+          next promote_to_class_method(literal(arg)) unless arg.is_a?(::Prism::DefNode)
+
+          # A bare `module_function` earlier in the body stays in effect afterwards.
+          previous = frame.module_function
+          frame.module_function = true
+          visit(arg)
+          frame.module_function = previous
+        end
+      end
+
+      def promote_to_class_method(name)
+        original = name && find_method(:instance, name)
+        return unless original
+
+        @registry.delete(original)
+        meth = Model::MethodObject.new(name, frame.namespace, scope: :class, parameters: original.parameters)
+        meth.docstring = original.docstring
+        original.files.each { |file, line| meth.add_file(file, line) }
+        @registry.register(meth)
+      end
+
+      def on_private_constant(node)
+        names(node).each do |name|
+          constant = @registry[frame.namespace.root? ? name : "#{frame.namespace.path}::#{name}"]
+          constant.visibility = :private if constant
+        end
+      end
+
+      def on_alias_method(node)
+        new_name, old_name = names(node)
+        add_alias(new_name, old_name, line(node)) if new_name && old_name
+      end
+
+      def on_define_method(node)
+        name = literal(arguments(node).first)
+        return unless name && node.block.is_a?(::Prism::BlockNode)
+
+        add_method(name, current_scope, line(node), parameters: node.block.parameters&.parameters&.slice)
+      end
+
+      def on_mixin(node)
+        kind = frame.singleton && node.name == :include ? :extend : node.name
+        arguments(node).each do |arg|
+          next unless arg.is_a?(::Prism::ConstantReadNode) || arg.is_a?(::Prism::ConstantPathNode)
+
+          list = frame.namespace.mixins[kind]
+          list << arg.slice unless list.include?(arg.slice)
+        end
+      end
+
+      # --- helpers ------------------------------------------------------------
+
+      def arguments(node)
+        node.arguments&.arguments || []
+      end
+
+      def names(node)
+        arguments(node).filter_map { |arg| literal(arg) }
+      end
+
+      def literal(node)
+        node.unescaped if node.is_a?(::Prism::SymbolNode) || node.is_a?(::Prism::StringNode)
+      end
+
+      def document(object, line)
+        object.add_file(@file, line)
+        text = @comments.docstring_for(line)
+        return unless text
+
+        docstring = DocstringParser.parse(text)
+        object.docstring = docstring if object.docstring.empty?
+      end
+    end
+  end
+end

data/lib/boondoc/generator.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Boondoc
+  # Runs the whole pipeline: parse, resolve, render, write.
+  #
+  # @example
+  #   Boondoc::Generator.new(Boondoc::Config.new(output: 'docs')).run
+  class Generator
+    # What a run produced.
+    Result = Struct.new(:pages, :assets, :removed, :warnings, keyword_init: true)
+
+    # @return [Model::Registry] the registry built by the last {#run}
+    attr_reader :registry
+
+    # @param config [Config]
+    # @param reporter [Reporter]
+    def initialize(config, reporter: Reporter.new(quiet: config.quiet))
+      @config = config
+      @reporter = reporter
+    end
+
+    # @return [Result]
+    # @raise [Error] when no source files match, the output directory is unsafe,
+    #   a file would be overwritten, or `strict` is set and there were warnings
+    def run
+      report_config
+      files = @config.source_files
+      raise Error, "no Ruby files matched #{Array(@config.sources).join(', ')}" if files.empty?
+
+      parse(files)
+      pages = render
+      removed = Writer.new(@config.output_dir, @reporter, force: @config.force).write(pages, assets: @assets)
+      finish(pages, @assets, removed)
+    end
+
+    # Builds every page without writing anything.
+    #
+    # @return [Hash{String => String}] page content keyed by path relative to the output directory
+    def render
+      context = build_context
+      pages = pages_for(context)
+      @assets = context.assets.files # linked assets are found while the pages render
+      pages
+    end
+
+    # Parses the given files into a fresh registry.
+    #
+    # @param files [Array<String>] paths relative to the project root
+    # @return [Model::Registry]
+    def parse(files)
+      @registry = Model::Registry.new
+      parser = Frontend::PrismParser.new(@registry, @reporter)
+      files.each { |file| parser.parse_file(File.expand_path(file, @config.root), label: file) }
+      expand_directives(files)
+      References.new(@registry).expand
+      @registry
+    end
+
+    private
+
+    def report_config
+      @config.output_dir # refuse an unsafe output directory before doing any work
+      @config.yardopts_notes.each { |note| @reporter.info(note) }
+      @config.outside_sources.each do |path|
+        @reporter.warn("skipping #{path}: it is outside the project (#{@config.root})")
+      end
+    end
+
+    def expand_directives(files)
+      if @config.yard_macros
+        merge_yard(files & @registry.directive_files)
+      elsif @registry.directive_sites.any?
+        sites = @registry.directive_sites
+        @reporter.warn("#{sites.size} comment(s) use YARD directives (@!macro, @!method, ...), which boondoc " \
+                       "does not expand on its own; rerun with --yard-macros. First: #{sites.first(3).join(', ')}")
+      end
+    end
+
+    def merge_yard(yard_files)
+      added = yard_files.empty? ? 0 : YardBridge.new(@registry, @reporter).merge(yard_files, root: @config.root)
+      @reporter.info("yard: added #{added} object(s) from macros and directives")
+      return unless yard_files.empty?
+
+      @reporter.warn('yard: no YARD macros or directives were found; disable yard_macros ' \
+                     '(--yard-macros) for this project')
+    end
+
+    def pages_for(context)
+      index = Renderer::Index.new(context)
+      pages = { @config.index => index.render }
+      context.stitcher.files.reject(&:readme).each { |file| pages[file.output] = index.render_file(file) }
+      namespace_pages(context).each { |namespace, page| pages[context.path_map.file_for(namespace)] = page.render }
+      pages
+    end
+
+    def build_context
+      Context.new(config: @config, registry: @registry, reporter: @reporter)
+    end
+
+    def namespace_pages(context)
+      namespaces = @registry.grep(Model::NamespaceObject).select { |namespace| context.filter.visible?(namespace) }
+      namespaces.to_h { |namespace| [namespace, Renderer::NamespacePage.new(context, namespace)] }
+                .select { |_, page| page.content? }
+    end
+
+    def finish(pages, assets, removed)
+      warnings = @reporter.warnings
+      @reporter.info("boondoc: wrote #{pages.size} file(s) and #{assets.size} asset(s) to #{@config.output}" \
+                     "#{", removed #{removed.size} stale" if removed.any?}" \
+                     "#{", #{warnings.size} warning(s)" if warnings.any?}")
+      raise Error, "#{warnings.size} warning(s) in strict mode" if @config.strict && warnings.any?
+
+      Result.new(pages: pages.keys.sort, assets: assets.keys.sort, removed: removed, warnings: warnings)
+    end
+  end
+end

data/lib/boondoc/linker.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module Boondoc
+  # Turns YARD-style references in markdown into relative links.
+  #
+  # Handles `{Foo#bar}`, `{Foo#bar custom text}`, `{https://example.com text}`,
+  # `{file:GUIDE.md}`, and the include forms `{include:Foo}` and
+  # `{include:file:path.md}`. Code blocks and code spans are left untouched.
+  class Linker # rubocop:disable Metrics/ClassLength -- references and asset links resolved in one place
+    REFERENCE = /(?<escape>\\)?\{(?<target>[^{}\s]+)(?:\s+(?<title>[^{}]+?))?\}/
+    INCLUDE = /\{include:(?<file>file:)?(?<target>[^{}\s]+)\}/
+    URL = %r{\A(?:[a-z][a-z0-9+.-]*://|mailto:)}i
+    CONSTANT = /(?:::)?[A-Z]\w*(?:::[A-Z]\w*)*/
+    CONSTANT_REFERENCE = /\A#{CONSTANT}(?:[#.].+)?\z/
+    METHOD_REFERENCE = /\A[#.][^\s]/
+    MAX_INCLUDE_DEPTH = 5
+
+    # @param config [Config]
+    # @param registry [Model::Registry]
+    # @param filter [Filter]
+    # @param path_map [PathMap]
+    # @param stitcher [Stitcher]
+    # @param assets [Assets] the files copied into the output, which image paths may name
+    # @param reporter [Reporter]
+    def initialize(config:, registry:, filter:, path_map:, stitcher:, reporter:, assets: Assets.new(config))
+      @config = config
+      @registry = registry
+      @filter = filter
+      @path_map = path_map
+      @stitcher = stitcher
+      @assets = assets
+      @reporter = reporter
+    end
+
+    # Expands includes and resolves references in a piece of markdown, then points
+    # paths to copied assets at their copies.
+    #
+    # @param text [String]
+    # @param object [Model::CodeObject] where the text was written; references
+    #   are looked up relative to it
+    # @param from [String] the output file the text will appear in
+    # @param assets [Boolean] whether to rewrite asset paths; false for stitched files,
+    #   whose paths {Stitcher} has already rewritten
+    # @return [String]
+    def text(text, object:, from:, assets: true)
+      text = Markdown.map_prose(expand_includes(text.to_s, object, 0)) do |prose|
+        prose.gsub(REFERENCE) { reference(Regexp.last_match, object, from) }
+      end
+      return text unless assets
+
+      Links.rewrite(text) { |target, embedded| asset_target(target, from, embedded) }
+    end
+
+    # @param object [Model::CodeObject]
+    # @param from [String] the output file the link is written in
+    # @param label [String, nil] link text; defaults to the object's path in a code span
+    # @return [String] a markdown link, or just the label when the object is not rendered
+    def link(object, from:, label: nil)
+      label ||= "`#{object.path}`"
+      @filter.visible?(object) ? "[#{label}](#{@path_map.href(object, from: from)})" : label
+    end
+
+    # Formats a type list, linking every constant that is documented.
+    #
+    # @param types [Array<String>, nil]
+    # @param object [Model::CodeObject]
+    # @param from [String]
+    # @return [String, nil] such as `` `Array<`[`Foo`](Foo.md)`>`, `nil` ``
+    def types(types, object:, from:)
+      return nil if types.nil? || types.empty?
+
+      types.map { |type| type_markup(type, object, from) }.join(', ')
+    end
+
+    private
+
+    def expand_includes(text, object, depth)
+      return text if depth > MAX_INCLUDE_DEPTH || !text.include?('{include:')
+
+      Markdown.map_prose(text) do |prose|
+        prose.gsub(INCLUDE) { included(Regexp.last_match, object, depth) }
+      end
+    end
+
+    def included(match, object, depth)
+      content = match[:file] ? read_file(match[:target]) : @registry.resolve(object, match[:target])&.docstring&.text
+      unless content
+        @reporter.warn("#{location(object)}: cannot include #{match[0]}")
+        return match[0]
+      end
+
+      expand_includes(content.strip, object, depth + 1)
+    end
+
+    # Only files inside the project can be included.
+    def read_file(path)
+      full = File.expand_path(path, @config.root)
+      File.read(full) if File.file?(full) && Containment.inside?(full, @config.root)
+    end
+
+    def reference(match, object, from)
+      target = match[:target]
+      title = match[:title]&.strip
+      return match[0].delete_prefix('\\') if match[:escape]
+      return match[0] if target.start_with?('include:')
+      return "[#{title || target}](#{target})" if URL.match?(target)
+      return file_reference(match, target.delete_prefix('file:'), title, from) if target.start_with?('file:')
+
+      object_reference(match, title, object, from)
+    end
+
+    # A path in a docstring is relative to the output directory first, the way YARD's
+    # `--asset` paths are, and to the project second. It is left as written unless it
+    # names a file that is copied.
+    def asset_target(target, from, embedded)
+      path, suffix = target.split(/(?=[#?])/, 2)
+      copied = @assets.copy_for(path, embedded: embedded)
+      copied ? "#{@path_map.relative(copied, from)}#{suffix}" : target
+    end
+
+    def file_reference(match, name, title, from)
+      file = @stitcher.find(name)
+      return "[#{title || file.title}](#{@path_map.relative(file.output, from)})" if file
+
+      @reporter.warn("cannot resolve #{match[0]}: #{name} is not one of the stitched files")
+      match[0]
+    end
+
+    def object_reference(match, title, object, from)
+      target = match[:target]
+      found = @registry.resolve(object, target)
+      return link(found, from: from, label: title || "`#{target}`") if found
+      return match[0] unless reference_like?(target, object)
+
+      return title || "`#{target}`" if core?(target)
+
+      @reporter.warn("#{location(object)}: cannot resolve link to #{target}")
+      match[0]
+    end
+
+    # Braces in prose are only reported as a broken reference when they look
+    # like one; `{key: value}` and friends pass through silently. Either way
+    # the original text is left exactly as written.
+    def reference_like?(target, object)
+      CONSTANT_REFERENCE.match?(target) || (!object.root? && METHOD_REFERENCE.match?(target))
+    end
+
+    # References to Ruby's own classes cannot be linked, but are not mistakes.
+    def core?(target)
+      CORE_CONSTANTS.include?(target.delete_prefix('::')[/\A[A-Z]\w*/])
+    end
+
+    def location(object)
+      object.root? ? 'docs' : object.path
+    end
+
+    def type_markup(type, object, from)
+      parts = []
+      position = 0
+      type.scan(CONSTANT) do
+        match = Regexp.last_match
+        found = @registry.resolve(object, match[0])
+        next unless found && @filter.visible?(found)
+
+        parts << code(type[position...match.begin(0)]) << link(found, from: from, label: code(match[0]))
+        position = match.end(0)
+      end
+      (parts << code(type[position..])).compact.join
+    end
+
+    def code(text)
+      "`#{text}`" unless text.empty?
+    end
+  end
+end