boondoc 0.1.0

data/lib/boondoc/reporter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Boondoc
+  # Collects warnings and prints progress messages.
+  class Reporter
+    # @return [Array<String>] every warning raised so far
+    attr_reader :warnings
+
+    # @param io [IO] where messages are written
+    # @param quiet [Boolean] suppress informational messages (warnings still print)
+    def initialize(io: $stderr, quiet: false)
+      @io = io
+      @quiet = quiet
+      @warnings = []
+    end
+
+    # Records and prints a warning. Repeats of the same message are ignored.
+    #
+    # @param message [String]
+    # @return [void]
+    def warn(message)
+      return if @warnings.include?(message)
+
+      @warnings << message
+      @io.puts("warning: #{message}")
+    end
+
+    # Prints an informational message unless quiet.
+    #
+    # @param message [String]
+    # @return [void]
+    def info(message)
+      @io.puts(message) unless @quiet
+    end
+  end
+end

data/lib/boondoc/stitcher.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Boondoc
+  # A non-API file (README, changelog, guide) that is copied into the output.
+  ExtraFile = Struct.new(:source, :output, :content, :readme, keyword_init: true) do
+    # @return [String] the first top-level heading, or the file name
+    def title
+      content[/^#\s+(.+?)\s*#*\s*$/, 1] || File.basename(source, '.*')
+    end
+  end
+
+  # Loads the extra files and keeps their relative links working after the move.
+  class Stitcher
+    MARKDOWN = /\.(?:md|markdown)\z/i
+
+    # @return [Array<ExtraFile>] the README (if any) first, then the other files
+    attr_reader :files
+
+    # @param config [Config]
+    # @param assets [Assets] the files that are copied into the output, which links follow to their copies
+    def initialize(config, assets: Assets.new(config))
+      @config = config
+      @assets = assets
+      readme = config.readme_file
+      @files = config.extra_files.map { |source| load(source, "files/#{output_name(source)}") }
+      @files.unshift(load(readme, config.index, readme: true)) if readme
+    end
+
+    # @return [ExtraFile, nil]
+    def readme
+      files.find(&:readme)
+    end
+
+    # Finds a stitched file by path, file name, or file name without extension.
+    #
+    # @param name [String]
+    # @return [ExtraFile, nil]
+    def find(name)
+      files.find { |file| file.source == name } ||
+        files.find { |file| [File.basename(file.source), File.basename(file.source, '.*')].include?(name) }
+    end
+
+    # Rewrites relative links and image paths so they still point at the same
+    # thing from the file's new location in the output directory. A path to a
+    # file that is copied into the output points at the copy.
+    #
+    # @param file [ExtraFile]
+    # @return [String] the file's content with links adjusted
+    def relocated_content(file)
+      Links.rewrite(file.content) { |target, embedded| relocate(target, file, embedded: embedded) }
+    end
+
+    private
+
+    def load(source, output, readme: false)
+      content = File.read(File.expand_path(source, @config.root))
+      content = "# #{File.basename(source)}\n\n```\n#{content.rstrip}\n```\n" unless MARKDOWN.match?(source)
+      ExtraFile.new(source: source, output: output, content: content, readme: readme)
+    end
+
+    # Sources are relative to the project root; {Config#extra_files} rejects any outside it.
+    def output_name(source)
+      MARKDOWN.match?(source) ? source.sub(MARKDOWN, '.md') : "#{source}.md"
+    end
+
+    def relocate(target, file, embedded:)
+      return target if Links::EXTERNAL.match?(target)
+
+      path, suffix = target.split(/(?=[#?])/, 2)
+      destination = destination_of(File.expand_path(path, File.dirname(source_path(file))), embedded)
+      origin = Pathname.new(File.expand_path(file.output, @config.output_dir)).dirname
+      "#{Pathname.new(destination).relative_path_from(origin)}#{suffix}"
+    end
+
+    def source_path(file)
+      File.expand_path(file.source, @config.root)
+    end
+
+    # A link to another stitched file or to a copied asset follows it to its new
+    # home; anything else keeps pointing at the original file in the project.
+    def destination_of(absolute, embedded)
+      stitched = files.find { |other| source_path(other) == absolute }
+      output = stitched ? stitched.output : @assets.output_for(absolute, embedded: embedded)
+      output ? File.expand_path(output, @config.output_dir) : absolute
+    end
+  end
+end

data/lib/boondoc/type_parser.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Boondoc
+  # Splits a YARD type list such as `[String, Hash{Symbol => Array<Integer>}]`.
+  module TypeParser
+    OPENERS = { '[' => ']', '<' => '>', '(' => ')', '{' => '}' }.freeze
+    # Duck types naming an operator method that contains bracket characters.
+    OPERATOR_DUCK_TYPE = /#(?:<=>|<<|>>|[<>]=?|\[\]=?)/
+
+    module_function
+
+    # Takes a leading bracketed type list off a string.
+    #
+    # @param text [String]
+    # @return [Array(Array<String>, String), Array(nil, String)] the types (nil
+    #   when the text does not start with `[`) and the remaining text
+    def extract(text)
+      text = text.lstrip
+      return [nil, text] unless text.start_with?('[')
+
+      close = closing_index(text)
+      return [nil, text] unless close
+
+      [split(text[1...close]), text[(close + 1)..].lstrip]
+    end
+
+    # Splits on top-level commas, leaving nested brackets intact.
+    #
+    # @param list [String] the text between the outer brackets
+    # @return [Array<String>]
+    def split(list)
+      types = ['']
+      each_with_depth(list) do |char, _, depth|
+        next types << +'' if char == ',' && depth.zero?
+
+        types[-1] += char
+      end
+      types.map(&:strip).reject(&:empty?)
+    end
+
+    # @api private
+    def closing_index(text)
+      each_with_depth(text) { |_, index, depth| return index if depth.zero? }
+      nil
+    end
+
+    # Yields each character with the nesting depth after it.
+    #
+    # @api private
+    def each_with_depth(text)
+      skip = operator_indices(text)
+      depth = 0
+      text.each_char.with_index do |char, index|
+        depth += depth_change(text, char, index) unless skip.include?(index)
+        yield char, index, depth
+      end
+    end
+
+    # @api private
+    def operator_indices(text)
+      indices = []
+      text.scan(OPERATOR_DUCK_TYPE) do
+        match = Regexp.last_match
+        indices.concat((match.begin(0)...match.end(0)).to_a)
+      end
+      indices
+    end
+
+    # @api private
+    def depth_change(text, char, index)
+      return 1 if OPENERS.key?(char)
+      return 0 unless OPENERS.value?(char)
+
+      # The `>` of a hash rocket (`=>`) does not close anything.
+      char == '>' && index.positive? && text[index - 1] == '=' ? 0 : -1
+    end
+  end
+end

data/lib/boondoc/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Boondoc
+  VERSION = '0.1.0'
+end

data/lib/boondoc/writer.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module Boondoc
+  # Writes generated pages and removes pages left over from earlier runs.
+  #
+  # A manifest in the output directory records what Boondoc wrote, so only its
+  # own files are ever deleted or silently overwritten. A page whose content has
+  # not changed is not written again.
+  class Writer
+    MANIFEST = '.boondoc-manifest'
+
+    # @param output_dir [String] absolute path of the output directory
+    # @param reporter [Reporter]
+    # @param force [Boolean] overwrite files that Boondoc did not generate
+    def initialize(output_dir, reporter, force: false)
+      @output_dir = output_dir
+      @reporter = reporter
+      @force = force
+    end
+
+    # @param pages [Hash{String => String}] content keyed by path relative to the output directory
+    # @param assets [Hash{String => String}] absolute source paths of files to copy, keyed by
+    #   the path relative to the output directory
+    # @return [Array<String>] stale files that were removed
+    # @raise [Error] when a page would overwrite a file Boondoc did not generate,
+    #   or would be written outside the output directory or through a symlink
+    def write(pages, assets: {})
+      outputs = pages.keys + assets.keys
+      previous = manifest
+      check(outputs, previous)
+      stale = remove_stale(previous - outputs)
+      pages.each { |path, content| write_page(path, content) }
+      assets.each { |path, source| copy_asset(path, source) }
+      write_file(File.join(@output_dir, MANIFEST), "#{outputs.sort.join("\n")}\n")
+      stale
+    end
+
+    private
+
+    def manifest
+      path = File.join(@output_dir, MANIFEST)
+      return [] unless File.file?(path)
+
+      File.readlines(path, chomp: true).reject(&:empty?)
+    end
+
+    def check(paths, previous)
+      paths.each { |path| check_destination(path) }
+      check_duplicates(paths)
+      check_conflicts(paths, previous)
+      warn_case_collisions(paths)
+    end
+
+    # A page and an asset cannot be written to the same path.
+    def check_duplicates(paths)
+      duplicate = paths.tally.find { |_, count| count > 1 }&.first
+      raise Error, "a page and an asset would both be written to #{duplicate}" if duplicate
+    end
+
+    # Stale files go first: on a case-insensitive filesystem a file renamed only by case is the same
+    # file, and removing it afterwards would delete the file just written.
+    def remove_stale(paths)
+      stale = paths.select { |path| removable?(path) }
+      stale.each { |path| remove(path) }
+      stale
+    end
+
+    def check_destination(path)
+      problem = unsafe(path)
+      raise Error, "refusing to write #{path}: #{problem}" if problem
+    end
+
+    # The manifest is part of the output directory and may have been edited,
+    # so stale entries get the same checks as pages before anything is deleted.
+    def removable?(path)
+      problem = unsafe(path)
+      @reporter.warn("not removing stale #{path}: #{problem}") if problem
+      problem.nil?
+    end
+
+    # @return [String, nil] why writing or deleting `path` would touch a file
+    #   outside the output directory
+    def unsafe(path)
+      full = File.expand_path(path, @output_dir)
+      return "it is outside #{@output_dir}" unless full.start_with?(File.join(@output_dir, ''))
+
+      # The symlink check below resolves `..` by name, but the file system follows a symlink first.
+      return 'it contains `..`' if path.split('/').include?('..')
+
+      link = Containment.symlink_between(full, @output_dir)
+      "#{link} is a symlink" if link
+    end
+
+    def check_conflicts(paths, previous)
+      return if @force
+
+      conflicts = paths.select do |path|
+        File.exist?(File.join(@output_dir, path)) &&
+          !previous.include?(path) && !same_file_as_earlier_page?(path, previous)
+      end
+      return if conflicts.empty?
+
+      raise Error, "refusing to overwrite #{conflicts.size} file(s) that boondoc did not generate " \
+                   "(#{conflicts.first(5).join(', ')}#{', ...' if conflicts.size > 5}). " \
+                   'Use --force to overwrite, or choose another --output directory.'
+    end
+
+    # True when a case-insensitive filesystem maps the path onto a page Boondoc wrote before.
+    def same_file_as_earlier_page?(path, previous)
+      previous.any? do |known|
+        known.casecmp?(path) && File.identical?(File.join(@output_dir, known), File.join(@output_dir, path))
+      end
+    end
+
+    # Pages for classes and modules never get here, because PathMap tells them apart. What is left
+    # are stitched files, assets and the index, whose names come from the project.
+    def warn_case_collisions(paths)
+      paths.group_by(&:downcase).each_value do |group|
+        next if group.size == 1
+
+        @reporter.warn("#{group.sort.join(' and ')} differ only by case and will collide on case-insensitive " \
+                       'filesystems; rename one of them')
+      end
+    end
+
+    def write_page(path, content)
+      full = File.join(@output_dir, path)
+      FileUtils.mkdir_p(File.dirname(full))
+      write_file(full, content)
+    end
+
+    # Leaves a file that already holds this content alone, so its modification
+    # time only changes when the page does. This runs after every destination
+    # has passed {#check}, so it never reads through a symlink.
+    def write_file(full, content)
+      return if File.file?(full) && File.size(full) == content.bytesize && File.binread(full) == content.b
+
+      File.write(full, content)
+    end
+
+    # Copies a file byte for byte. A copy that already matches is left alone, so
+    # its modification time only changes when the file does.
+    def copy_asset(path, source)
+      full = File.join(@output_dir, path)
+      FileUtils.mkdir_p(File.dirname(full))
+      return if File.file?(full) && FileUtils.compare_file(source, full)
+
+      FileUtils.cp(source, full)
+    end
+
+    def remove(path)
+      full = File.join(@output_dir, path)
+      FileUtils.rm_f(full)
+      directory = File.dirname(full)
+      while directory.start_with?("#{@output_dir}/") && Dir.empty?(directory)
+        Dir.rmdir(directory)
+        directory = File.dirname(directory)
+      end
+    end
+  end
+end

data/lib/boondoc/yard_bridge.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Boondoc
+  # Optional second pass that uses YARD to expand macros and directives.
+  #
+  # Prism stays the source of truth. YARD only contributes objects that Prism
+  # could not see (those created by `@!method`, `@!attribute`, or an attached
+  # macro) and the expanded docstrings of objects whose comment uses a directive.
+  #
+  # The `yard` gem is loaded on demand and is not a dependency of boondoc.
+  class YardBridge
+    DEFAULT_SUPERCLASSES = %w[Object BasicObject].freeze
+    # The "error" severity, which has the same value in every YARD logger implementation.
+    ERROR_LEVEL = 3
+
+    # @param registry [Model::Registry]
+    # @param reporter [Reporter]
+    def initialize(registry, reporter)
+      @registry = registry
+      @reporter = reporter
+    end
+
+    # @param files [Array<String>] the Ruby files that contain YARD directives, relative to `root`
+    # @param root [String] the project directory. YARD is given absolute paths, so the
+    #   working directory does not matter, and the files it records are made relative again.
+    # @return [Integer] how many objects were added from YARD
+    # @raise [Error] when the yard gem is not installed
+    def merge(files, root:)
+      load_yard
+      @root = File.expand_path(root)
+      @added = 0
+      ::YARD::Registry.clear
+      paths = files.map { |file| File.expand_path(file, @root) }
+      ::YARD::Logger.instance.enter_level(ERROR_LEVEL) { ::YARD.parse(paths) }
+      yard_objects.each { |object| merge_object(object) }
+      @added
+    ensure
+      ::YARD::Registry.clear if defined?(::YARD::Registry)
+    end
+
+    private
+
+    def load_yard
+      require 'yard'
+    rescue LoadError
+      raise Error, "--yard-macros needs the yard gem. Add `gem 'yard'` to your Gemfile or run `gem install yard`."
+    end
+
+    def yard_objects
+      namespaces = ::YARD::Registry.all(:module, :class).sort_by { |object| object.path.count(':') }
+      namespaces + ::YARD::Registry.all(:constant, :method)
+    end
+
+    def merge_object(yard_object)
+      existing = @registry[yard_object.path]
+      return add(yard_object) unless existing
+
+      existing.docstring = docstring(yard_object) if existing.docstring.directives.any?
+    end
+
+    def add(yard_object)
+      namespace = @registry[yard_object.namespace.path]
+      return unless namespace.is_a?(Model::NamespaceObject)
+
+      object = build(yard_object, namespace)
+      object.visibility = yard_object.visibility
+      object.docstring = docstring(yard_object)
+      object.add_file(relative(yard_object.file), yard_object.line) if yard_object.file
+      @registry.register(object)
+      @added += 1
+    end
+
+    def relative(file)
+      prefix = File.join(@root, '')
+      file.start_with?(prefix) ? file.delete_prefix(prefix) : file
+    end
+
+    def build(yard_object, namespace)
+      name = yard_object.name.to_s
+      case yard_object.type
+      when :constant then Model::ConstantObject.new(name, namespace, value: yard_object.value.to_s)
+      when :method then build_method(yard_object, name, namespace)
+      else build_namespace(yard_object, name, namespace)
+      end
+    end
+
+    def build_namespace(yard_object, name, namespace)
+      object = Model::NamespaceObject.new(name, namespace, kind: yard_object.type)
+      superclass = yard_object.superclass&.path if yard_object.type == :class
+      object.superclass = superclass unless DEFAULT_SUPERCLASSES.include?(superclass)
+      object
+    end
+
+    def build_method(yard_object, name, namespace)
+      attribute = (name.end_with?('=') ? :writer : :reader) if yard_object.is_attribute?
+      Model::MethodObject.new(name, namespace, scope: yard_object.scope, attribute: attribute,
+                                               parameters: parameters(yard_object))
+    end
+
+    def parameters(yard_object)
+      list = yard_object.parameters.map do |name, default|
+        next name.to_s unless default
+
+        name.to_s.end_with?(':') ? "#{name} #{default}" : "#{name} = #{default}"
+      end
+      list.empty? ? nil : list.join(', ')
+    end
+
+    def docstring(yard_object)
+      DocstringParser.parse(yard_object.docstring.to_raw)
+    end
+  end
+end

data/lib/boondoc/yardopts.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require 'shellwords'
+
+module Boondoc
+  # The part of a project's `.yardopts` that Boondoc understands.
+  #
+  # Only the source globs, the extra files after `-`, `--files`, `--asset`, `--readme`,
+  # `--title`, `--private`, `--protected`, and `--exclude` are used. `--output-dir`
+  # is left out on purpose: in a `.yardopts` it names YARD's HTML directory, which is
+  # usually ignored by git, while Boondoc's output is meant to be committed.
+  # Everything else is collected in {#ignored} so the run can say what was skipped.
+  #
+  # @example
+  #   Boondoc::Yardopts.parse('--title Foo -o doc lib - CHANGELOG.md').settings
+  #   # => { title: "Foo", sources: ["lib"], files: ["CHANGELOG.md"] }
+  class Yardopts
+    FILE = '.yardopts'
+    # Long options of the yard gem that take an argument.
+    ARGUMENT_OPTIONS = %w[
+      --load --plugin --db --exclude --api --no-api --hide-api --default-return --query --title
+      --readme --main --files --asset --output-dir --markup --markup-provider --charset --template
+      --template-path --format --locale --po-dir --tag --type-tag --type-name-tag --name-tag
+      --title-tag --hide-tag --transitive-tag --non-transitive-tag
+    ].freeze
+    # Long options whose argument is optional; it is the next token unless that starts with `-`.
+    OPTIONAL_ARGUMENT_OPTIONS = %w[--use-cache --yardopts --no-yardopts --embed-mixin].freeze
+    SHORT_ARGUMENT_OPTIONS = {
+      'b' => '--db', 'e' => '--load', 'o' => '--output-dir', 'm' => '--markup', 'M' => '--markup-provider',
+      't' => '--template', 'p' => '--template-path', 'f' => '--format', 'r' => '--readme'
+    }.freeze
+    SHORT_OPTIONAL_ARGUMENT_OPTIONS = { 'c' => '--use-cache' }.freeze
+    NON_RUBY_SOURCE = /\.(?:c|cc|cpp|cxx|h|hpp|y)\z/
+    # How long a pattern from `--exclude` may take to match one path.
+    REGEXP_TIMEOUT = 1.0
+
+    # @param root [String] project directory
+    # @return [Yardopts, nil] nil when the project has no `.yardopts`
+    # @raise [Error] when the file cannot be read or parsed
+    def self.read(root)
+      path = File.join(root, FILE)
+      return nil unless File.file?(path)
+
+      parse(File.read(path))
+    rescue SystemCallError => e
+      raise Error, "could not read #{FILE}: #{e.message}"
+    end
+
+    # @param text [String] the contents of a `.yardopts`
+    # @return [Yardopts]
+    # @raise [Error] when the quoting is unbalanced or an `--exclude` pattern is not a regexp
+    def self.parse(text)
+      new(Shellwords.split(text))
+    rescue ArgumentError => e
+      raise Error, "could not read #{FILE}: #{e.message}"
+    end
+
+    # @return [Array<Regexp>] patterns from `--exclude`, matched against source paths
+    attr_reader :exclude
+    # @return [String, nil] the value of `--output-dir`, which Boondoc does not use
+    attr_reader :output_dir
+    # @return [Array<String>] options and sources that were read but are not used
+    attr_reader :ignored
+
+    # @param tokens [Array<String>] the file split into words, the way a shell would
+    def initialize(tokens)
+      @sources = []
+      @files = []
+      @assets = []
+      @exclude = []
+      @ignored = []
+      @settings = {}
+      read_tokens(tokens)
+    end
+
+    # @return [Hash{Symbol => Object}] Boondoc options: `sources`, `files`, `assets`, `readme`, `title`,
+    #   `private`, and `protected`, for those that were given
+    def settings
+      @settings.merge({ sources: @sources, files: @files, assets: @assets }.reject { |_, list| list.empty? })
+    end
+
+    # @param output [String] the output directory Boondoc will use
+    # @param root [String] project directory, for comparing paths
+    # @return [Array<String>] messages telling the user what was left out
+    def notes(output:, root:)
+      notes = []
+      if output_dir && File.expand_path(output_dir, root) != File.expand_path(output, root)
+        notes << "ignoring --output-dir #{output_dir} from #{FILE}; writing to #{output} " \
+                 '(set output in .boondoc.yml or pass -o to change)'
+      end
+      notes << "ignoring from #{FILE}: #{ignored.join(', ')}" if ignored.any?
+      notes
+    end
+
+    private
+
+    def read_tokens(tokens)
+      tokens = tokens.dup
+      extra = false
+      until tokens.empty?
+        token = tokens.shift
+        if token == '-' then extra = true
+        elsif extra then @files << token
+        elsif token.start_with?('-') then read_option(token, tokens)
+        else add_source(token)
+        end
+      end
+    end
+
+    def read_option(token, rest)
+      name, value = split_option(token)
+      value ||= rest.shift if takes_argument?(name, rest)
+      apply(name, value)
+    end
+
+    def takes_argument?(name, rest)
+      return false if rest.empty?
+
+      ARGUMENT_OPTIONS.include?(name) || (OPTIONAL_ARGUMENT_OPTIONS.include?(name) && !rest.first.start_with?('-'))
+    end
+
+    # @return [Array(String, String)] the canonical long name (or the token itself) and its inline value, if any
+    def split_option(token)
+      return token.split('=', 2).values_at(0, 1) if token.start_with?('--')
+
+      letter = token[1]
+      long = SHORT_ARGUMENT_OPTIONS[letter] || SHORT_OPTIONAL_ARGUMENT_OPTIONS[letter]
+      return [token, nil] unless long
+
+      [long, token.length > 2 ? token[2..] : nil]
+    end
+
+    def apply(name, value)
+      case name
+      when '--output-dir' then @output_dir = value
+      when '--title' then @settings[:title] = value
+      when '--readme', '--main' then @settings[:readme] = value
+      when '--files', '--asset' then add_list(name, value)
+      when '--private', '--protected' then @settings[name.delete_prefix('--').to_sym] = true
+      when '--exclude' then add_exclude(value)
+      else @ignored << name
+      end
+    end
+
+    # `--files` and `--asset` take comma-separated lists, as YARD does for `--files`.
+    def add_list(name, value)
+      list = name == '--asset' ? @assets : @files
+      list.concat(value.to_s.split(','))
+    end
+
+    def add_source(glob)
+      if glob.match?(NON_RUBY_SOURCE)
+        @ignored << glob
+      else
+        @sources << glob
+      end
+    end
+
+    def add_exclude(pattern)
+      @exclude << Regexp.new(pattern.to_s, timeout: REGEXP_TIMEOUT)
+    rescue RegexpError => e
+      raise ArgumentError, "--exclude #{pattern.inspect} is not a regular expression (#{e.message})"
+    end
+  end
+end