giblish 0.8.0 → 1.0.0

data/lib/giblish/configurator.rb

@@ -0,0 +1,163 @@
+require_relative "config_utils"
+require_relative "resourcepaths"
+require_relative "docattr_providers"
+require_relative "adocsrc_providers"
+require_relative "subtreeinfobuilder"
+require_relative "docid/docid"
+require_relative "search/headingindexer"
+require_relative "indexbuilders/depgraphbuilder"
+require_relative "indexbuilders/subtree_indices"
+require_relative "gitrepos/history_pb"
+
+module Giblish
+  class HtmlLayoutConfig
+    attr_reader :pre_builders, :post_builders, :adoc_extensions, :adoc_api_opts, :docattr_providers
+
+    def initialize(config_opts)
+      @adoc_api_opts = {backend: "html"}
+      @pre_builders = []
+      @post_builders = []
+      @adoc_extensions = {}
+      @docattr_providers = []
+      case config_opts
+        in resource_dir:
+          # copy local resources to dst and link the generated html with
+          # the given css
+          @pre_builders << CopyResourcesPreBuild.new(config_opts)
+
+          # make sure generated html has relative link to the copied css
+          @docattr_providers << RelativeCssDocAttr.new(ResourcePaths.new(config_opts).dst_style_path_rel)
+        in server_css:
+          # do not copy any local resources, use the given web path to link to css
+          @docattr_providers << AbsoluteLinkedCss.new(config_opts.server_css)
+        else
+          4 == 5 # workaround for bug in standardrb formatting
+        end
+
+      if config_opts.make_searchable
+        # enabling text search
+        search_provider = HeadingIndexer.new(config_opts.srcdir)
+        @adoc_extensions[:tree_processor] = search_provider
+        @post_builders << search_provider
+
+        # add search form to all docs
+        @adoc_extensions[:docinfo_processor] = AddSearchForm.new(config_opts.search_action_path)
+      end
+    end
+  end
+
+  class PdfLayoutConfig
+    attr_reader :pre_builders, :post_builders, :adoc_extensions, :adoc_api_opts, :docattr_providers
+
+    def initialize(config_opts)
+      @adoc_api_opts = {backend: "pdf"}
+      @pre_builders = []
+      @post_builders = []
+      @adoc_extensions = {}
+      @docattr_providers = []
+
+      unless config_opts.resource_dir.nil?
+        # generate pdf using asciidoctor-pdf with custom styling
+        rp = ResourcePaths.new(config_opts)
+        @docattr_providers << PdfCustomStyle.new(rp.src_style_path_abs, *rp.font_dirs_abs.to_a)
+      end
+    end
+  end
+
+  # configure all parts needed to execute the options specified by
+  # the user
+  class Configurator
+    attr_reader :build_options, :doc_attr, :config_opts
+
+    # config_opts:: a Cmdline::Options instance with config info
+    def initialize(config_opts)
+      @config_opts = config_opts
+      @build_options = {
+        pre_builders: [],
+        post_builders: [],
+        adoc_api_opts: {},
+        # add a hash where all values are initiated as empty arrays
+        adoc_extensions: Hash.new { |h, k| h[k] = [] }
+      }
+
+      # Initiate the doc attribute repo used during 'run-time'
+      @doc_attr = DocAttrBuilder.new(
+        GiblishDefaultDocAttribs.new
+      )
+
+      layout_config = case config_opts
+        in format: "html" then HtmlLayoutConfig.new(config_opts)
+        in format: "pdf" then PdfLayoutConfig.new(config_opts)
+        else
+          raise OptionParser::InvalidArgument, "The given cmd line flags are not supported: #{config_opts.inspect}"
+      end
+
+      # setup all options from the chosen layout configuration but
+      # override doc attributes with ones from the supplied configuration to
+      # ensure they have highest pref
+      @doc_attr.add_doc_attr_providers(
+        *layout_config.docattr_providers, CmdLineDocAttribs.new(config_opts)
+      )
+
+      setup_docid(config_opts, @build_options, @doc_attr)
+      setup_index_generation(config_opts, @build_options, @doc_attr)
+
+      # setup all pre,post, and build options
+      @build_options[:adoc_api_opts] = layout_config.adoc_api_opts
+      @build_options[:pre_builders] += layout_config.pre_builders
+      @build_options[:post_builders] += layout_config.post_builders
+      layout_config.adoc_extensions.each do |type, instance|
+        @build_options[:adoc_extensions][type] << instance
+      end
+
+      # add copy of asset dirs if options stipulates this
+      @build_options[:post_builders] << CopyAssetDirsPostBuild.new(@config_opts) unless @config_opts.copy_asset_folders.nil?
+    end
+
+    protected
+
+    def setup_index_generation(config_opts, build_options, doc_attr)
+      return if config_opts.no_index
+
+      # setup index generation
+      idx = SubtreeInfoBuilder.new(doc_attr, nil, SubtreeIndexBase, config_opts.index_basename)
+      build_options[:post_builders] << idx
+    end
+
+    def setup_docid(config_opts, build_options, doc_attr)
+      return unless config_opts.resolve_docid
+
+      # setup docid resolution
+      d = DocIdExtension::DocidPreBuilder.new
+      build_options[:pre_builders] << d
+      docid_pp = DocIdExtension::DocidProcessor.new({id_2_node: d.id_2_node})
+      build_options[:adoc_extensions][:preprocessor] << docid_pp
+
+      # early exit if user does not want indices
+      return if config_opts.no_index
+
+      # generate dep graph if graphviz is available
+      dg = DependencyGraphPostBuilder.new(docid_pp.node_2_ids, doc_attr, nil, nil, config_opts.graph_basename)
+      build_options[:post_builders] << dg
+    end
+  end
+
+  # swap standard index generation from the base class to ones including
+  # git history.
+  class GitRepoConfigurator < Configurator
+    def initialize(config_opts, git_repo_dir)
+      @git_repo_dir = git_repo_dir
+      config_opts.search_action_path ||= "/gibsearch.cgi"
+      super(config_opts)
+    end
+
+    protected
+
+    def setup_index_generation(config_opts, build_options, doc_attr)
+      return if config_opts.no_index
+
+      build_options[:post_builders] << AddHistoryPostBuilder.new(@git_repo_dir)
+      build_options[:post_builders] << SubtreeInfoBuilder.new(doc_attr, nil, SubtreeIndexGit, config_opts.index_basename)
+    end
+  end
+end

data/lib/giblish/conversion_info.rb

@@ -0,0 +1,120 @@
+require "pathname"
+
+module Giblish
+  class FileHistory
+    # History info from git
+    LogEntry = Struct.new(:date, :author, :message, :sha1)
+
+    attr_accessor :history, :branch
+
+    def initialize(branch)
+      @branch = branch
+      @history = []
+    end
+  end
+
+  # Base class for bundling together data we always cache for
+  # each asciidoc file we come across.
+  #
+  # Users are expected to use the derived classes.
+  class ConversionInfo
+    attr_reader :converted, :src_node, :dst_node, :dst_top
+
+    # The relative Pathname from the root dir to the src doc
+    # Ex Pathname("my/subdir/file1.adoc")
+    attr_reader :src_rel_path
+
+    @@nof_missing_titles = 0
+
+    def initialize(converted:, src_node:, dst_node:, dst_top:)
+      @converted = converted
+      @src_node = src_node
+      @dst_node = dst_node
+      @dst_top = dst_top
+      @title = nil
+
+      @src_rel_path = dst_node.relative_path_from(dst_top).dirname / src_node.pathname.basename
+    end
+
+    # return:: a String with the basename of the source file
+    def src_basename
+      @src_node.pathname.basename.to_s
+    end
+
+    def title
+      return @title if @title
+
+      "NO TITLE FOUND (#{@@nof_missing_titles += 1}) !"
+    end
+
+    def docid
+      nil
+    end
+
+    def to_s
+      "Conversion #{@converted ? "succeeded" : "failed"} - src: #{@src_node.pathname} dst: #{@dst_node.pathname}"
+    end
+  end
+
+  # Provide data and access methods available when a conversion has
+  # succeeded
+  class SuccessfulConversion < ConversionInfo
+    attr_reader :stderr, :adoc
+
+    # The relative Pathname from the root dir to the dst file
+    # Ex Pathname("my/subdir/file1.html")
+    attr_reader :dst_rel_path
+
+    attr_reader :purpose_str
+
+    def initialize(src_node:, dst_node:, dst_top:, adoc:, adoc_stderr: "")
+      super(converted: true, src_node: src_node, dst_node: dst_node, dst_top: dst_top)
+
+      @adoc = adoc
+      @stderr = adoc_stderr
+      @title = @adoc.doctitle
+
+      @dst_rel_path = dst_node.relative_path_from(dst_top)
+
+      # Cach the purpose info if it exists
+      @purpose_str = get_purpose_info adoc
+    end
+
+    def docid
+      @adoc.attributes["docid"]
+    end
+
+    private
+
+    # TODO: Move this somewhere else
+    def get_purpose_info(adoc)
+      # Get the 'Purpose' section if it exists
+      purpose_str = +""
+      adoc.blocks.each do |section|
+        next unless section.is_a?(Asciidoctor::Section) &&
+          (section.level == 1) &&
+          (section.name =~ /^Purpose$/)
+
+        # filter out 'odd' text, such as lists etc...
+        section.blocks.each do |bb|
+          next unless bb.is_a?(Asciidoctor::Block)
+
+          purpose_str << "#{bb.source}\n+\n"
+        end
+      end
+      purpose_str
+    end
+  end
+
+  # Provide data and access methods available when a conversion has
+  # failed
+  class FailedConversion < ConversionInfo
+    attr_reader :error_msg
+
+    def initialize(src_node:, dst_node:, dst_top:, error_msg: nil)
+      super(converted: false, src_node: src_node, dst_node: dst_node, dst_top: dst_top)
+
+      @error_msg = error_msg
+    end
+  end
+end

data/lib/giblish/docattr_providers.rb

@@ -0,0 +1,125 @@
+module Giblish
+  class DocAttributesBase
+    def document_attributes(src_node, dst_node, dst)
+      raise NotImplementedError
+    end
+  end
+
+  class AbsoluteLinkedCss < DocAttributesBase
+    def initialize(css_path)
+      @css_path = Pathname.new(css_path)
+    end
+
+    def document_attributes(src_node, dst_node, dst_top)
+      {
+        "stylesdir" => @css_path.dirname.to_s,
+        "stylesheet" => @css_path.basename.to_s,
+        "linkcss" => true,
+        "copycss" => nil
+      }
+    end
+  end
+
+  class RelativeCssDocAttr < DocAttributesBase
+    # dst_css_path_rel:: the relative path to the dst top to the location of the
+    # css file to use
+    def initialize(dst_css_path_rel)
+      @css_path = Pathname.new(dst_css_path_rel)
+    end
+
+    def document_attributes(src_node, dst_node, dst_top)
+      href_path = dst_top.relative_path_from(dst_node).dirname / @css_path
+      {
+        "stylesdir" => href_path.dirname.to_s,
+        "stylesheet" => href_path.basename.to_s,
+        "linkcss" => true,
+        "copycss" => nil
+      }
+    end
+  end
+
+  class PdfCustomStyle < DocAttributesBase
+    # pdf_style_path:: the path name (preferable absolute) to the yml file
+    # pdf_font_dirs:: one or more Pathnames to each font dir that shall be
+    # checked for fonts
+    def initialize(pdf_style_path, *pdf_font_dirs)
+      @pdf_style_path = pdf_style_path
+      # one can specify multiple font dirs as:
+      # -a pdf-fontsdir="path/to/fonts;path/to/more-fonts"
+      # Always use the GEM_FONTS_DIR token to load the adoc-pdf gem's font dirs as well
+      @pdf_fontsdir = (Array(pdf_font_dirs) << "GEM_FONTS_DIR").collect { |d| d.to_s }&.join(";")
+    end
+
+    def document_attributes(src_node, dst_node, dst_top)
+      result = {
+        "pdf-style" => @pdf_style_path.basename.to_s,
+        "pdf-stylesdir" => @pdf_style_path.dirname.to_s,
+        "icons" => "font"
+      }
+      result["pdf-fontsdir"] = @pdf_fontsdir unless @pdf_fontsdir.nil?
+      result
+    end
+  end
+
+  # provides the default, opinionated doc attributes used
+  # when nothing else is set
+  class GiblishDefaultDocAttribs
+    def document_attributes(src_node, dst_node, dst_top)
+      {
+        "source-highlighter" => "rouge",
+        "rouge-css" => "style",
+        "rouge-style" => "github",
+        "xrefstyle" => "short",
+        "icons" => "font"
+      }
+    end
+  end
+
+  class CmdLineDocAttribs
+    def initialize(cmd_opts)
+      @cmdline_attrs = cmd_opts.doc_attributes.dup
+    end
+
+    def document_attributes(src_node, dst_node, dst_top)
+      @cmdline_attrs
+    end
+  end
+
+  # a builder class that can be setup with one or more document
+  # attribute providers and then used as the sole doc attrib provider
+  # where the merged sum of all added providers will be presented.
+  #
+  # If more than one added provider set the same doc attrib, the last
+  # added has preference.
+  #
+  # Each added provider must conform to the itf defined in
+  # the DocAttributesBase class.
+  class DocAttrBuilder
+    attr_reader :providers
+
+    def initialize(*attr_providers)
+      @providers = []
+      add_doc_attr_providers(*attr_providers)
+    end
+
+    def add_doc_attr_providers(*attr_providers)
+      return if attr_providers.empty?
+
+      # check itf compliance of added providers
+      attr_providers.each do |ap|
+        unless ap.respond_to?(:document_attributes) &&
+            ap.method(:document_attributes).arity == 3
+          raise ArgumentError, "The supplied doc attribute provider of type: #{ap.class} did not conform to the interface"
+        end
+      end
+
+      @providers += attr_providers
+    end
+
+    def document_attributes(src_node, dst_node, dst_top)
+      result = {}
+      @providers.each { |p| result.merge!(p.document_attributes(src_node, dst_node, dst_top)) }
+      result
+    end
+  end
+end

data/lib/giblish/docid/docid.rb

@@ -0,0 +1,181 @@
+require "asciidoctor"
+require_relative "../pathtree"
+
+module Giblish
+  module DocIdExtension
+    # Build a hash of {docid => src_node} that can be used to resolve
+    # doc id references to valid dst paths
+    class DocidPreBuilder
+      attr_accessor :id_2_node
+
+      # The minimum number of characters required for a valid doc id
+      ID_MIN_LENGTH = 2
+
+      # The maximum number of characters required for a valid doc id
+      ID_MAX_LENGTH = 10
+
+      # the regex used to find :docid: entries in the doc header
+      DOCID_REGEX = /^:docid: +(.*)$/
+
+      def initialize
+        @id_2_node = {}
+      end
+
+      # called during the pre-build phase
+      def on_prebuild(src_tree, dst_tree, converter)
+        src_tree.traverse_preorder do |level, src_node|
+          next unless src_node.leaf?
+
+          parse_node(src_node)
+        end
+      end
+
+      private
+
+      # Check if a :docid: <id> entry exists in the header of the doc
+      # denoted by 'node'.
+      # According to http://www.methods.co.nz/asciidoc/userguide.html#X95
+      # the header is optional, but if it exists it:
+      # - must start with a titel (=+ <My Title>)
+      # - ends with one or more blank lines
+      # - does not contain any blank line
+      def parse_node(node)
+        p = node.pathname
+        Giblog.logger.debug { "parsing file #{p} for docid..." }
+        Giblish.process_header_lines_from_file(p) do |line|
+          next unless DOCID_REGEX.match(line)
+
+          # There is a docid defined, cache the path and doc id
+          id = Regexp.last_match(1).strip
+          Giblog.logger.debug { "found possible docid: #{id}" }
+          if doc_id_ok?(id, p)
+            @id_2_node[id] = node
+          else
+            Giblog.logger.error { "Invalid docid: #{id} in file #{p}, this will be ignored!" }
+          end
+        end
+      end
+
+      # make sure the id is within the designated length and
+      # does not contain a '#' symbol
+      def doc_id_ok?(doc_id, path)
+        if @id_2_node.key? doc_id
+          Giblog.logger.warn { "Found same doc id twice: (#{doc_id}). This id will be associated with '#{path}' and _not_ with '#{@id_2_node[id]}'." }
+        end
+        (doc_id.length.between?(ID_MIN_LENGTH, ID_MAX_LENGTH) && !doc_id.include?("#"))
+      end
+    end
+
+    # A preprocessor extension to the Asciidoctor engine that transforms all
+    # <<:docid:>> references found in the adoc source into the matching
+    # file reference.
+    #
+    # It requiers a populated 'id_2_node' with {docid => src_node} before
+    # the first invokation of the 'process' method via Asciidoctor.
+    #
+    # When running, it builds a publicly available 'node_2_ids' map.
+    class DocidProcessor < Asciidoctor::Extensions::Preprocessor
+      # {src_node => [referenced doc_id's]}
+      attr_reader :node_2_ids
+
+      # required options:
+      #
+      # id_2_node:: {docid => src_node }
+      def initialize(opts)
+        raise ArgumentError, "Missing required option: :id_2_node!" unless opts.key?(:id_2_node)
+
+        super(opts)
+        @id_2_node = opts[:id_2_node]
+
+        # init new keys in the hash with an empty array
+        @node_2_ids = Hash.new { |h, k| h[k] = [] }
+      end
+
+      # The regex that matches docid references in files
+      DOCID_REF_REGEX = /<<\s*:docid:\s*(.*?)>>/
+      PASS_MACRO_REGEX = /pass:\[.*\]/
+
+      # This hook is called by Asciidoctor once for each document _before_
+      # Asciidoctor processes the adoc content.
+      #
+      # It replaces references of the format <<:docid: ID-1234,Hello >> with
+      # references to a resolved relative path.
+      def process(document, reader)
+        # Add doc as a source dependency for doc ids
+        gib_data = document.attributes["giblish-info"]
+        if gib_data.nil?
+          Giblog.logger.error "The docid preprocessor did not find required info in the doc attribute. (Doc title: #{document.title}"
+          return reader
+        end
+
+        src_node = gib_data[:src_node]
+
+        # Convert all docid refs to valid relative refs
+        reader.lines.each do |line|
+          # remove commented lines
+          next if line.start_with?("//")
+
+          @node_2_ids[src_node] += parse_line(line, src_node)
+        end
+
+        # we only care for one ref to a specific target, remove duplicates
+        @node_2_ids[src_node] = @node_2_ids[src_node].uniq
+
+        # the asciidoctor engine wants the reader back
+        reader
+      end
+
+      private
+
+      # substitutes docid references with the corresponding relative path
+      # references.
+      #
+      # returns:: Array of found docid references
+      def parse_line(line, src_node)
+        refs = []
+        # remove all content within a 'pass:[]' macro from the parser
+        line.gsub!(PASS_MACRO_REGEX)
+
+        line.gsub!(DOCID_REF_REGEX) do |_m|
+          # parse the ref
+          target_id, section, display_str = parse_doc_id_ref(Regexp.last_match(1))
+          Giblog.logger.debug { "Found docid ref to #{target_id} in file: #{src_node.pathname}..." }
+
+          # make sure it exists in the cache
+          unless @id_2_node.key?(target_id)
+            Giblog.logger.warn { "Could not resolve ref to #{target_id} from file: #{src_node.pathname}..." }
+            break "<<UNKNOWN_DOC, Could not resolve doc id reference !!!>>"
+          end
+
+          # add the referenced doc id as a target dependency of this document
+          refs << target_id
+
+          # get the relative path from this file to the target file
+          target_node = @id_2_node[target_id]
+          rel_path = target_node.pathname.relative_path_from(src_node.pathname.dirname)
+
+          # return the resolved reference
+          "<<#{rel_path}##{section}#{display_str}>>"
+        end
+        refs
+      end
+
+      # input_str shall be the expression between
+      # <<:docid:<input_str>>> where the <input_str> is in the form
+      # <id>[#section][,display_str]
+      #
+      # returns an array with [id, section, display_str]
+      def parse_doc_id_ref(input_str)
+        ref, display_str = input_str.split(",").each(&:strip)
+        id, section = ref.split "#"
+
+        display_str = id.dup if display_str.nil?
+        display_str.prepend ","
+
+        section = "" if section.nil?
+
+        [id, section, display_str]
+      end
+    end
+  end
+end

data/lib/giblish/github_trigger/webhook_manager.rb

@@ -0,0 +1,64 @@
+require "git"
+require_relative "../application"
+
+module Giblish
+  # Generate documentation using giblish based on the supplied parameters
+  class GenerateFromRefs
+    STANDARD_GIBLISH_FLAGS = %W[-f html -m --copy-asset-folders _assets$ --server-search-path /cgi-bin/gibsearch.cgi]
+
+    # doc_repo_url:: the url of the repo hosting the docs to be generated. this repo will be cloned
+    # ref_regexp:: a regexp that both defines what git refs that trigger a document generation and what refs will be
+    # generated.
+    # clone_dir_parent:: path to the local directory under which the doc_repo_url will be cloned.
+    # clone_name:: the name of the local clone of the doc_repo_url
+    # doc_src_rel:: the relative path from the repo root to the directory where the docs reside
+    # doc_dst_abs:: the absolute path to the target location for the generated docs
+    # logger:: a ruby Logger instance that will receive log messages
+    def initialize(doc_repo_url, ref_regexp, clone_dir_parent, clone_name, giblish_args, doc_src_rel, doc_dst_abs, logger)
+      @doc_repo_url = doc_repo_url
+      @ref_regexp = ref_regexp
+      @giblish_args = STANDARD_GIBLISH_FLAGS
+      @giblish_args += giblish_args unless giblish_args.nil?
+      @doc_src_rel = doc_src_rel
+      @dstdir = doc_dst_abs
+      @logger = logger
+
+      @repo_root = clone(doc_repo_url, clone_dir_parent, clone_name)
+    end
+
+    # Generate documents from the git refs found in the GitHub
+    # webhook payload.
+    def docs_from_gh_webhook(github_hash)
+      # TODO Implement support for other refs than branches
+      ref = github_hash.fetch(:ref).sub("refs/heads/", "")
+      if ref.empty? || !(@ref_regexp =~ ref)
+        @logger&.info { "Ref '#{ref}' does not match the document generation trigger -> No document generation triggereed" }
+        return
+      end
+
+      generate_docs(ref)
+    end
+
+    private
+
+    def clone(doc_repo_url, dst_dir, clone_name)
+      p = Pathname.new(dst_dir).join(clone_name)
+
+      @logger&.info { "Cloning #{doc_repo_url} to #{p}..." }
+      repo = Git.clone(doc_repo_url, clone_name, path: dst_dir.to_s, logger: @logger)
+      repo.config("user.name", "Giblish Webhook Manager")
+      repo.config("user.email", "dummy@giblish.com")
+      @logger&.info { "Cloning done" }
+      p
+    end
+
+    def generate_docs(ref)
+      srcdir = @repo_root.join(@doc_src_rel)
+      args = @giblish_args + %W[-g #{@ref_regexp} #{srcdir} #{@dstdir}]
+
+      @logger&.info { "Generate docs using parameters: #{args}" }
+      # run giblish with all args
+      EntryPoint.run(args, @logger)
+    end
+  end
+end