giblish 0.8.2 → 1.0.0.rc2

data/docs/howtos/trigger_generation_im/post-receive-example.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# this hook runs a script which, in turn, generates html documentation
+# and publishes them to the given destination.
+#
+
+# the git refs that should trigger a doc generation at update
+TRIGGERING_REFS_REGEX=$'main'
+
+# the staging repo root (where the working tree exists)
+STAGING_REPO="/usr/local/git_staging/myrepo_staging"
+
+# The web-server top dir for the published documents
+DST_DIR="/var/www/mysite/html/public/docs"
+
+# "true" runs an 'rm -rf' of the destination dir before generating new htmls
+CLEAR_DST="false"
+
+echo "post-receive hook running..."
+
+# read the input that git sends to this hook
+read oldrev newrev ref
+
+# remove the 'refs/heads/' prefix from the git ref
+ref="${ref/refs\/heads\//}"
+
+# filter out refs that are irrelevant for doc generation
+if [[ ! "${ref}" =~ "${TRIGGERING_REFS_REGEX}" ]]; then
+  echo "Ref '${ref}' received. Doing nothing: only refs matching the regex: /${TRIGGERING_REFS_REGEX}/ will trigger a doc generation."
+  exit 0
+fi
+
+echo "Document generation triggered by an update to ${ref}."
+echo ""
+
+# use a subshell with the correct working dir for the actual doc generation
+(
+  cd "${STAGING_REPO}"
+
+  # need to unset the GIT_DIR env set by the invoking hook for giblish to work correctly
+  unset GIT_DIR
+
+  if [[ "${CLEAR_DST}" -eq "true" ]]; then
+    echo "Remove everything under ${DST_DIR}/"
+    rm -rf ${DST_DIR}/*
+  fi
+
+  # Generate html docs
+  giblish --copy-asset-folders "_assets$" -g "${TRIGGERING_REFS_REGEX}" -r scripts/resources -s mystyle . "${DST_DIR}"
+)

data/docs/reference/box_flow_spec.adoc

@@ -0,0 +1,22 @@
+= Tree flow 
+
+== Algorithm
+
+Preconditions::
+ * No padding
+ * Left-to-right filling
+ * 1 sublevel
+ * Width for outer box is fixed, height is variable
+
+Outer box = B [bx, by, bw, bh]
+Inner boxes = In [In_x, In_y, In_w, In_h]
+
+Horizontal strips = Sn
+
+foreach In in I
+  if In_w > S_free_width
+    S = S(n+1)
+  end
+  In_coord = S_free_coord
+end
+

data/docs/reference/search_spec.adoc

@@ -0,0 +1,185 @@
+= Text search spec for giblish
+:docid: G-002
+
+== Information expected to be included in a search request
+
+URL endpoint for the search service::
+When a search is triggered, the correct URL for triggering the search service must be POSTed to.
+
+URI parameters::
+The search service expects a number of parameters as part of the search query, see <<uri_params>>.
+
+.URI parameters
+[[uri_params]]
+[cols="2,2,5,3"]
+|===
+|name |status |comment |example
+
+|calling-url
+|required
+|the URL to the file from which the search request originated
+|www.example.com/my/doc/site/subdir/file_1.html
+
+|search-assets-top-rel
+|required
+|the relative path from the originating file's directory to the top dir of the search assets. The default implementation in giblish uses the name `gibsearch_assets` for this directory.
+|../../gibsearch_assets
+
+|search-phrase
+|required
+|a string with the phrase to search for
+|meaning of life
+
+|css-path
+|optional
+a|the path to the css file to use when generating the result. 
+
+absolute::
+the path is used as-is. 
+
+relative::
+the relative path to the css file from the originating file's directory.
+
+not set::
+the styling is left to the search implementation.
+
+a| * *absolute* - /var/www/html/my/site/style.css
+* *relative* - ../../web_assets/css/style.css
+
+|consider-case
+|optional
+a|
+if set::
+search case sensitive
+
+not set::
+ignore case during search
+
+|true
+
+|as-regexp
+|optional
+a|
+if set::
+treat the search phrase as a regexp
+
+not set::
+treat the search phrase as a string
+|true
+
+|===
+
+== The implementation in giblish
+
+=== At document generation
+
+ . Collect all adoc docs in a search asset directory.
+ . Embed html code for a search form as part of running gibish on the source tree. During generation, the following search parameters are known for each document to be generated:
+ .. search-assets-top-rel
+ .. css-path
+ . The `calling-url` is not known but it is possible to embed javascript code as part of the search form code that will resolve the url at runtime.
+ . The URL for the search service endpoint must be given by the user or hard-coded in giblish.
+
+=== At a user search query
+
+When a user submit a 'search' query, the following parameters must be filled in and submitted to the text search service:
+
+ . searchphrase
+ . usecase (optional)
+ . useregexp (optional)
+
+The other required parameters comes from the generated document itself.
+
+=== The search index
+
+The search index is created by giblish during html generation and consists of 
+
+ . the adoc source files (when `include` directives have been resolved) 
+ . a `heading_db.json` file mapping sections in the source files to line numbers
+
+.The storage location for the search index information
+----
+|- <top_path>
+|    |- file1.html
+|    |- subdir
+|         |- file2.html
+|    |- web_assets
+|        |- mystyle.css
+|    |- gibsearch_assets
+|         |- heading_db.json
+|         |- file1.adoc
+|              |- subdir
+|                   |file2.adoc
+----
+
+.The format of the heading_db.json file
+[source,json]
+----
+{
+  fileinfos : [
+    {
+      filepath : "file1.adoc",
+      title : "File 1 Title",
+      sections : [
+        {
+          id : "section_id_1",
+          title : "Purpose",
+          line_no : 10
+        },
+        {
+          id : "section_id_2",
+          ...
+        }
+      ]
+    },
+    {
+      filepath: "subdir/file2.adoc",
+      ...
+    }
+  ]
+}
+----
+
+=== The search form
+
+giblish inserts the below html/JavaScript at the top of each generated html document.
+
+.A minimal search form
+[source,html]
+----
+      <script type="text/javascript">
+      window.onload = function () {
+        document.getElementById("calingurl_input").value = window.location.href;
+      };
+      </script>
+
+      <style>
+      #gibsearch-form {
+        position:fixed;
+        top:0.5rem;
+        left:70%;
+        width:30%;
+        height:3rem;
+        background:white;
+        z-index:2000; 
+        }
+      </style>
+
+      <div id=gibsearch-form>
+        <form class="gibsearch" action="<%=action_path%>">
+          <input type="search" placeholder="Search the docs.." name="search-phrase" />
+          <button type="submit">Search</button>
+          <br>
+
+          <input type="checkbox" id="consider-case" name="consider-case" />
+          <label for="consider-case">case sensitive</label>
+          &nbsp;&nbsp;
+          <input type="checkbox" id="as-regexp" name="as-regexp" />
+          <label for="as-regexp">use regexp</label>
+
+          <input type="hidden" name="calling-url" id=calingurl_input />
+          <input type="hidden" name="search-assets-top-rel" value="<%=sa_top_rel%>"/>
+          <input type="hidden" name="css-path" value="<%=css_path%>"/>
+        </form>
+      </div>
+----

data/giblish.gemspec

@@ -1,24 +1,28 @@
-# coding: utf-8
-
-lib = File.expand_path("../lib", __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-require "giblish/version"
+begin
+  require_relative "lib/giblish/version"
+rescue LoadError
+  require "giblish/version"
+end
 
 Gem::Specification.new do |spec|
-  spec.name          = "giblish"
-  spec.version       = Giblish::VERSION
-  spec.authors       = ["Anders Rillbert"]
-  spec.email         = ["anders.rillbert@kutso.se"]
+  spec.name = "giblish"
+  spec.version = Giblish::VERSION
+  spec.summary = "A tool for publishing asciidoc docs stored in git repos"
+  spec.description = <<~EOF
+    giblish generates indexed and searchable documents from a tree of
+    asciidoc files.
+  EOF
+  spec.authors = ["Anders Rillbert"]
+  spec.email = ["anders.rillbert@kutso.se"]
+  spec.homepage = "https://github.com/rillbert/giblish"
+  spec.license = "MIT"
+  # NOTE required ruby version is informational only; it's not enforced since it can't be overridden and can cause builds to break
+  # spec.required_ruby_version = ">= 2.7"
 
-  spec.summary       = "A tool for publishing asciidoc docs stored in git repos"
-  spec.description   = <<~EOF
-                           giblish generates indexed and searchable documents from a tree of
-                           asciidoc files.
-                       EOF
-  spec.homepage      = "https://github.com/rillbert/giblish"
-  spec.license       = "MIT"
-  spec.required_ruby_version = ">= 2.7"
+  spec.metadata = {
+    "bug_tracker_uri" => "https://github.com/rillbert/giblish/issues",
+    "source_code_uri" => "https://github.com/rillbert/giblish"
+  }
 
   # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
   # delete this section to allow pushing this gem to any host.
@@ -28,22 +32,36 @@ Gem::Specification.new do |spec|
   #   raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
   # end
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  # filter out files not included in the shipped gem
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    skip_dirs = %r{^(data|bin|test|spec|features)/}
+    skip_files = %r{^(Rakefile|Gemfile)}
+    f.match(skip_dirs) || f.match(skip_files)
+  end
+
+  # Follow the bundler convention to have the exe:s in "exe" instead of 'bin'
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 2.0"
+  # Development deps
   spec.add_development_dependency "minitest", "~> 5.0"
-  spec.add_development_dependency "rake", "~> 11.0"
-  spec.add_development_dependency "oga", "~> 2.15"
-  spec.add_development_dependency 'thor', '~> 0.20.3'
+  spec.add_development_dependency "standard", "~> 1.1"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "oga", "~> 3.3"
+  spec.add_development_dependency "thor", "~> 0.20.3"
   spec.add_development_dependency "asciidoctor-mathematical", "~> 0.3.5"
+  # needed for the sinatra-based apps
+  spec.add_development_dependency "sinatra", "~>2.1"
+  spec.add_development_dependency "thin", "~>1.8"
+  spec.add_development_dependency "rack", "2.2.3"
+  spec.add_development_dependency "rack-test", "1.1"
 
-  # Usage: spec.add_runtime_dependency "[gem name]", [[version]]
-  spec.add_runtime_dependency "asciidoctor", "~>2.0", ">= 2.0.16"
-  spec.add_runtime_dependency "asciidoctor-diagram", ["~> 2.1"]
-  spec.add_runtime_dependency "asciidoctor-pdf", ["~> 1.6", ">= 1.6.0"]
+  # Run-time deps
+  spec.add_runtime_dependency "warning", "~>1.2"
+  spec.add_runtime_dependency "asciidoctor", "~>2.0", ">= 2.0.17"
+  spec.add_runtime_dependency "asciidoctor-diagram", ["~> 2.2"]
+  spec.add_runtime_dependency "asciidoctor-pdf", ["~> 1.6", ">= 1.6.2"]
   spec.add_runtime_dependency "git", "~> 1.9"
   spec.add_runtime_dependency "rouge", "~> 3.0"
   spec.add_runtime_dependency "prawn-svg", "~> 0.32.0"

data/lib/giblish/adocsrc_providers.rb

@@ -0,0 +1,23 @@
+module Giblish
+  class AdocSrcItf
+    def adoc_source(src_node, dst_node, dst_top)
+      raise NotImplementedError
+    end
+  end
+
+  class SrcFromFile < AdocSrcItf
+    def adoc_source(src_node, dst_node, dst_top)
+      File.read(src_node.pathname)
+    end
+  end
+
+  class SrcFromString < AdocSrcItf
+    def initialize(src_str)
+      @adoc_source = src_str
+    end
+
+    def adoc_source(src_node, dst_node, dst_top)
+      @adoc_source
+    end
+  end
+end

data/lib/giblish/application.rb

@@ -1,68 +1,241 @@
 require_relative "cmdline"
-require_relative "core"
-require_relative "utils"
+require_relative "configurator"
+require_relative "treeconverter"
+require_relative "gitrepos/checkoutmanager"
 
 module Giblish
-  # The 'main' class of giblish
+  # The app class for the giblish application
   class Application
-    # does not return, exits with status code
-    def run_from_cmd_line
-      status = run(ARGV)
-      exit(status)
-    end
-
-    # return exit status (0 for success)
+    # returns on success, raises otherwise
     def run(args)
       # force immediate output
       $stdout.sync = true
 
       # setup logging
       Giblog.setup
+      Giblog.logger.level = Logger::INFO
 
       # Parse cmd line
-      cmdline = CmdLineParser.new args
-      Giblog.logger.debug { "cmd line args: #{cmdline.args}" }
+      cmdline = CmdLine.new.parse(args)
+      Giblog.logger.level = cmdline.log_level
+
+      Giblog.logger.debug { "cmd line args: #{cmdline.inspect}" }
+
+      # build a tree of files matching user's regexp selection
+      src_tree = PathTree.build_from_fs(cmdline.srcdir) do |p|
+        if cmdline.exclude_regex&.match(p.to_s)
+          false
+        else
+          (cmdline.include_regex =~ p.to_s)
+        end
+      end
+      if src_tree.nil?
+        Giblog.logger.warn { "Did not find any files to convert" }
+        return
+      end
+
+      app = Configurator.new(cmdline, src_tree)
+      app.tree_converter.run
 
-      exit_code = execute_conversion(cmdline)
-      Giblog.logger.info { "Giblish is done!" } if exit_code.zero?
-      exit_code
+      Giblog.logger.info { "Giblish is done!" }
+    end
+
+    # does not return, exits with status code
+    def run_from_cmd_line
+      begin
+        run(ARGV)
+        exit_code = 0
+      rescue => exc
+        Giblog.logger.error { exc.message }
+        Giblog.logger.error { exc.backtrace }
+        exit_code = 1
+      end
+      exit(exit_code)
+    end
+  end
+
+  class DirTreeConvert
+    # This class provides a file as the source for the asciidoc info and
+    # sets the document attributes required by Asciidoctor to resolve
+    # 'imagesdir' et al.
+    class AdocFileProvider
+      def adoc_source(src_node, dst_node, dst_top)
+        File.read(src_node.pathname)
+      end
+
+      def document_attributes(src_node, dst_node, dst_top)
+        p = src_node.pathname
+        {
+          "docfile" => p.to_s,
+          "docdir" => p.dirname.to_s,
+          "docname" => p.basename.to_s
+        }
+      end
+    end
+
+    def initialize(user_opts)
+      @user_opts = user_opts.dup
+
+      # get all adoc source files from disk
+      o = @user_opts
+      @src_tree = build_src_tree(o.srcdir, o.include_regex, o.exclude_regex)
+    end
+
+    # returns on success, raises otherwise
+    def run(configurator = nil)
+      return if @src_tree.nil?
+
+      # assign/setup a configurator containing all api options and doc attributes
+      build_config = configurator || Configurator.new(@user_opts)
+
+      tc = setup_converter(@src_tree, AdocFileProvider.new, build_config)
+      tc.run
     end
 
     private
 
-    # Convert using given args
-    # return exit code (0 for success)
-    def execute_conversion(cmdline)
-      conv_error = false
-      begin
-        conv_error = converter_factory(cmdline).convert
-      rescue StandardError => e
-        log_error e
-        conv_error = true
+    # build a tree of files matching user's regexp selection
+    def build_src_tree(srcdir, include_regex, exclude_regex)
+      pt = PathTree.build_from_fs(srcdir) do |p|
+        if exclude_regex&.match(p.to_s)
+          false
+        else
+          include_regex =~ p.to_s
+        end
+      end
+      if pt.nil?
+        Giblog.logger.warn { "Did not find any files to convert!" }
+        Giblog.logger.warn { "Built srctree using srcdir: #{srcdir} include_regex: #{include_regex} exclude_regex: #{exclude_regex}" }
+      end
+      pt
+    end
+
+    def setup_converter(src_tree, adoc_src_provider, configurator)
+      # compose the doc attribute provider.
+      configurator.doc_attr.add_doc_attr_providers(adoc_src_provider)
+      # NOTE: The order in the line below is important!
+      data_provider = DataDelegator.new(configurator.doc_attr, adoc_src_provider)
+
+      # associate the data providers with each source node in the tree
+      src_tree.traverse_preorder do |level, node|
+        next unless node.leaf?
+
+        node.data = data_provider
       end
-      conv_error ? 1 : 0
+
+      TreeConverter.new(src_tree, @user_opts.dstdir, configurator.build_options)
+    end
+  end
+
+  # Converts a number of branches/tags in a gitrepo according to the given
+  # options.
+  #
+  # Each branch/tag is converted into a subdir of the given root dir and a
+  # summary page with links to the converted docs for each branch/tag is
+  # generated within the root dir.
+  class GitRepoConvert
+    def initialize(user_opts)
+      raise ArgumentError, "No selection for git branches or tags were found!" unless user_opts.branch_regex || user_opts.tag_regex
+
+      @user_opts = user_opts.dup
+
+      # cache the root dir
+      @dst_topdir = @user_opts.dstdir
+
+      @gm = GitCheckoutManager.new(@user_opts)
     end
 
-    # return the converter corresponding to the given cmd line
-    # options
-    def converter_factory(cmdline)
-      if cmdline.args[:gitRepoRoot]
-        Giblog.logger.info { "User asked to parse a git repo" }
-        GitRepoConverter.new(cmdline.args)
-      else
-        FileTreeConverter.new(cmdline.args)
+    def run
+      # convert all docs found in the branches/tags that the user asked to parse
+      @gm.each_checkout do |name|
+        # tweak the destination dir to a subdir per branch/tag
+        @user_opts.dstdir = @dst_topdir / Giblish.to_fs_str(name)
+
+        Giblog.logger.debug { "cmdline: #{@user_opts.inspect}" }
+        configurator = GitRepoConfigurator.new(@user_opts, @gm.repo_root)
+        DirTreeConvert.new(@user_opts).run(configurator)
+      rescue => e
+        Giblog.logger.error { "Conversion of #{name} failed!" }
+        raise e if @user_opts.abort_on_error
+        Giblog.logger.error { e.message }
       end
+      make_summary
+    end
+
+    def make_summary
+      # reset the dst dir to the user-given-top
+      @user_opts.dstdir = @dst_topdir
+
+      # Make sure the summary page is just 'bare-bone'
+      @user_opts.make_searchable = nil
+      @user_opts.copy_asset_folders = nil
+      @user_opts.no_index = true
+      @user_opts.resolve_docid = false
+      @user_opts.doc_attributes["table-caption"] = nil
+
+      # assign/setup the doc_attr and layout using the same user options as
+      # for the adoc source files on each checkout
+      conf = Configurator.new(@user_opts)
+      s = @gm.summary_provider
+      s.index_basename = conf.config_opts.index_basename
+      data_provider = DataDelegator.new(
+        SrcFromString.new(s.source),
+        conf.doc_attr
+      )
+      srctree = PathTree.new("/" + conf.config_opts.index_basename + ".adoc", data_provider)
+      TreeConverter.new(srctree, @dst_topdir, conf.build_options).run
     end
+  end
+
+  class EntryPoint
+    def initialize(args, logger = nil)
+      # force immediate output
+      # $stdout.sync = true
 
-    def log_error(exc)
-      Giblog.logger.error do
-        <<~ERR_MSG
-          Error: #{exc.message}
-          Backtrace:
-          \t#{exc.backtrace.join("\n\t")}
+      # setup logging
+      Giblog.setup(logger)
+      Giblog.logger.level = Logger::INFO
+
+      # Parse cmd line
+      user_opts = CmdLine.new.parse(args)
+      Giblog.logger.level = user_opts.log_level
+      Giblog.logger.debug { "cmd line args: #{user_opts.inspect}" }
+
+      # Select the coversion instance to use
+      @converter = select_conversion(user_opts)
+    end
+
+    def run
+      # do the conversion
+      @converter.run
+    end
+
+    def self.run(args, logger = nil)
+      EntryPoint.new(args, logger).run
+    end
+
+    # does not return, exits with status code
+    def self.run_from_cmd_line
+      begin
+        EntryPoint.run(ARGV)
+        Giblog.logger.info { "Giblish is done!" }
+        exit_code = 0
+      rescue => exc
+        Giblog.logger.error { exc.message }
+        Giblog.logger.error { exc.backtrace }
+        exit_code = 1
+      end
+      exit(exit_code)
+    end
+
+    private
 
-          cmdline.usage
-        ERR_MSG
+    def select_conversion(user_opts)
+      case user_opts
+        in {branch_regex: _} | {tag_regex: _}
+          GitRepoConvert.new(user_opts)
+        else
+          DirTreeConvert.new(user_opts)
       end
     end
   end