giblish 0.7.6 → 1.0.0.rc2

data/lib/giblish/utils.rb

@@ -1,23 +1,27 @@
 require "logger"
 require "pathname"
 require "fileutils"
+require "find"
 
-class GiblogFormatter
-  def call severity, datetime, progname, msg
-    "#{datetime.strftime('%H:%M:%S')} #{severity} - #{msg}\n"
+# The logger used from within giblish
+class Giblog
+  # Defines the format for log messages from giblish.
+  class GiblogFormatter
+    def call(severity, datetime, _progname, msg)
+      "#{datetime.strftime("%H:%M:%S")} #{severity} - #{msg}\n"
+    end
   end
-end
 
-class Giblog
-  def self.setup
+  # bootstrap the application-wide logger object
+  def self.setup(logger = nil)
+    @logger = logger unless logger.nil?
     return if defined? @logger
-    @logger = Logger.new(STDOUT)
+
+    @logger = Logger.new($stdout)
     @logger.formatter = GiblogFormatter.new
-    # @logger.formatter = proc do |severity, datetime, _progname, msg|
-    #   "#{datetime.strftime('%H:%M:%S')} #{severity} - #{msg}\n"
-    # end
   end
 
+  # returns the application-wide logger instance.
   def self.logger
     unless defined? @logger
       puts "!!!! Error: Trying to access logger before setup !!!!"
@@ -28,282 +32,90 @@ class Giblog
   end
 end
 
-
-
 # Public: Contains a number of generic utility methods.
 module Giblish
-
-  class UserInfoFormatter
-    SEVERITY_LABELS = { 'WARN' => 'WARNING', 'FATAL' => 'FAILED' }
-
-    # {:text=>"...",
-    #  :source_location=>#<Asciidoctor::Reader::Cursor:0x000055e65a8729e0
-    #          @file="<full adoc filename>",
-    #          @dir="<full src dir>",
-    #          @path="<only file name>",
-    #          @lineno=<src line no>
-    # }
-    def call severity, datetime, progname, msg
-      message = case msg
-                when ::String
-                  msg
-                when ::Hash
-                  # asciidoctor seem to emit a hash with error text and source location info
-                  # for warnings and errors
-                  str = ""
-                  str << "Line #{msg[:source_location].lineno.to_s} - " if msg[:source_location]
-                  str << "#{msg[:text].to_s}" if msg[:text]
-                  str
-                else
-                  msg.inspect
-                end
-      %(#{datetime.strftime('%H:%M:%S')} #{progname}: #{SEVERITY_LABELS[severity] || severity}: #{message}\n)
-    end
-  end
+  # This logger is customized to receive log messages via the Asciidoctor API.
+  # It parses the messages and 'source_location' objects from the Asciidoctor API
+  # into messages using an opinionated format.
+  #
+  # The output is written to both $stdout and an in-memory StringIO instance. The log level
+  # can be set separately for each of these output channels.
   class AsciidoctorLogger < ::Logger
+    attr_reader :max_severity, :in_mem_storage
 
-    attr_reader :max_severity
-    attr_reader :user_info_str
-
-    def initialize user_info_log_level
-      super(STDOUT,progname: "(from asciidoctor)", formatter: UserInfoFormatter.new)
-      @user_info_str = StringIO.new
-      @user_info_logger = ::Logger.new(@user_info_str, formatter: UserInfoFormatter.new, level: user_info_log_level)
-    end
-
-    def add severity, message = nil, progname = nil
-      if (severity ||= UNKNOWN) > (@max_severity ||= severity)
-        @max_severity = severity
-      end
-      @user_info_logger.add(severity,message,progname)
-      super
-    end
-  end
-
-  class DeploymentPaths
-
-    attr_reader :web_path
-
-    def initialize(web_path, search_asset_path)
-      @search_assets_path = if search_asset_path.nil?
-                              nil
-                            else
-                              Pathname.new("/#{search_asset_path.to_s}").cleanpath
-                            end
-      @web_path = if web_path.nil?
-                    nil
-                  else
-                    Pathname.new("/#{web_path.to_s}/web_assets").cleanpath
-                  end
-    end
+    # log formatter specialized for formatting messages from
+    # asciidoctor's stdout, handles the different log record types that Asciidoctor
+    # emits
+    class UserInfoFormatter
+      SEVERITY_LABELS = {"WARN" => "WARNING", "FATAL" => "FAILED"}.freeze
 
-    def search_assets_path(branch_dir=nil)
-      if branch_dir.nil?
-        @search_assets_path
-      else
-        @search_assets_path.join(branch_dir)
+      def call(severity, datetime, progname, msg)
+        %(#{datetime.strftime("%H:%M:%S")} #{progname}: #{SEVERITY_LABELS[severity] || severity}: #{UserInfoFormatter.adoc_to_message(msg)}\n)
       end
-    end
-
-    def search_assets_path=(path)
-      @search_assets_path = path
-    end
-  end
-
-  # Helper class to ease construction of different paths for input and output
-  # files and directories
-  class PathManager
-    attr_reader :src_root_abs
-    attr_reader :dst_root_abs
-    attr_reader :resource_dir_abs
-    attr_reader :search_assets_abs
-
-    # Public:
-    #
-    # src_root - a string or pathname with the top directory of the input file
-    #            tree
-    # dst_root - a string or pathname with the top directory of the output file
-    #            tree
-    # resource_dir - a string or pathname with the directory containing
-    #                resources
-    # create_search_asset_dir - true if this instance shall create a dir for storing
-    #                search artefacts, false otherwise
-    def initialize(src_root, dst_root, resource_dir = nil,create_search_asset_dir=false)
-      # Make sure that the source root exists in the file system
-      @src_root_abs = Pathname.new(src_root).realpath
-      self.dst_root_abs = dst_root
-
-      self.search_assets_abs = create_search_asset_dir ?
-                                   @dst_root_abs.join("search_assets") : nil
-
-      # Make sure that the resource dir exists if user gives a path to it
-      resource_dir && (@resource_dir_abs = Pathname.new(resource_dir).realpath)
-    end
 
-    def search_assets_abs=(path)
-      if path.nil?
-        @search_assets_abs = nil
-        return
+      # The hash that can be emitted as the msg from asciidoctor have the
+      # following format:
+      # {:text=>"...",
+      #  :source_location=>#<Asciidoctor::Reader::Cursor:0x000055e65a8729e0
+      #          @file="<full adoc filename>",
+      #          @dir="<full src dir>",
+      #          @path="<only file name>",
+      #          @lineno=<src line no>
+      # }
+      def self.adoc_to_message(msg)
+        case msg
+        when ::String
+          msg
+        when ::Hash
+          # asciidoctor seem to emit a hash with the following structure on errors:
+          # :text => String
+          # :source_location => Reader::Cursor with the following props:
+          #   dir, file, lineno, path
+          # Only the lineno prop can be trusted when Asciidoctor is used via Giblish
+          #
+          src_loc = msg.fetch(:source_location, nil)
+          err_txt = msg.fetch(:text, "")
+          str = ""
+          str << "Line #{src_loc.lineno} - " if src_loc.lineno
+          str << err_txt
+          str
+        else
+          msg.inspect
+        end
       end
-      # Make sure that the destination root exists and expand it to an
-      # absolute path
-      dir = Pathname.new(path)
-      dir.mkpath
-      @search_assets_abs = dir.realpath
     end
 
-    def dst_root_abs=(dst_root)
-      # Make sure that the destination root exists and expand it to an
-      # absolute path
-      Pathname.new(dst_root).mkpath
-      @dst_root_abs = Pathname.new(dst_root).realpath
-    end
-
-    # Public: Get the relative path from the source root dir to the
-    #         directory where the supplied path points.
-    #
-    # in_path - an absolute or relative path to a file or dir
-    def reldir_from_src_root(in_path)
-      p = self.class.closest_dir in_path
-      p.relative_path_from(@src_root_abs)
-    end
-
-    # Public: Get the relative path from the
-    #         directory where the supplied path points to
-    #         the src root dir
+    # The level is one of the standard ::Logger levels
     #
-    # path - an absolute or relative path to a file or dir
-    def reldir_to_src_root(path)
-      src = self.class.closest_dir path
-      @src_root_abs.relative_path_from(src)
+    # stdout_level:: the log level to use for gating the messages to stdout
+    # string_level:: the log level to use for gating the messages to the in-memory string.
+    # defaults to 'stdout_level' if not set.
+    def initialize(user_logger, string_level = nil)
+      # def initialize(stdout_level, string_level = nil)
+      string_level = stdout_level if string_level.nil?
+      @user_logger = user_logger
+
+      @max_severity = UNKNOWN
+
+      # create a new, internal logger that echos messages to an in-memory string
+      @in_mem_storage = StringIO.new
+      # @in_mem_logger = ::Logger.new(@in_mem_storage, formatter: UserInfoFormatter.new, level: string_level)
+      super(@in_mem_storage, progname: "(asciidoctor)", formatter: UserInfoFormatter.new, level: string_level)
     end
 
-    # Public: Get the relative path from the dst root dir to the
-    #         directory where the supplied path points.
-    #
-    # path - an absolute or relative path to a file or dir
-    def reldir_from_dst_root(path)
-      dst = self.class.closest_dir path
-      dst.relative_path_from(@dst_root_abs)
-    end
+    def add(severity, message = nil, progname = nil, &block)
+      # add message to adoc log
+      super(severity, message.dup, progname)
 
-    # Public: Get the relative path from the
-    #         directory where the supplied path points to
-    #         the dst root dir
-    #
-    # path - an absolute or relative path to a file or dir
-    def reldir_to_dst_root(path)
-      dst = self.class.closest_dir path
-      @dst_root_abs.relative_path_from(dst)
-    end
+      # add message to user log (wierd interface... see Logger::add(...))
+      message = yield if block
+      message ||= progname
+      @user_logger&.add(severity, "(asciidoctor) #{UserInfoFormatter.adoc_to_message(message)}")
 
-    # return the destination dir corresponding to the given src path
-    # the src path must exist in the file system
-    def dst_abs_from_src_abs(src_path)
-      src_abs = (self.class.to_pathname src_path).realpath
-      src_rel = reldir_from_src_root src_abs
-      @dst_root_abs.join(src_rel)
+      # update the maximum severity received by this logger
+      @max_severity = severity if severity != UNKNOWN && severity > @max_severity
     end
-
-    # return the relative path from a generated document to
-    # the supplied folder given the corresponding absolute source
-    # file path
-    def relpath_to_dir_after_generate(src_filepath,dir_path)
-      dst_abs = dst_abs_from_src_abs(src_filepath)
-      dir = self.class.to_pathname(dir_path)
-      dir.relative_path_from(dst_abs)
-    end
-
-    def adoc_output_file(infile_path, extension)
-      # Get absolute source dir path
-      src_dir_abs = self.class.closest_dir infile_path
-
-      # Get relative path from source root dir
-      src_dir_rel = src_dir_abs.relative_path_from(@src_root_abs)
-
-      # Get the destination path relative the absolute source
-      # root
-      dst_dir_abs = @dst_root_abs.realpath.join(src_dir_rel)
-
-      # return full file path with correct extension
-      dst_dir_abs + get_new_basename(infile_path, extension)
-    end
-
-    # Public: Get the path to the directory where to generate the given
-    #         file. The path is given as the relative path from the source adoc
-    #         file to the desired output directory (required by the Asciidoctor
-    #         API).
-    #
-    # infile_path - a string or Pathname containing the absolute path of the
-    #               source adoc file
-    #
-    # Returns: a Pathname with the relative path from the source file to the
-    #          output directory
-    def adoc_output_dir(infile_path)
-      # Get absolute source dir path
-      src_abs = self.class.closest_dir infile_path
-
-      # Get relative path from source root dir
-      src_rel = src_abs.relative_path_from(@src_root_abs)
-
-      # Get the destination path relative the absolute source
-      # root
-      dst_abs = @dst_root_abs.realpath.join(src_rel)
-      dst_abs.relative_path_from(src_abs)
-    end
-
-    # return a pathname, regardless if the given path is a Pathname or
-    # a string
-    def self.to_pathname(path)
-      path.is_a?(Pathname) ? path : Pathname.new(path)
-    end
-
-    # Public: Get the basename for a file by replacing the file
-    #         extention of the source file with the supplied one.
-    #
-    # src_filepath - the full path of the source file
-    # file_ext - the file extention of the resulting file name
-    #
-    # Example
-    #
-    # Giblish::PathManager.get_new_basename(
-    #  "/my/old/file.txt","pdf") => "file.pdf"
-    #
-    # Returns: the basename of a file that uses the supplied file extention.
-    def self.get_new_basename(src_filepath, file_ext)
-      p = Pathname.new src_filepath
-      newname = p.basename.to_s.reverse.sub(p.extname.reverse, ".").reverse
-      newname << file_ext
-    end
-
-    # Public: Return the absolute path to the closest directory (defined as
-    #          - the parent dir when called with an existing file
-    #          - the directory itself when called with an existing directory
-    #          - the parent dir when called with a non-existing file/directory
-    def self.closest_dir(in_path)
-      sr = self.to_pathname(in_path)
-      if sr.exist?
-        sr.directory? ? sr.realpath : sr.dirname.realpath
-      else
-        sr.parent.expand_path
-      end
-    end
-
-    # Public: Find the root directory of the git repo in which the
-    #         given dirpath resides.
-    #
-    # dirpath - a relative or absolute path to a directory that resides
-    #           within a git repo.
-    #
-    # Returns: the root direcotry of the git repo or nil if the input path
-    #          does not reside within a git repo.
-    def self.find_gitrepo_root(dirpath)
-      Pathname.new(dirpath).realpath.ascend do |p|
-        git_dir = p.join(".git")
-        return p if git_dir.directory?
-      end
-    end
-  end # end of PathManager
+  end
 
   # Helper method that provides the user with a way of processing only the
   # lines within the asciidoc header block.
@@ -312,13 +124,15 @@ module Giblish
   # ex:
   # process_header_lines(file_path) do |line|
   #   if line == "Quack!"
-  #      puts "Donald!"
+  #      myvar = "Donald!"
   #      1
   #   else
   #      nil
   #   end
   # end
-  def process_header_lines(lines)
+  def process_header_lines(lines, &block)
+    return unless block
+
     state = "before_header"
     lines.each do |line|
       case state
@@ -337,15 +151,17 @@ module Giblish
   # ex:
   # process_header_lines_from_file(file_path) do |line|
   #   if line == "Quack!"
-  #      puts "Donald!"
+  #      myvar = "Donald!"
   #      1
   #   else
   #      nil
   #   end
   # end
-  def process_header_lines_from_file(path)
+  def process_header_lines_from_file(path, &block)
+    return unless block
+
     lines = File.readlines(path)
-    process_header_lines(lines,&Proc.new)
+    process_header_lines(lines, &block)
   end
   module_function :process_header_lines_from_file
 
@@ -362,9 +178,15 @@ module Giblish
   module_function :with_captured_stderr
 
   # transforms strings to valid asciidoctor id strings
-  def to_valid_id(input_str,id_prefix="_", id_separator="_")
-    id_str = input_str.strip.downcase.gsub(%r{[^a-z0-9]+}, id_separator)
-    id_str = "#{id_prefix}#{id_str}"
+  # in some cases you want to add a semi-unique checksum. If you do, you are no longer
+  # compatible with asciidoctor's generated ids
+  def to_valid_id(input_str, id_prefix = "_", id_separator = "_", use_checksum = false)
+    # use a basic checksum to reduce the risk for duplicate ids
+    check_sum = use_checksum ? input_str.chars.reduce(0) { |sum, c| sum + c.ord } : ""
+
+    id_str = input_str.strip.downcase.gsub(/[^a-z0-9]+/, id_separator)
+    id_str = "#{id_prefix}#{check_sum}#{id_prefix}#{id_str}"
+    id_str.gsub!(/#{Regexp.quote(id_separator)}+/, id_separator)
     id_str.chomp(id_separator)
   end
   module_function :to_valid_id
@@ -375,58 +197,82 @@ module Giblish
   # Ex
   #   which('ruby') #=> /usr/bin/ruby
   def which(cmd)
-    exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
-    ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
-      exts.each { |ext|
+    exts = ENV["PATHEXT"] ? ENV["PATHEXT"].split(";") : [""]
+    ENV["PATH"].split(File::PATH_SEPARATOR).each do |path|
+      exts.each do |ext|
         exe = File.join(path, "#{cmd}#{ext}")
         return exe if File.executable?(exe) && !File.directory?(exe)
-      }
+      end
     end
-    return nil
+    nil
   end
   module_function :which
 
-
-  # returns raw html that displays a search box to let the user
-  # acces the text search functionality.
+  # Convert a string into a string where all characters forbidden as part of
+  # filenames are replaced by an underscore '_'.
   #
-  # css          - the name of the css file to use for the search result layout
-  # cgi_path     - the (uri) path to a cgi script that implements the server side
-  #                functionality of searching the text
-  # opts:
-  # :web_assets_top => string   # the path to the 'web_assets' dir as seen when serving
-  #                               the web server (eg www.mysite.com/blah/doc_root ->
-  #                               web_assets_top shall be '/blah/doc_root')
-  # :search_assets_top => string   # the path to where the 'heading.json' file is located (
-  #                                  as seen from the local file system on the machine that
-  #                                  runs the search script)
-  def generate_search_box_html(css, cgi_path, opts)
-
-    # Replace the button with the below to use a text-only version of the btn
-    # <button id="search" type="submit">Search</button>
-    <<~SEARCH_INFO
-      ++++
-        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
-        <form class="example" action="#{cgi_path}" style="margin:20px 0px 20px 0px;max-width:790px">
-            Search all documents: 
-            <input id="searchphrase" type="text" placeholder="Search.." name="searchphrase"/>
-            <button id="search" type="submit"><i class="fa fa-search"></i></button>
-            <br>
+  # returns:: a String most likely a valid filename in windows & linux
+  #
+  # A comprehensive list of forbidden chars in different file systems can be
+  # found here: https://stackoverflow.com/a/31976060
+  # In short, chars forbidden in any of Windows and Linux are:
+  # / < > : " \\ | ? *
+  def to_fs_str(str)
+    # printable chars -> '_'
+    tmp = str.gsub(/[\/<>:"\\|?*]/, "_")
+    # non-printable chars -> '_'
+    tmp.gsub!(/[\x00-\x1F]/, "_")
+    # remove heading/trailing spaces
+    tmp.strip!
+    # Windows disallows files ending in '.'
+    tmp += "_" if tmp.end_with?(".")
+
+    tmp
+  end
+  module_function :to_fs_str
 
-            <input id="ignorecase" type="checkbox" value="true" name="ignorecase" checked/>
-            <label for="ignorecase">Ignore Case</label>
-            &nbsp;&nbsp;
-            <input id="useregexp" type="checkbox" value="true" name="useregexp"/>
-            <label for="useregexp">Use Regexp</label>
+  # Break a line into rows of max_length, using '-' semi-intelligently
+  # to split words if needed
+  #
+  # return:: an Array with the resulting rows
+  def break_line(line, max_length)
+    too_short = 3
+    return [line] if line.length <= too_short
+    raise ArgumentError, "max_length must be larger than #{too_short - 1}" if max_length < too_short
+
+    rows = []
+    row = ""
+
+    until line.empty?
+      word, _sep, _remaining = line.strip.partition(" ")
+      row_space = max_length - row.length
+
+      # start word with a space if row is not empty
+      sep = row.empty? ? "" : " "
+
+      # if word fits in row, just insert it and take next word
+      if row_space - (word.length + sep.length) >= 0
+        row = "#{row}#{sep}#{word}"
+        line = line.sub(word, "").strip
+        next
+      end
 
-            <input type="hidden" name="searchassetstop" value="#{opts[:search_assets_top]}"</input>
-            <input type="hidden" name="webassetstop" value="#{opts[:web_assets_top]}"</input>
-            #{'<input type="hidden" name="css" value="' + css +'"</input>' unless css.nil? }
-        </form>
-      ++++
+      # shall we split word or just move it to next row?
+      if (row_space == max_length && word.length > row_space) ||
+          (word.length > too_short && (row_space > too_short) && (word.length - row_space).abs > too_short)
+        # we will split the word, using a '-'
+        first_part = word[0..row_space - (1 + sep.length)]
+        row = "#{row}#{sep}#{first_part}-"
+        line = line.sub(first_part, "").strip
+      end
 
-    SEARCH_INFO
+      # start a new row
+      rows << row
+      row = ""
+    end
+    # need to add unfinished row if any
+    rows << row unless row.empty?
+    rows
   end
-  module_function :generate_search_box_html
-
+  module_function :break_line
 end

data/lib/giblish/version.rb

@@ -1,3 +1,3 @@
 module Giblish
-  VERSION = "0.7.6".freeze
+  VERSION = "1.0.0.rc2".freeze
 end

data/lib/giblish.rb

@@ -1,18 +1,21 @@
 #!/usr/bin/env ruby
+if /^2/.match?(RUBY_VERSION)
+  # suppress warnings for 'experimental' pattern matching
+  # for ruby versions < 3.x
+  require "warning"
+  Warning.ignore(/Pattern matching/)
+end
 
 require_relative "giblish/version"
-require_relative "giblish/utils"
-require_relative "giblish/core"
-require_relative "giblish/buildindex"
-require_relative "giblish/cmdline"
-require_relative "giblish/pathtree"
 require_relative "giblish/application"
+require_relative "giblish/search/request_manager"
+require_relative "giblish/github_trigger/webhook_manager"
 
 module Giblish
+  # The main entry point to the giblish application
   class << self
-
     def application
-      @application ||= Giblish::Application.new
+      @application ||= Giblish::EntryPoint
     end
   end
 end

data/scripts/hooks/post-receive.example

@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# this hook runs giblish on a document tree according to the 
+# users settings.
+# Typically used to publish html documents when a push to a matching
+# branch is detected.
+
+# the git refs that should trigger a doc generation at update
+TRIGGERING_REFS_REGEX=$'main'
+
+# the relative path to a subfolder in the repo where the docs are
+REPO_DOC_SUBFOLDER="docs"
+
+# the staging repo root (where the working tree exists)
+STAGING_REPO="/usr/local/git_staging/rillbert_se_staging"
+
+# The web-server top dir for the published documents
+DST_DIR="/var/www/rillbert_se/html/public/docs"
+
+# path to a top dir of the styling css and other resources
+RESOURCE_DIR="scripts/resources"
+
+# the name of the css file to use for styling, without the 'css' extension
+LAYOUT_STYLE="giblish"
+
+# "true" runs an 'rm -rf' of the destination dir before generating new htmls
+CLEAR_DST="true"
+
+echo "post-receive hook running..."
+
+# read the input that git sends to this hook
+read oldrev newrev ref
+
+# remove the 'refs/heads/' prefix from the git ref
+ref="${ref/refs\/heads\//}"
+
+# filter out refs that are irrelevant for doc generation
+if [[ ! "${ref}" =~ "${TRIGGERING_REFS_REGEX}" ]]; then
+  echo "Ref '${ref}' received. Doing nothing: only refs matching the regex: /${TRIGGERING_REFS_REGEX}/ will trigger a doc generation."
+  exit 0
+fi
+
+echo "Document generation triggered by an update to ${ref}."
+echo ""
+
+# use a subshell with the correct working dir for the actual doc generation
+(
+  cd "${STAGING_REPO}"
+
+  # need to unset the GIT_DIR env set by the invoking hook for giblish to work correctly
+  unset GIT_DIR
+
+  if [[ "${CLEAR_DST}" -eq "true" ]]; then
+    echo "Remove everything under ${DST_DIR}/"
+    rm -rf ${DST_DIR}/*
+  fi
+
+  # Generate html docs
+  echo "running giblish..."
+  giblish -a xrefstyle=basic \
+          --copy-asset-folders "_assets$" \
+          -g "${TRIGGERING_REFS_REGEX}" \
+          -r "${RESOURCE_DIR}" \
+          -s "${LAYOUT_STYLE}" \
+          "${REPO_DOC_SUBFOLDER}" "${DST_DIR}"
+)