giblish 0.8.2 → 1.0.0.rc2

data/lib/giblish/resourcepaths.rb

@@ -0,0 +1,150 @@
+require "pathname"
+
+module Giblish
+  # Provides relevant paths for layout resources based on the given options
+  class ResourcePaths
+    STYLE_EXTENSIONS = {
+      "html5" => ".css",
+      "html" => ".css",
+      "pdf" => ".yml",
+      "web-pdf" => ".css"
+    }
+
+    FONT_REGEX = /.*\.(ttf)|(TTF)$/
+    RESOURCE_DST_TOP_BASENAME = Pathname.new("web_assets")
+
+    # the relative path from the top of the resource area to the
+    # style file
+    attr_reader :src_style_path_rel
+
+    # the absolute path from the top of the resource area to the
+    # style file
+    attr_reader :src_style_path_abs
+
+    # a set with all dirs containing ttf files, paths are relative to resource area top
+    attr_reader :font_dirs_abs
+
+    # the relative path from the dst top dir to the copied style file (if it would be copied)
+    attr_reader :dst_style_path_rel
+
+    # the abs path to the top of the destination dir for resources
+    attr_reader :dst_resource_dir_abs
+
+    def initialize(cmd_opts)
+      @style_ext = STYLE_EXTENSIONS.fetch(cmd_opts.format, nil)
+      raise OptionParser::InvalidArgument, "Unsupported format: #{cmd_opts.format}" if @style_ext.nil?
+
+      # Cache all file paths in the resource area
+      r_top = cmd_opts.resource_dir
+      file_tree = PathTree.build_from_fs(r_top)
+
+      # find and validate paths
+      @dst_resource_dir_abs = cmd_opts.dstdir / RESOURCE_DST_TOP_BASENAME
+      @src_style_path_rel = find_style_file(file_tree, cmd_opts)
+      @src_style_path_abs = r_top / @src_style_path_rel
+      @dst_style_path_rel = RESOURCE_DST_TOP_BASENAME / @src_style_path_rel
+      @font_dirs_abs = find_font_dirs(file_tree)
+    end
+
+    private
+
+    # returns:: the relative path from the top of the file_tree to
+    # the style file
+    def find_style_file(file_tree, cmd_opts)
+      # Get all files matching the style name
+      style_basename = Pathname.new(cmd_opts.style_name).sub_ext(@style_ext)
+      style_tree = file_tree.match(/.*#{style_basename}$/)
+
+      # make sure we have exactly one css file with the given name
+      raise OptionParser::InvalidArgument, "Did not find #{style_basename} under #{file_tree.pathname}" if style_tree.nil?
+
+      l = style_tree.leave_pathnames(prune: true)
+      if l.count > 1
+        raise OptionParser::InvalidArgument, "Found #{l.count} instances of #{style_basename} under #{file_tree.pathname}. Requires exactly one."
+      end
+
+      # return the (pruned) path
+      l[0]
+    end
+
+    def find_font_dirs(file_tree)
+      tree = file_tree.match(FONT_REGEX)
+      dirs = Set.new
+      tree&.traverse_preorder do |level, node|
+        next unless node.leaf?
+
+        dirs << node.pathname.dirname
+      end
+      dirs
+    end
+  end
+
+  # copy everything under cmd_opts.resource_dir to
+  # dst_top/web_assets/.
+  class CopyResourcesPreBuild
+    # required opts:
+    # resource_dir:: Pathname to the top of the resource area to be copied
+    # style_name:: basename of the css to use for styling
+    # dstdir:: Pathname to the destination dir where the copied resources will be
+    # stored.
+    def initialize(cmd_opts)
+      @opts = cmd_opts.dup
+      @paths = ResourcePaths.new(cmd_opts)
+    end
+
+    def on_prebuild(src_tree, dst_tree, converter)
+      copy_resource_area(@opts)
+    end
+
+    private
+
+    # copy everyting under cmd_opts.resource_dir to @web_asset_dir
+    def copy_resource_area(cmd_opts)
+      web_assets_dir = @paths.dst_resource_dir_abs
+      web_assets_dir.mkpath unless web_assets_dir.exist?
+
+      resource_dir = cmd_opts.resource_dir.cleanpath.to_s + "/."
+
+      Giblog.logger&.info "Copy web assets (stylesheets et al) from #{resource_dir} to #{web_assets_dir}"
+      FileUtils.cp_r(
+        resource_dir,
+        web_assets_dir.to_s
+      )
+    end
+  end
+
+  class CopyAssetDirsPostBuild
+    def initialize(cmd_opts)
+      @asset_regex = cmd_opts.copy_asset_folders
+      @srcdir = cmd_opts.srcdir
+      @dstdir = cmd_opts.dstdir
+    end
+
+    # Called from TreeConverter during post build phase
+    #
+    # copy all directories matching the regexp pattern from the
+    # src tree to the dst tree
+    def on_postbuild(src_tree, dst_tree, converter)
+      return if @asset_regex.nil?
+
+      # build a tree with all dirs matching the given regexp
+      st = PathTree.build_from_fs(@srcdir, prune: true) do |p|
+        p.directory? && @asset_regex =~ p.to_s
+      end
+
+      return if st.nil?
+
+      Giblog.logger&.info "Copy asset directories from #{@srcdir} to #{@dstdir}"
+      st.traverse_preorder do |level, node|
+        next unless node.leaf?
+
+        n = node.relative_path_from(st)
+        src = @srcdir.join(n)
+        dst = @dstdir.join(n).dirname
+        dst.mkpath
+
+        FileUtils.cp_r(src.to_s, dst.to_s)
+      end
+    end
+  end
+end

data/lib/giblish/search/expand_adoc.rb

@@ -0,0 +1,55 @@
+require "asciidoctor"
+
+module Giblish
+  # Expands any 'include' preprocessor directives found in the given document
+  # and merges the the lines in the included document with the ones
+  # from the including document.
+  # Nesting of includes is supported to the given level.
+  #
+  # NOTE: Only includes of asciidoc files are supported, other includes (eg url)
+  # are silently dropped.
+  class ExpandAdoc
+    INCLUDE_DIRECTIVE_REGEX = /^(\\)?include::([^\[][^\[]*)\[(.+)?\]$/
+    def initialize(document, target_lines, max_depth = 3)
+      source_lines = document.reader.source_lines
+      source_lines.each do |line|
+        if INCLUDE_DIRECTIVE_REGEX =~ line
+          next unless max_depth > 0
+
+          p = resolve_include_path(document, $2, $3)
+          next if p.nil?
+
+          sub_doc = Asciidoctor.load_file(p, {parse: false, safe: :unsafe})
+          ExpandAdoc.new(sub_doc, target_lines, max_depth - 1)
+        else
+          target_lines << wash_line(document, line)
+        end
+      end
+    end
+
+    def resolve_include_path(document, target, attrlist)
+      target = replace_attrs(document.attributes, target)
+      parsed_attrs = document.parse_attributes attrlist, [], sub_input: true
+
+      # use an asciidoctor-internal method to resolve the path in an attempt to keep compatibility
+      inc_path, target_type, _relpath = document.reader.send(:resolve_include_path, target, attrlist, parsed_attrs)
+      return nil unless target_type == :file
+
+      inc_path
+    end
+
+    def wash_line(document, line)
+      replace_attrs(document.attributes, line)
+    end
+
+    # replace {a_doc_attr} with the value of the attribute
+    def replace_attrs(attrs, line)
+      # find all '{...}' occurrences
+      m_arr = line.scan(/\{\w+\}/)
+      # replace each found occurence with its doc attr if exists
+      m_arr.inject(line) do |memo, match|
+        attrs.key?(match[1..-2]) ? memo.gsub(match.to_s, attrs[match[1..-2]]) : memo
+      end
+    end
+  end
+end

data/lib/giblish/search/headingindexer.rb

@@ -0,0 +1,312 @@
+# frozen_string_literal: true
+
+require "erb"
+require "json"
+require "pathname"
+require "asciidoctor"
+require "asciidoctor/extensions"
+require_relative "../utils"
+require_relative "expand_adoc"
+
+module Giblish
+  # Implements both an Asciidoctor TreeProcessor hook and a giblish post-build
+  # hook.
+  #
+  # The TreeProcessor hook indexes all headings found in all
+  # documents in the tree and copies a 'washed' version of the source lines
+  # to a search asset folder in the destination tree.
+  #
+  # The post build hook copies the completed heading index to the search
+  # assets folder.
+  #
+  # Format of the heading index database:
+  # {
+  #   fileinfos : [{
+  #     filepath : filepath_1,
+  #     title : Title,
+  #     sections : [{
+  #       id : section_id_1,
+  #       title : section_title_1,
+  #       line_no : line_no
+  #     },
+  #     {
+  #       id : section_id_1,
+  #       title : section_title_1,
+  #       line_no : line_no
+  #     },
+  #     ...
+  #     ]
+  #   },
+  #   {
+  #     filepath : filepath_2,
+  #     ...
+  #   }]
+  # }
+  class HeadingIndexer < Asciidoctor::Extensions::TreeProcessor
+    HEADING_REGEX = /^=+\s+(.*)$/
+    ANCHOR_REGEX = /^\[\[(\w+)\]\]\s*$/
+    HEADING_DB_BASENAME = "heading_db.json"
+    SEARCH_ASSET_DIRNAME = "gibsearch_assets"
+
+    # src_topdir:: a Pathname to the top dir of the src files
+    def initialize(src_topdir)
+      super({})
+
+      @src_topdir = src_topdir
+      @heading_index = {fileinfos: []}
+    end
+
+    # called by Asciidoctor during the conversion of the document.
+    def process(document)
+      attrs = document.attributes
+      src_node = attrs["giblish-info"][:src_node]
+
+      # only index source files that reside on the 'physical' file system
+      return if src_node.nil? || !src_node.pathname.exist?
+
+      # make sure we use the correct id elements when indexing
+      # sections
+      opts = {
+        id_prefix: (attrs.key?("idprefix") ? attrs["idprefix"] : "_"),
+        id_separator: (attrs.key?("id_separator") ? attrs["id_separator"] : "_")
+      }
+
+      # index sections and wash source lines
+      # Copy the washed document to the search asset folder
+      dst_top = attrs["giblish-info"][:dst_top]
+      write_washed_doc(
+        parse_document(document, src_node, opts),
+        dst_top.pathname / SEARCH_ASSET_DIRNAME / rel_src_path(src_node)
+      )
+      nil
+    end
+
+    # called by the TreeConverter during the post_build phase
+    def on_postbuild(src_topdir, dst_tree, converter)
+      search_topdir = dst_tree.pathname / SEARCH_ASSET_DIRNAME
+
+      # store the JSON file
+      serialize_section_index(search_topdir, search_topdir)
+    end
+
+    private
+
+    # get the relative path from the src_topdirdir to the source node
+    #
+    # returns:: a Pathname with the relative path
+    def rel_src_path(src_node)
+      src_node.pathname.relative_path_from(@src_topdir)
+    end
+
+    # returns:: the source lines after substituting attributes
+    def parse_document(document, src_node, opts)
+      Giblog.logger.debug "index headings in #{src_node.pathname} using prefix '#{opts[:id_prefix]}' and separator '#{opts[:id_separator]}'"
+      attrs = document.attributes
+      expanded_lines = []
+      ExpandAdoc.new(document, expanded_lines, 3)
+      # puts expanded_lines
+
+      doc_info = index_sections(expanded_lines, opts)
+
+      @heading_index[:fileinfos] << {
+        filepath: rel_src_path(src_node),
+        title: attrs.key?("doctitle") ? attrs["doctitle"] : "No title found!",
+        sections: doc_info[:sections]
+      }
+      doc_info[:washed_lines]
+    end
+
+    # lines:: [lines]
+    # dst_path:: Pathname to destination file
+    def write_washed_doc(lines, dst_path)
+      Giblog.logger.debug { "Copy searchable text to #{dst_path}" }
+      dst_path.dirname.mkpath
+      File.write(dst_path.to_s, lines.join("\n"))
+    end
+
+    # replace {a_doc_attr} with the value of the attribute
+    def replace_attrs(attrs, line)
+      # find all '{...}' occurrences
+      m_arr = line.scan(/\{\w+\}/)
+      # replace each found occurence with its doc attr if exists
+      m_arr.inject(line) do |memo, match|
+        attrs.key?(match[1..-2]) ? memo.gsub(match.to_s, attrs[match[1..-2]]) : memo
+      end
+    end
+
+    # provide a 'washed' version of all source lines in the document and
+    # index all sections.
+    #
+    # returns:: { washed_lines: [lines], sections: [{:id, :title, :line_no}]}
+    def index_sections(lines, opts)
+      indexed_doc = {
+        washed_lines: [],
+        sections: []
+      }
+      sections = indexed_doc[:sections]
+      # lines = document.reader.source_lines.dup
+
+      line_no = 0
+      match_str = ""
+      state = :text
+      lines.each do |line|
+        line_no += 1
+        # line = replace_attrs(document.attributes, line)
+        indexed_doc[:washed_lines] << line.strip
+
+        # implement a state machine that supports both custom
+        # anchors for a heading and the default heading ids generated
+        # by asciidoctor
+        case state
+        when :text
+          case line
+          # detect a heading or an anchor preceeding a heading
+          when HEADING_REGEX
+            state = :heading
+            match_str = Regexp.last_match(1)
+          when ANCHOR_REGEX
+            state = :expecting_heading
+            match_str = Regexp.last_match(1)
+          end
+        when :expecting_heading
+          case line
+          when HEADING_REGEX
+            # we got a heading, index it
+            sections << {
+              "id" => match_str,
+              "title" => Regexp.last_match(1).strip,
+              "line_no" => line_no
+            }
+          else
+            # we did not get a heading, this is ok as well but we can not
+            # index it
+            Giblog.logger.debug do
+              "Did not index the anchor: #{match_str} at "\
+              "line #{line_no}, probably not associated with a heading."
+            end
+          end
+          state = :text
+        when :heading
+          # last line was a heading without an accompanying anchor, index it
+          # by creating a new, unique id that matches the one that
+          # asciidoctor will assign to this heading when generating html
+          sections << {
+            "id" => create_unique_id(sections, match_str, opts),
+            "title" => Regexp.last_match(1).strip,
+            "line_no" => line_no - 1
+          }
+          state = :text
+        end
+      end
+      indexed_doc
+    end
+
+    # find the section id delimiters from the document
+    # header, if they are set there
+    def find_section_id_attributes(lines, result)
+      Giblish.process_header_lines(lines) do |line|
+        m = /^:idprefix:(.*)$/.match(line)
+        n = /^:idseparator:(.*)$/.match(line)
+        result[:id_prefix] = m[1].strip if m && !result[:id_prefix]
+        result[:id_separator] = n[1].strip if n && !result[:id_separator]
+      end
+      result
+    end
+
+    # create the anchor id for the given heading in a way that complies
+    # with how asciidoctor creates the same id when generating html
+    def create_unique_id(sections, heading_str, opts)
+      # create the 'default' id the same way as asciidoctor will do it
+      id_base = Giblish.to_valid_id(heading_str, opts[:id_prefix], opts[:id_separator])
+      return id_base unless sections.find { |s| s["id"] == id_base }
+
+      # handle the case with several sections with the same name by adding
+      # a sequence number at the end
+      idx = 1
+      heading_id = ""
+      loop do
+        heading_id = "#{id_base}_#{idx += 1}"
+        break unless sections.find { |s| s["id"] == heading_id }
+      end
+      heading_id
+    end
+
+    # write the index to a file in dst_dir and remove the base_dir
+    # part of the path for each filename
+    def serialize_section_index(dst_dir, base_dir)
+      dst_dir.mkpath
+
+      heading_db_path = dst_dir.join(HEADING_DB_BASENAME)
+      Giblog.logger.info { "writing json to #{heading_db_path}" }
+
+      File.write(heading_db_path.to_s, @heading_index.to_json)
+    end
+  end
+
+  class AddSearchForm < Asciidoctor::Extensions::DocinfoProcessor
+    use_dsl
+    at_location :header
+
+    FORM_DATA = <<~FORM_HTML
+      <script type="text/javascript">
+      window.onload = function () {
+        document.getElementById("calingurl_input").value = window.location.href;
+      };
+      </script>
+
+      <style>
+      #gibsearch-form {
+        position:fixed;
+        top:0.5rem;
+        left:70%;
+        width:30%;
+        height:3rem;
+        background:white;
+        z-index:2000; 
+        }
+      </style>
+
+      <div id=gibsearch-form>
+      <form class="gibsearch" action="<%=action_path%>">
+        <input type="search" placeholder="Search the docs.." name="search-phrase" />
+        <button type="submit">Search</button>
+        <br>
+
+        <input type="checkbox" id="consider-case" name="consider-case" />
+        <label for="consider-case">case sensitive</label>
+        &nbsp;&nbsp;
+        <input type="checkbox" id="as-regexp" name="as-regexp" />
+        <label for="as-regexp">use regexp</label>
+
+        <input type="hidden" name="calling-url" id=calingurl_input />
+        <input type="hidden" name="search-assets-top-rel" value="<%=sa_top_rel%>"/>
+        <input type="hidden" name="css-path" value="<%=css_path%>"/>
+      </form>
+      </div>
+    FORM_HTML
+
+    def initialize(action_path = nil, opts = {})
+      @action_path = action_path
+      super(opts)
+    end
+
+    def process(document)
+      attrs = document.attributes
+      src_node = attrs["giblish-info"][:src_node]
+      dst_node = attrs["giblish-info"][:dst_node]
+      dst_top = attrs["giblish-info"][:dst_top]
+
+      to_top_rel = dst_top.relative_path_from(dst_node.parent)
+      sa_top_rel = to_top_rel.join("gibsearch_assets").cleanpath
+      action_path = @action_path || to_top_rel.join("gibsearch.cgi").cleanpath
+
+      # only include css_path if it is given in the document's attributes
+      doc_attrs = src_node.data.document_attributes(src_node, dst_node, dst_top)
+      css_path = if %w[stylesdir stylesheet].all? { |k| doc_attrs.key?(k) }
+        doc_attrs["stylesdir"] + "/" + doc_attrs["stylesheet"]
+      end
+
+      ERB.new(FORM_DATA).result(binding)
+    end
+  end
+end

data/lib/giblish/search/request_manager.rb

@@ -0,0 +1,110 @@
+require "asciidoctor"
+require_relative "textsearcher"
+
+module Giblish
+  class DefaultHtmlGenerator
+    # search_result:: a hash conforming to the output of
+    #                 the TextSearcher::search method.
+    # returns::       a string with the html to return to the client
+    def response(search_result, css_path = nil)
+      adoc_2_html(search_2_adoc(search_result), css_path)
+    end
+
+    private
+
+    def adoc_2_html(adoc_source, css_path = nil)
+      # setup default document attributes used in the html conversion
+      doc_attr = css_path ? {
+        "stylesdir" => css_path.dirname.to_s,
+        "stylesheet" => css_path.basename.to_s,
+        "linkcss" => true,
+        "copycss" => nil
+      } : {}
+      doc_attr["data-uri"] = 1
+      doc_attr["example-caption"] = nil
+
+      # setup default conversion options
+      converter_options = {
+        backend: "html5",
+        # need this to let asciidoctor include the default css if user
+        # has not specified any css
+        safe: Asciidoctor::SafeMode::SAFE,
+        header_footer: true,
+        attributes: doc_attr
+      }
+
+      # for debugging of adoc source
+      # File.write("search.adoc", adoc_source)
+
+      # convert to html and return result
+      Asciidoctor.convert(adoc_source, converter_options)
+    end
+
+    # search_result:: a hash conforming to the output of
+    #                 the TextSearcher::search method.
+    def search_2_adoc(search_result)
+      str = ""
+      search_result.each do |filepath, info|
+        str << ".From: '#{info[:doc_title]}'\n"
+        str << "====\n\n"
+        info[:sections].each do |section|
+          str << "#{section[:url]}[#{section[:title]}]::\n\n"
+          section[:lines].each do |line|
+            str << line
+          end.join("\n+\n")
+          str << "\n\n"
+        end
+        str << "\n"
+        str << "====\n"
+      end
+
+      <<~ADOC
+        = Search Result
+
+        #{str}
+      ADOC
+    end
+  end
+
+  # A gateway class that implements everything needed to produce
+  # an html page with search results given a search request.
+  #
+  # The class implements internal caching for better performance. It
+  # is thus probably wise to instantiate this class once and then use
+  # that instance for all subsequent search queries.
+  class RequestManager
+    # url_path_mappings:: a Hash with mappings from url paths to
+    # local file system directories.
+    # html_generator:: an object that generates html by implementing
+    # the method 'def response(search_result, css_path = nil)'. See
+    # eg DefaultHtmlGenerator. If nil, a DefaultHtmlGenerator is used.
+    def initialize(uri_mappings, html_generator = nil)
+      @uri_mappings = uri_mappings || {"/" => "/var/www/html/"}
+
+      @html_generator = html_generator || DefaultHtmlGenerator.new
+    end
+
+    # Return an html page with the search result from the given query.
+    #
+    # search_params:: a Hash containing the parameters of the search query.
+    def response(search_params)
+      sp = SearchParameters.from_hash(search_params, uri_mappings: @uri_mappings)
+      @html_generator.response(searcher.search(sp), sp.css_path)
+    end
+
+    private
+
+    # a convenience method to give shorter access to the class-wide
+    # TextSearcher instance.
+    def searcher
+      RequestManager.searcher
+    end
+
+    # provide a class-wide TextSearcher instance
+    class << self
+      def searcher
+        @searcher ||= TextSearcher.new(SearchRepoCache.new)
+      end
+    end
+  end
+end

data/lib/giblish/search/searchquery.rb

@@ -0,0 +1,68 @@
+module Giblish
+  class SearchQuery
+    attr_reader :parameters
+
+    REQUIRED_PARAMS = %w[calling-url search-assets-top-rel search-phrase]
+    OPTIONAL_PARAMS = %w[css-path consider-case as-regexp]
+
+    def initialize(uri: nil, query_params: nil)
+      @parameters = case [uri, query_params]
+        in [String, nil]
+          uri_2_params(uri)
+        in [nil, _]
+          validate_parameters(query_params)
+        else
+          raise ArgumentError, "You must supply one of 'uri: [String]' or 'query_params: [Hash]' got: #{query_params}"
+      end
+    end
+
+    def css_path
+      @parameters.key?("css-path") && !@parameters["css-path"].empty? ? Pathname.new(@parameters["css-path"]) : nil
+    end
+
+    def search_assets_top_rel
+      # convert to pathname and make sure we always are relative
+      Pathname.new("./" + @parameters["search-assets-top-rel"]).cleanpath
+    end
+
+    def consider_case?
+      @parameters.key?("consider-case")
+    end
+
+    def as_regexp?
+      @parameters.key?("as-regexp")
+    end
+
+    def method_missing(meth, *args, &block)
+      unless respond_to_missing?(meth)
+        super(meth, *args, &block)
+        return
+      end
+
+      @parameters[meth.to_s.tr("_", "-")]
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      @parameters.key?(method_name.to_s.tr("_", "-"))
+    end
+
+    private
+
+    def uri_2_params(uri_str)
+      uri = URI(uri_str)
+      raise ArgumentError, "No query parameters found!" if uri.query.nil?
+
+      parameters = URI.decode_www_form(uri.query).to_h
+      validate_parameters(parameters)
+    end
+
+    #  validate that all required parameters are included
+    def validate_parameters(uri_params)
+      REQUIRED_PARAMS.each do |p|
+        raise ArgumentError, "Missing or empty parameter: #{p}" if !uri_params.key?(p) || uri_params[p].empty?
+      end
+
+      uri_params
+    end
+  end
+end