giblish 0.3.1 → 0.4.0

data/lib/giblish/buildgraph.rb

@@ -53,7 +53,7 @@ module Giblish
           doc.src_file.to_s.eql? src_file
         end
         raise "Inconsistent docs when building graph!! found no match for #{src_file}" if d.nil?
-        result[d] = id_array
+        result[d] = id_array if d.converted
       end
       result
     end

data/lib/giblish/buildindex.rb

@@ -10,17 +10,20 @@ module Giblish
   # Base class with common functionality for all index builders
   class BasicIndexBuilder
     # set up the basic index building info
-    def initialize(processed_docs, path_manager, handle_docid = false)
+    def initialize(processed_docs, converter, path_manager, handle_docid = false)
       @paths = path_manager
       @nof_missing_titles = 0
       @processed_docs = processed_docs
+      @converter = converter
       @src_str = ""
       @manage_docid = handle_docid
     end
 
-    def source(dep_graph_exists = false)
+    def source(dep_graph_exists = false, make_searchable = false)
       <<~DOC_STR
-        #{generate_header}
+        #{generate_title_and_header}
+        #{generate_date_info}
+        #{add_search_box if make_searchable}
         #{generate_tree(dep_graph_exists)}
         #{generate_details}
         #{generate_footer}
@@ -28,28 +31,60 @@ module Giblish
     end
 
     protected
+    def generate_title_and_header
+      <<~DOC_HEADER
+        = Document index
+        from #{@paths.src_root_abs}
+        :icons: font
+
+      DOC_HEADER
+    end
+
     # return the adoc string for displaying the source file
     def display_source_file(doc_info)
       <<~SRC_FILE_TXT
         Source file::
-        #{doc_info.srcFile_utf8}
+        #{doc_info.src_file}
 
       SRC_FILE_TXT
     end
 
-    def generate_header
+    def generate_date_info
       t = Time.now
       <<~DOC_HEADER
-        = Document index
-        from #{@paths.src_root_abs}
-
-        Generated by Giblish at::
-
-        #{t.strftime('%Y-%m-%d %H:%M')}
-
+        *Generated by Giblish at:* #{t.strftime('%Y-%m-%d %H:%M')}
       DOC_HEADER
     end
 
+    def add_search_box
+      # TODO: Fix the hard-coded path
+      cgi_path = "/cgi-bin/giblish-search.cgi"
+      css = @converter.converter_options[:attributes]["stylesheet"]
+
+      # button with magnifying glass icon (not working when deployed)
+      # <button id="search" type="submit"><i class="fa fa-search"></i></button>
+      <<~SEARCH_INFO
+      ++++
+        <form class="example" action="#{cgi_path}" style="margin:20px 0px 20px 0px;max-width:380px">
+            Search all documents: 
+            <input id="searchphrase" type="text" placeholder="Search.." name="searchphrase"/>
+            <button id="search" type="submit">Search</button>
+            <br>
+
+            <input id="ignorecase" type="checkbox" value="true" name="ignorecase" checked/>
+            <label for="ignorecase">Ignore Case</label>
+            &nbsp;&nbsp;
+            <input id="useregexp" type="checkbox" value="true" name="regexp"/>
+            <label for="useregexp">Use Regexp</label>
+
+            <input type="hidden" name="topdir" value="#{@paths.dst_root_abs.to_s}"</input>
+            <input type="hidden" name="reltop" value="#{@paths.reldir_from_web_root(@paths.dst_root_abs)}"</input>
+            <input type="hidden" name="css" value="#{css}"</input>
+        </form>
+      ++++
+
+      SEARCH_INFO
+    end
     def get_docid_statistics
       largest = ""
       clash = []
@@ -91,8 +126,7 @@ module Giblish
 
       if @manage_docid
         <<~DOC_ID_INFO
-        Document id numbers::
-        The generation of this repository uses document id numbers. #{docid_info_str} #{dep_graph_str}
+        *Document id numbers:* #{docid_info_str} #{dep_graph_str}
   
         #{docid_warn_str}
 
@@ -105,12 +139,6 @@ module Giblish
     def generate_tree(dep_graph_exists)
       # output tree intro
       tree_string = <<~DOC_HEADER
-        == Document Overview
-
-        _Click on the title to open the document or on `details` to see more
-        info about the document. A `(warn)` label indicates that there were
-        warnings while converting the document from its asciidoc format._
-
         #{generate_doc_id_info dep_graph_exists}
 
         [subs=\"normal\"]
@@ -172,8 +200,8 @@ module Giblish
                 doc_info.title
               end
 
-      [title, "<<#{doc_info.rel_path}#,#{title}>>".encode("utf-8"),
-       "<<#{Giblish.to_valid_id(doc_info.title)},details>>\n".encode("utf-8")]
+      [title, "<<#{doc_info.rel_path}#,#{title}>>",
+       "<<#{Giblish.to_valid_id(doc_info.title)},details>>\n"]
     end
 
     # Generate an adoc string that will display as
@@ -206,7 +234,7 @@ module Giblish
         tree_entry_converted prefix_str, d
       else
         # no converted file exists, show what we know
-        "#{prefix_str} FAIL: #{d.srcFile_utf8}      <<#{d.srcFile_utf8},details>>\n"
+        "#{prefix_str} FAIL: #{d.src_file}      <<#{d.src_file},details>>\n"
       end
     end
 
@@ -217,7 +245,7 @@ module Giblish
 
     def generate_detail_fail(d)
       <<~FAIL_INFO
-        === #{d.srcFile_utf8}
+        === #{d.src_file}
 
         #{display_source_file(d)}
 
@@ -288,16 +316,16 @@ module Giblish
 
   # A simple index generator that shows a table with the generated documents
   class SimpleIndexBuilder < BasicIndexBuilder
-    def initialize(processed_docs, path_manager, manage_docid = false)
-      super processed_docs, path_manager, manage_docid
+    def initialize(processed_docs, converter, path_manager, manage_docid = false)
+      super processed_docs, converter, path_manager, manage_docid
     end
   end
 
   # Builds an index of the generated documents and includes some git metadata
   # from the repository
   class GitRepoIndexBuilder < BasicIndexBuilder
-    def initialize(processed_docs, path_manager, manage_docid, git_repo_root)
-      super processed_docs, path_manager, manage_docid
+    def initialize(processed_docs, converter, path_manager, manage_docid, git_repo_root)
+      super processed_docs, converter, path_manager, manage_docid
 
       # no repo root given...
       return unless git_repo_root
@@ -328,14 +356,13 @@ module Giblish
     end
 
 
-    def generate_header
+    def generate_title_and_header
       t = Time.now
       <<~DOC_HEADER
         = Document index
         #{@git_repo.current_branch}
 
-        Generated by Giblish at::
-        #{t.strftime('%Y-%m-%d %H:%M')}
+        :icons:
 
       DOC_HEADER
     end

data/lib/giblish/cmdline.rb

@@ -69,6 +69,28 @@ class CmdLineParser
                              generate the documents and use the collected
                              doc ids to resolve relative paths between the
                              generated documents
+  -m --make-searchable       (only supported for html generation)
+                             take steps to make it possible to
+                             search the published content via a cgi-script. This 
+                             flag will do the following:
+                               1. index all headings in all source files and store
+                                  the result in a JSON file
+                               2. copy the JSON file and all source (adoc) files to
+                                  a 'search_assets' folder in the top-level dir of
+                                  the destination.
+                               3. add html code that displays a search field in the 
+                                  index page that will try to call the cgi-script
+                                  'giblish-search' when the user inputs some text. 
+                             To actually provide search functionality for a user, you 
+                             need to provide the cgi-script and configure your web-server
+                             to invoke it when needed. NOTE: The generated search box cgi 
+                             is currently hard-coded to look for the cgi script at the URL: 
+                             http://<your-web-domain>/cgi-bin/giblish-search.cgi
+                             E.g.
+                             http://example.com/cgi-bin/giblish-search.cgi
+                             An implementation of the giblish-search cgi-script is found
+                             within the lib folder of this gem, you can copy that to your 
+                             cgi-bin dir in your webserver and rename it from .rb to .cgi 
   --log-level                set the log level explicitly. Must be one of
                              debug, info (default), warn, error or fatal.
 ENDHELP
@@ -139,6 +161,7 @@ ENDHELP
         suppressBuildRef: false,
         localRepoOnly: false,
         resolveDocid: false,
+        make_searchable: false,
         webRoot: false
     }
 
@@ -163,6 +186,7 @@ ENDHELP
         when "-t", "--git-tags"     then next_arg = :gitTagRegexp
         when "-c", "--local-only"   then @args[:localRepoOnly] = true
         when "-d", "--resolve-docid" then @args[:resolveDocid] = true
+        when "-m", "--make-searchable" then @args[:make_searchable] = true
         when "-s", "--style"        then next_arg = :userStyle
         when "-w", "--web-root"     then next_arg = :webRoot
         when "--log-level"          then next_arg = :logLevel
@@ -181,6 +205,9 @@ ENDHELP
     if !@args[:resourceDir] && @args[:userStyle]
       puts "Error: The given style would not be used since no resource dir "\
            "was specified (-s specified without -r)"
+    elsif @args[:make_searchable] && @args[:format] != "html"
+      puts "Error: The --make-searchable option is only supported for "\
+           "html rendering"
     else
       return
     end

data/lib/giblish/core.rb

@@ -6,6 +6,7 @@ require "pathname"
 require_relative "buildindex"
 require_relative "docconverter"
 require_relative "docid"
+require_relative "indexheadings"
 require_relative "docinfo"
 require_relative "buildgraph"
 
@@ -27,10 +28,12 @@ module Giblish
       @paths = Giblish::PathManager.new(
           @options[:srcDirRoot],
           @options[:dstDirRoot],
-          @options[:resourceDir]
+          @options[:resourceDir],
+          @options[:webRoot]
       )
       @processed_docs = []
       @converter = converter_factory
+      @search_assets_path = @paths.dst_root_abs.realpath.join("search_assets")
     end
 
     def convert
@@ -38,6 +41,9 @@ module Giblish
       # valid references to adoc files
       manage_doc_ids if @options[:resolveDocid]
 
+      # register add-on for handling searchability
+      manage_searchability if @options[:make_searchable]
+
       # traverse the src file tree and convert all files deemed as
       # adoc files
       Find.find(@paths.src_root_abs) do |path|
@@ -45,6 +51,9 @@ module Giblish
         to_asciidoc(p) if adocfile? p
       end if @paths.src_root_abs.directory?
 
+      # create necessary search assets if needed
+      create_search_assets if @options[:make_searchable]
+
       # check if we shall build index or not
       return if @options[:suppressBuildRef]
 
@@ -64,7 +73,7 @@ module Giblish
 
       # build a reference index
       ib = index_factory
-      @converter.convert_str ib.source(dep_graph_exist), @paths.dst_root_abs, "index"
+      @converter.convert_str ib.source(dep_graph_exist,@options[:make_searchable]), @paths.dst_root_abs, "index"
 
       # clean up cached files and adoc resources
       remove_diagram_temps if dep_graph_exist
@@ -77,7 +86,7 @@ module Giblish
     # user options
     def index_factory
       raise "Internal logic error!" if @options[:suppressBuildRef]
-      SimpleIndexBuilder.new(@processed_docs, @paths,
+      SimpleIndexBuilder.new(@processed_docs, @converter, @paths,
                              @options[:resolveDocid])
     end
 
@@ -112,7 +121,7 @@ module Giblish
 
       # the only info we have is the source file name
       info.converted = false
-      info.src_file = filepath
+      info.src_file = filepath.to_s
       info.error_msg = exception.message
 
       @processed_docs << info
@@ -121,6 +130,11 @@ module Giblish
 
     private
 
+    def create_search_asset_dir
+      Dir.exist?(@search_assets_path) || FileUtils.mkdir_p(@search_assets_path.to_s)
+      @search_assets_path
+    end
+
     # remove cache dir and svg image created by asciidoctor-diagram
     # when creating the document dependency graph
     def remove_diagram_temps
@@ -165,12 +179,57 @@ module Giblish
       return !ir.match(fs).nil?
     end
 
+    def manage_searchability
+      # register the extension
+      Giblish.register_index_heading_extension
+
+      # make sure we start from a clean slate
+      IndexHeadings.clear_index
+    end
+
+    # top_dir
+    # |- web_assets
+    # |- branch_1_top_dir
+    # |     |- index.html
+    # |     |- file1.html
+    # |     |- dir_1
+    # |     |   |- file2.html
+    # |- search_assets
+    # |     |- branch_1
+    # |           |- heading_index.json
+    # |           |- file1.adoc
+    # |           |- dir_1
+    # |           |   |- file2.html
+    # |           |- ...
+    # |     |- branch_2
+    # |           | ...
+    # |- branch_2_top_dir
+    # | ...
+    def create_search_assets
+      # get the proper dir for the search assets
+      assets_dir = create_search_asset_dir
+
+      # store the JSON file
+      IndexHeadings.serialize assets_dir, @paths.src_root_abs
+
+      # traverse the src file tree and copy all published adoc files
+      # to the search_assets dir
+      Find.find(@paths.src_root_abs) do |path|
+        p = Pathname.new(path)
+        next unless adocfile? p
+
+        dst_dir = assets_dir.join(@paths.reldir_from_src_root(p))
+        FileUtils.mkdir_p(dst_dir)
+        FileUtils.cp(p.to_s, dst_dir)
+      end if @paths.src_root_abs.directory?
+    end
+
     # Register the asciidoctor extension that handles doc ids and traverse
     # the source tree to collect all :docid: attributes found in document
     # headers.
     def manage_doc_ids
       # Register the docid preprocessor hook
-      Giblish.register_extensions
+      Giblish.register_docid_extension
 
       # Make sure that no prior docid's are hangning around
       DocidCollector.clear_cache
@@ -218,7 +277,7 @@ module Giblish
     protected
 
     def index_factory
-      GitRepoIndexBuilder.new(@processed_docs, @paths,
+      GitRepoIndexBuilder.new(@processed_docs, @converter, @paths,
                               @options[:resolveDocid], @options[:gitRepoRoot])
     end
 
@@ -305,7 +364,7 @@ module Giblish
       # Update needed base class members before converting a new checkout
       @processed_docs = []
       @paths.dst_root_abs = @master_paths.dst_root_abs.realpath.join(dir_name)
-      # @converter.paths = @paths
+      @search_assets_path = @master_paths.dst_root_abs.realpath.join("search_assets").join(dir_name)
 
       # Parse and convert docs using given args
       Giblog.logger.info {"Convert docs into dir #{@paths.dst_root_abs}"}

data/lib/giblish/docid.rb

@@ -52,7 +52,7 @@ module Giblish
     # - does not contain any blank line
     def parse_file(path)
       Giblog.logger.debug { "parsing file #{path} for docid..." }
-      process_header_lines(path) do |line|
+      Giblish.process_header_lines_from_file(path) do |line|
         m = /^:docid: +(.*)$/.match(line)
         if m
           # There is a docid defined, cache the path and doc id
@@ -111,30 +111,6 @@ module Giblish
 
     private
 
-    # Helper method that provides the user with a way of processing only the
-    # lines within the asciidoc header block.
-    # The user must return nil to get the next line.
-    #
-    # ex:
-    # process_header_lines(file_path) do |line|
-    #   if line == "Quack!"
-    #      puts "Donald!"
-    #      1
-    #   else
-    #      nil
-    #   end
-    # end
-    def process_header_lines(path)
-      state = "before_header"
-      File.foreach(path) do |line|
-        case state
-        when "before_header" then (state = "in_header" if line =~ /^=+.*$/)
-        when "in_header" then (state = "done" if line =~ /^\s*$/ || yield(line))
-        when "done" then break
-        end
-      end
-    end
-
     # Helper method to shorten calls to docid_cache from instance methods
     def docid_cache
       self.class.docid_cache
@@ -203,10 +179,10 @@ module Giblish
 
   # Helper method to register the docid preprocessor extension with
   # the asciidoctor engine.
-  def register_extensions
+  def register_docid_extension
     Asciidoctor::Extensions.register do
       preprocessor DocidCollector
     end
   end
-  module_function :register_extensions
+  module_function :register_docid_extension
 end

data/lib/giblish/docinfo.rb

@@ -12,29 +12,44 @@ module Giblish
     end
 
     attr_accessor :converted
-    attr_accessor :title
     attr_accessor :doc_id
     attr_accessor :purpose_str
     attr_accessor :status
     attr_accessor :history
     attr_accessor :error_msg
     attr_accessor :stderr
-    # these two members can have encoding issues when
-    # running in a mixed Windows/Linux setting.
-    # that is why the explicit utf-8 read methods are
-    # provided.
-    attr_accessor :rel_path
-    attr_accessor :src_file
 
-    def relPath_utf8
-      return nil if @rel_path.nil?
-      @rel_path.to_s.encode("utf-8")
+    # these members can have encoding issues when
+    # running in a mixed Windows/Linux setting.
+    # that is why we explicitly encodes them when
+    # writing to them
+    def title
+      @title
     end
-
-    def srcFile_utf8
-      return nil if @src_file.nil?
-      @src_file.to_s.encode("utf-8")
+    def title=(rhs)
+      @title = rhs.nil? ? nil : rhs.encode("utf-8")
+    end
+    def rel_path
+      @rel_path
     end
+    # attr_accessor :rel_path
+    def src_file
+      @src_file
+    end
+    def src_file=(rhs)
+      @src_file = rhs.nil? ? nil : rhs.encode("utf-8")
+    end
+    # attr_accessor :src_file
+
+    # def relPath_utf8
+    #   return nil if @rel_path.nil?
+    #   @rel_path.to_s.encode("utf-8")
+    # end
+    #
+    # def srcFile_utf8
+    #   return nil if @src_file.nil?
+    #   @src_file.to_s.encode("utf-8")
+    # end
 
     def initialize(adoc: nil, dst_root_abs: nil, adoc_stderr: "")
       @src_file = nil
@@ -48,14 +63,14 @@ module Giblish
 
       # fill in doc meta data
       d_attr = adoc.attributes
+      self.src_file=(d_attr["docfile"])
+      self.title=(adoc.doctitle)
       @doc_id = d_attr["docid"]
-      @src_file = d_attr["docfile"]
-      @title = adoc.doctitle
       return if dst_root_abs.nil?
 
       # Get the relative path beneath the root dir to the doc
       @rel_path = Pathname.new(
-          "#{d_attr['outdir']}/#{d_attr['docname']}#{d_attr['docfilesuffix']}"
+          "#{d_attr['outdir']}/#{d_attr['docname']}#{d_attr['docfilesuffix']}".encode("utf-8")
       ).relative_path_from(dst_root_abs)
     end