beta_tools 0.0.4

data/lib/it_tools/options.rb

@@ -0,0 +1,46 @@
+require 'optparse'
+require 'logger'
+
+class Options
+  attr_accessor :options, :log
+  def initialize(ops = {})
+    @options = {}
+    options.merge! ops
+    @log = Logger.new('log.txt')
+    if level = @options[:debug_level]
+      @log.level = level
+    else
+      @log.level = Logger::DEBUG
+    end
+    optparse = OptionParser.new do |opts|
+      opts.banner = "Usage: pub -e <environment_to_publish_to>"
+      @options[:debug] = false
+      opts.on( '-d', '--debug', 'Turn on program debugging' ) do |level|
+        options[:debug] = true
+        options[:debug_level] = level
+      end
+     @options[:verbose] = false
+     opts.on( '-v', '--verbose', 'Output more information' ) do
+       @options[:verbose] = true
+     end
+      @options[:environment] = 'loc'
+      opts.on( '-e', '--environment ENV', 'Publish to environment ENV. REQUIRED.' ) do |env|
+        options[:environment] = env
+      end
+      @options[:indexer_url] = nil
+      opts.on( '-i', '--indexer_ulr URL', 'Submit docs to indexer with url: URL' ) do |solr_base_url|
+        options[:indexer_url] = solr_base_url
+      end
+      @options[:source_folder] = '.'
+      opts.on( '-s', '--source_folder FOLDER', 'Use FOLDER as source of artifact to deploy.') do |source_folder|
+        options[:source_folder] = source_folder
+      end
+      opts.on( '-h', '--help', 'Display this screen' ) do
+        puts opts
+        exit
+      end
+    end
+    optparse.parse!
+    @log.debug "Deploying to #{options[:environment]} environment." if @options[:debug]
+  end
+end

data/lib/it_tools/pub_driver.rb

@@ -0,0 +1,188 @@
+#!/usr/bin/env ruby
+
+require_relative 'html_publish'
+require_relative 'network_tools'
+require_relative 'publisher2'
+require 'logger'
+require 'optparse'
+
+module PubDriver
+  class Pub
+    attr_accessor :ops
+    def initialize( ops = {} )
+      @ops = {}
+      @ops.merge! ops
+      @log = Logger.new('log.txt')
+      if level = @ops[:debug_level]
+        @log.level = level
+        @log.debug "Set logger to debug."
+      else
+        @log.level = Logger::INFO
+        @log.info "Set logger to info."
+      end
+    end
+    def validate( options, required, values ) 
+      missing_msg = "Missing required parameter: '"
+      required.each do |entry|
+        value = values[entry]
+        raise missing_msg + value[:short] + "', or '" + value[:long] + "', which specifies the " + value[:mesg] unless options[entry]
+      end
+    end
+    def publish_locally2
+      options = get_local_pub_options
+      solr_search_files = [ "search.html",
+                            "search.js",
+                            "ajax-loader.gif",
+                            "help.png" ]
+
+      parameters = { 
+        :solr_base_url => "http://searcher:8983/solr/",
+        :style_sheet => "inputStyles.css",
+        :solr_search_files => solr_search_files,
+        :src_dir => options[:src_dir],
+        :target_dir => options[:solr_dir],
+        :static_dir => options[:static_dir],
+        :debug_level => options[:debug_level],
+        :solr_search_files_dir => options[:solr_search_files_dir],
+        :search_category => options[:search_category] }
+      
+      @log.debug "Params: " + parameters.to_s
+      publisher = Publisher::Markdown.new parameters
+      publisher.process_files
+    end
+    def publish_locally
+      solr_indexer = "http://127.0.0.1:8983/solr/"
+      converter = Website::MarkdownConverter.new :indexer_url => solr_indexer
+      converter.go
+    end
+    def validate_local_publish options
+      dir_counts, ext_counts = count_files_in_dir options, [:src_dir, :solr_dir, :static_dir], ['.mmd']
+      dir_count_printer = lambda { |key,count,hash| p hash[key] + ": [#{count}]" }
+      print_counts dir_counts, dir_count_printer
+      print_counts ext_counts, options
+    end
+    def print_counts hash, printer
+      hash.each do |key,value|
+        printer.call key, value, hash
+      end
+    end
+    def count_files_in_dir options, dirs, extension = []
+      file_count = {}
+      ext_count = {}
+      dirs.each do |dir|
+        file_count[dir] = Dir.entries(options[dir]).size - 2
+        next unless extension.size > 0
+        files_with_extension_count = {}
+        extension.each do |curr_ext|
+          ext_count[curr_ext] = 0
+        end
+        Dir.entries(options[dir]).each do |file|
+          full_path = File.join options[dir], file
+          next unless File.exists? full_path
+          next if File.directory? full_path
+          extension.each do |curr_ext|
+            if (File.extname file) == curr_ext
+              ext_count[curr_ext] += 1
+            end
+          end
+        end
+        return [file_count,ext_count]
+      end
+    end
+    def rsync_it ops
+      command = "rsync -avP --stats #{ops[:from_dir]} "
+      command += "#{ops[:as_user]}@" if ops[:as_user]
+      command += "#{ops[:to_host]}:#{ops[:to_dir]} "
+      p "[command]: " + command
+      system command
+    end
+
+    def publish_remote params
+      vpnTools = NetworkTools::VpnTools.new
+      rsync_ops = {}
+      rsync_ops[:from_dir] = params[:doc]
+      rsync_ops[:as_user] = params[:as_user]
+      if vpnTools.on_vpn
+        @log.debug "On VPN, so sync'ing regular docs AND work docs"
+        rsync_ops[:to_host] = params[:int_host]
+
+        rsync_ops[:to_dir] = params[:int_docs]
+        rsync_it rsync_ops
+
+        rsync_ops[:from_dir] = params[:wdoc]
+        rsync_ops[:to_dir] = params[:int_wdocs]
+        rsync_it rsync_ops
+      else
+        @log.debug "On VPN, so only sync'ing regular docs."
+        rsync_ops[:to_host] = params[:ext_host]
+        rsync_ops[:to_dir] = params[:ext_docs]
+        rsync_it rsync_ops
+      end
+    end
+    def get_local_pub_options
+      options = {}
+      src = { :short => '-s', :long => '--src_dir DIR', :mesg => 'Directory of source markdown files.'}
+      solr = { :short => '-w', :long => '--solr_write_dir DIR', :mesg => 'Directory to write solr files.'}
+      stat = { :short => '-t', :long => '--static_html_dir DIR', :mesg => 'Directory to write static html files.'}
+      solr_search = { :short => '-l', :long => '--solr_search_files_dir DIR', :mesg => 'Directory to read solr search files from.'}
+
+      values = { :src_dir => src, :solr_dir => solr, :static_dir => stat, :ssf => solr_search }
+      optparse = OptionParser.new do |opts|
+        opts.on( '-h', '--help', 'Display this screen' ) do
+          puts opts
+          exit
+        end
+        curr_opt = values[:src_dir]
+        opts.on( curr_opt[:short], curr_opt[:long], curr_opt[:mesg] ) do |dir|
+          @log.debug "Setting source directory for markdown files."
+          raise "Directory not specified! " + curr_opt[:mesg] if dir.nil?
+          raise "Can't find markdown source dir as specified #{dir}" unless File.directory? dir
+          options[:src_dir] = dir
+        end
+        curr_opt = values[:solr_dir]
+        opts.on( curr_opt[:short], curr_opt[:long], curr_opt[:mesg] ) do |dir|
+          raise "Directory not specified! " + curr_opt[:mesg] if dir.nil?
+          FileUtils.mkdir dir unless File.exists? dir
+          options[:solr_dir] = dir
+        end
+        curr_opt = values[:static_dir]
+        opts.on( curr_opt[:short], curr_opt[:long], curr_opt[:mesg] ) do |dir|
+          raise "Directory not specified! " + curr_opt[:mesg] if dir.nil?
+          FileUtils.mkdir dir unless File.exists? dir
+          options[:static_dir] = dir
+        end
+        curr_opt = values[:ssf]
+        opts.on( curr_opt[:short], curr_opt[:long], curr_opt[:mesg] ) do |dir|
+          raise "Directory not specified! " + curr_opt[:mesg] if dir.nil?
+          FileUtils.mkdir dir unless File.exists? dir
+          options[:solr_search_files_dir] = dir
+        end
+
+        mesg = "Specify the search category: 'public', 'work', etc..."
+        opts.on( '-c', '--index_category CATEGORY', mesg ) do |category|
+          raise "Missing Field: " + mesg if category.nil?
+          options[:search_category] = category
+        end
+
+        mesg = "Set to 'info' or 'debug'"
+        opts.on( '-d', '--debug_level LEVEL', mesg ) do |level|
+          raise "Debug level not specified! #{mesg}" if level.nil?
+          case level
+            when "info"
+            options[:debug_level] = Logger::INFO
+            @log.level = Logger::INFO
+            when "debug"
+            options[:debug_level] = Logger::DEBUG
+            @log.level = Logger::DEBUG
+            else
+            raise "Debug level not specified properly! #{mesg}"
+          end
+        end
+      end
+      optparse.parse!
+      validate(options, [:src_dir, :solr_dir, :static_dir], values)
+      @ops.merge! options
+      return options
+    end
+  end
+end

data/lib/it_tools/publisher2.rb

@@ -0,0 +1,232 @@
+require "pygments"
+require "redcarpet"
+require 'logger'
+require_relative 'solr'
+
+module Publisher
+  class Markdown
+    attr_accessor :is_markdown_file, :convert_to_html_filename, :ops, :log
+
+    def initialize(ops = {})
+      @ops = { :output_style_sheet => 'style.css' }
+      @ops.merge! ops
+      @is_markdown_file = lambda { |file| File.extname(file) == '.mmd' }
+      @convert_to_html_filename = lambda { |filename| File.basename(filename, File.extname(filename)) + ".html" }
+      @log = Logger.new('log.txt')
+      if level = @ops[:debug_level]
+        @log.level = level
+      else
+        @log.level = Logger::INFO
+      end
+    end
+
+    def copy_src_if_newer(read_dir, write_dir)
+      @log.info "Copying all files from: #{read_dir}, to: #{write_dir}, if they are newer in from dir."
+      return unless Dir.exists? read_dir
+      FileUtils.mkdir write_dir unless Dir.exists? write_dir
+      Dir.foreach read_dir do |curr_file|
+        read = File.absolute_path(File.join(read_dir,curr_file))
+        next if File.directory? read
+        write = File.absolute_path(File.join(write_dir,curr_file))
+        FileUtils.cp_r read, write_dir if file_newer? read, write
+        @log.debug "Copied from: #{read}, to: #{write}"
+      end
+    end
+    def insert_css_link(file_list)
+      style_info = '<head><LINK REL=StyleSheet HREF="' + @ops[:output_style_sheet] + '" TYPE="text/css"/></head>'
+      prepend_string style_info, file_list
+    end
+
+    def prepend_string(string, file_list)
+      file_list.each do |file|
+        contents = File.open(file,'r'){|f| f.read }
+        tmp_file = file + '.tmp'
+        File.open(tmp_file, 'w'){|f| f.write string + contents}
+        FileUtils.rm_f file
+        FileUtils.mv tmp_file, file
+      end
+    end
+    def copy_files_to_dir(to_dir, file_list)
+      @log.debug "Copy to: #{to_dir}.  Files: #{file_list}"
+      FileUtils.mkdir to_dir unless File.exists? to_dir
+      raise "#{to_dir} is not a directory" unless File.directory? to_dir
+      static_file_list = []
+      file_list.each do |file| 
+        FileUtils.cp file, to_dir
+        static_file_list.push File.absolute_path(File.join(to_dir, File.basename(file)))
+      end
+      return static_file_list
+    end
+    def copy_solr_search_files ops, to_dir
+      search_files = ops[:solr_search_files]
+      solr_search_files_dir = ops[:solr_search_files_dir]
+      @log.debug "Copying solr search files from: #{solr_search_files_dir}, to: #{to_dir}"
+      search_files.each do |file| 
+        from = File.join(solr_search_files_dir, file)
+        FileUtils.cp from, to_dir
+      end
+    end
+    def process_files(from_dir = @ops[:src_dir], to_dir = @ops[:target_dir])
+      # Files for searching
+      newer_files = get_newer_src_files from_dir, to_dir, @is_markdown_file, @convert_to_html_filename
+      converted_files = convert_mmd_files newer_files, to_dir
+      send_to_indexer converted_files, @ops[:search_category]
+      copy_solr_search_files @ops, to_dir
+      copy_src_if_newer File.join(from_dir, 'images'), File.join(to_dir, 'images')
+      write_stylesheet from_dir, to_dir
+      # Now static only files
+      static_dir = to_dir + "_static"
+      static_file_list = copy_files_to_dir static_dir, converted_files
+      copy_src_if_newer File.join(from_dir, 'images'), File.join(static_dir, 'images')
+      insert_css_link static_file_list
+      write_stylesheet from_dir, static_dir
+      create_index static_dir
+    end
+    def send_to_indexer(file_list, search_category)
+      uploader = Solr::Upload.new(ops)
+      file_list.each do |file|
+        contents = File.open(file,'r'){ |f| f.read }
+        extension = File.extname file
+        file_name = File.basename file, extension
+        @log.debug "Indexing the following: #{file_name}"
+        uploader.upload_file file_name, contents, file_name, search_category
+      end
+    end
+    def create_index(dir)
+      files = File.join dir,'*.html'
+      index = Dir.glob [files]
+      index.sort! { |a,b| a.downcase <=> b.downcase }
+      file = File.open(File.join(dir,"index.html"), 'w')
+      file.write("<html><body><ul>")
+      index.each { |filename|
+        basename = File.basename(filename, File.extname(filename))
+        file.write("<li/><a href=\"#{basename}.html\">#{basename}")
+      }
+      file.write("</ul></body></html>")
+      file.close()
+    end
+
+    def write_stylesheet(read_dir,write_dir)
+      contents = File.open(File.join(read_dir, @ops[:style_sheet]), "rb"){|f| f.read}
+      css = Pygments.css
+      File.open(File.join(write_dir,@ops[:output_style_sheet]), 'w') {|f| f.write(css + "\n" + contents) }
+    end
+    def convert_mmd_files(files_to_convert, target_dir)
+      markdown = Redcarpet::Markdown.new(HTMLwithPygments,
+                                         :fenced_code_blocks => true,
+                                         :tables => true,
+                                         :autolink => true,
+                                         :with_toc_data => true)
+      converted_files = []
+      files_to_convert.each do |file|
+        log.info "Working on file: " + file
+        contents = File.open(file, "r"){|f| f.read}
+        contents = contents.remove_non_ascii
+
+        html = markdown.render(contents.force_encoding('US-ASCII'))
+        log.debug "Converted markdown"
+        target_file = get_target_file file, target_dir, @convert_to_html_filename
+        write_file html, target_file
+        converted_files.push target_file
+      end
+      return converted_files
+    end
+    def write_file( contents, out_file )
+      File.open(out_file, 'w') { |f| f.write contents }
+    end
+    def get_target_file(src_filename, target_dir, make_target_filename = nil)
+      if make_target_filename
+        target_file_name = make_target_filename.call src_filename 
+      else
+        target_file_name = src_filename
+      end
+      target_file = File.absolute_path target_file_name, target_dir
+    end
+    def get_newer_src_files(src_dir, target_dir, is_wanted_src_file = nil, make_target_filename = nil)
+      newer_src_files = []
+      Dir.foreach(src_dir) do |curr_file|
+        next if File.directory? curr_file
+        if is_wanted_src_file 
+          next unless is_wanted_src_file.call curr_file
+        end
+        target_file = get_target_file(curr_file, target_dir, make_target_filename)
+        src_file = File.absolute_path curr_file, src_dir
+        newer_src_files.push src_file if file_newer? src_file, target_file
+      end
+      return newer_src_files
+    end
+    def file_newer?(first_file, second_file)
+      first_file_ts = File.stat(first_file).mtime
+      return true unless File.exist?(second_file) 
+      second_file_ts = File.stat(second_file).mtime
+      if second_file_ts > first_file_ts
+        return false
+      else
+        return true
+      end
+    end
+  end
+
+#---------------------------------------
+
+  class HTMLwithPygments < Redcarpet::Render::HTML
+    attr_accessor :ops, :log
+
+    def initialize(options = {})
+      super
+      @ops = {:log_filename => 'log.txt'}
+      @ops.merge! options
+      @log = Logger.new(@ops[:log_filename])
+      if level = @ops[:debug_level]
+        @log.level = level
+      else
+        @log.level = Logger::INFO
+      end
+      $toc1 = ""
+      $last_header_level = 0
+      $last_header_count = 0
+      $header = Array.new()
+      0.upto(6) { |i| $header[i] = 0 }
+    end
+
+    def block_code(code, language)
+      Pygments.highlight(code, :lexer => language)
+    end
+    def get_header_label( header_level )
+      index = header_level - 1
+      $header[index] = $header[index] + 1
+      header_level.upto($header.length - 1) { |i| $header[i] = 0 }
+      header_string = ""
+      0.upto(header_level - 1) { 
+        |i| if i == header_level - 1
+              header_string += $header[i].to_s
+              @log.debug "Header String: #{header_string}.  Header Entry: #{$header[i]}"
+            else
+              header_string += $header[i].to_s + "."
+            end
+      }
+      @log.info "processed: " + header_string
+      return header_string
+    end
+    def set_less_significant_to_zero(header_level)
+      10.times {|i| p i+1}
+    end
+    def header(text, header_level)
+      header_label = get_header_label(header_level)
+      id = text.gsub(" ", "_") unless text.nil?
+      $toc1 += "<div class=\"toc#{header_level}\"><a href=\"##{id}\">#{header_label} - #{text}</a></div><br/>\n"
+      return "\n<h#{header_level} id=\"#{id}\">#{header_label} - #{text}</h#{header_level}>\n"
+    end
+    def postprocess(full_document)
+      temp = $toc1
+      $toc1 = ""
+      0.upto(6) { |i| $header[i] = 0 }
+      return temp + full_document
+    end
+  end
+end
+class String
+  def remove_non_ascii(replacement="") 
+    self.force_encoding('ASCII-8BIT').gsub(/[\x80-\xff]/,replacement)
+  end
+end

data/lib/it_tools/sample.rb

@@ -0,0 +1,9 @@
+add_me = lambda { |x,y| puts x + y }
+func2 add_me
+multiply_me = lambda { |x,y| puts x * y }
+func2 multiply_me
+
+def func2(func)
+  func.call 2,3
+end
+

data/lib/it_tools/shared.rb

@@ -0,0 +1,22 @@
+require 'xmlsimple'
+module SharedTool
+  class RegularExpression
+    def has_string(data, regex)
+      if first_occurence data, regex != ""
+        return true
+      end
+      return false
+    end
+    def first_occurrence(data, regex)
+      if data =~ regex
+        return $1
+      end 
+      return ""
+    end
+  end
+end
+class String
+  def remove_non_ascii(replacement="") 
+    self.force_encoding('ASCII-8BIT').gsub(/[\x80-\xff]/,replacement)
+  end
+end

data/lib/it_tools/solr.rb

@@ -0,0 +1,157 @@
+require 'net/http'
+require 'uri'
+require 'sinatra/base'
+require 'sanitize'
+require 'logger'
+require 'erb'
+require_relative 'multipart'
+require_relative 'shared'
+
+class WebApp < Sinatra::Base
+  configure do
+    set :public_folder, '/data/documentation/'
+    set :environment, :production
+    set :port, 80
+    set :solr_host, 'localhost'
+    set :solr_port, '8983'
+    enable :logging, :dump_errors, :raise_errors
+  end 
+  get '/' do
+    "it works!"
+  end
+  get '/srch' do
+    handler = Handler.new
+    params[:solr_host] = settings.solr_host
+    params[:solr_port] = settings.solr_port
+    return "<p>No 'query' specified.</p>" unless params["query"]
+    begin
+      @highlights = handler.get_srch params
+      result = erb :search_results
+      return result
+    rescue NoResults => reason
+      return reason.mesg
+    end
+  end
+  def check_highlights
+    retval = "<ul>"
+    @highlights.each do |key, value|
+      if value['body'].nil? 
+        next  
+      end
+      body = value['body'][0]
+      body = body.remove_non_ascii
+      clean = Sanitize.clean(body, Sanitize::Config::RELAXED)
+      retval += "<li>#{key}<br><div>#{clean}</div>"
+    end
+    retval += "</ul>"
+  end
+end
+class NoResults < RuntimeError
+  attr :mesg
+  def initialize(mesg)
+    @mesg = mesg
+  end
+end 
+
+
+class Handler
+  def get_srch params
+    query = params["query"]
+    if query.nil?
+      raise NoResults.new "<p>No Query Specified</p>"
+    end
+    query = Solr::Query.new params
+    resp = query.do_query params
+    @highlights = resp['highlighting']
+    if @highlights.length == 0 
+      raise NoResults.new "<p>No Results</p>"
+    end
+    return @highlights
+  end
+end
+module Solr
+  class Query
+    attr_accessor :ops, :log
+    def initialize(options = {})
+      @ops = {}
+      @ops.merge! options unless options.nil?
+      @log = Logger.new('log.txt')
+      if level = @ops[:debug_level]
+        @log.level = level
+      else
+        @log.level = Logger::DEBUG
+      end
+      begin
+        @ops[:solr_host] = settings.solr_host
+      rescue
+      end
+      mesg = "Must specify ':solr_host' and ':solr_port' in constructor hash"
+      raise mesg unless @ops[:solr_host] and @ops[:solr_port]
+    end
+    def do_query(params)
+      h = Net::HTTP.new(@ops[:solr_host], @ops[:solr_port])
+      query = params["query"]
+      category = params["category"]
+      query = "category:" + category + " AND " + params["query"] unless category.nil?
+      all = { 
+        "q" => URI.escape(query),
+        "wt" => "ruby",
+        "hl" => "true",
+        "hl.fl" => "*"
+      }
+      query = "/solr/select?"
+      all.each do |key, value|
+        query += key + "=" + value + "&"
+      end
+      hresp, data = h.get(query)
+      
+      if data.nil? 
+        return "<p>nothing</p>"
+      else
+        rsp = eval(data)
+        return rsp
+      end
+    end
+  end
+  class Upload
+    attr_accessor :ops, :debug
+    def initialize(options = {})
+      @ops = { :debug => true }
+      @ops.merge! options
+      mesg = 'Missing param.  Example :solr_base_url => "http://192.168.0.22:8983/solr/"' 
+      raise mesg unless options[:solr_base_url]
+      @log = Logger.new('log.txt')
+      if level = @ops[:debug_level]
+        @log.level = level
+      else
+        @log.level = Logger::DEBUG
+      end
+    end
+    def upload_file(filename, data, file_id, search_category)
+      params = { 
+        "literal.id" => file_id,
+        "commit" => "true",
+        "uprefix" => "attr_",
+        "fmap.content" => "body"
+      }
+      if @ops[:search_category]
+        params["literal.category"] = @ops[:search_category]
+      end
+      url = @ops[:solr_base_url] + "update/extract"
+      @log.debug "[URL]: " + url
+      post = Multipart::Post.new
+      post.add_params(params)
+      post.add_file(filename, data)
+      resp = post.post(url)
+      case resp
+      when Net::HTTPOK
+        @log.debug "Successfully submitted file to indexer."
+      else
+        @log.error "Failed to submit file to indexer."
+      end
+    end
+    def foo
+      puts "bar"
+    end
+  end
+end

data/lib/it_tools/views/search_results.erb

@@ -0,0 +1,16 @@
+<ul>
+<%
+  @highlights.each do |key, value|
+    if value['body'].nil? 
+      next
+    end
+    body = value['body'][0]
+    body = body.remove_non_ascii
+    clean = Sanitize.clean(body, Sanitize::Config::RELAXED)
+%>
+  <li>
+  <span class="click_me_span" onclick="load_content('<%= key %>' )">
+  <%=key%></span><br/>
+  <div><%= clean %></div>
+  <%end%>
+</ul>