html2index 1.2.1

data/lib/dictionary.rb

@@ -0,0 +1,46 @@
+#encoding: UTF-8 
+=begin
+/***************************************************************************
+ *   ©2015-2017 Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 3 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
+ *   GNU General Public License for more details.                          *
+ *                                                                         *
+ *   You should have received a copy of the GNU General Public License     *
+ *   along with this program; if not, write to the                         *
+ *   Free Software Foundation, Inc.,                                       *
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
+ ***************************************************************************/
+=end
+require 'constants'
+require_relative 'translating'
+
+class Dictionary
+
+	attr_reader :url, :name, :xpath
+	attr_accessor :color
+	def initialize(name, url, xpath, color = nil)
+		@name = name
+		@url = url
+		@xpath = xpath
+		@color = color
+	end
+end
+
+##### TEST
+if __FILE__ == $0
+	d1 = Dictionary.new('my_dict1','www.1.html',  '../nix[@nix=nil]')
+	d2 = Dictionary.new('my_dict2','www.2.html',  '../nil[@nix=nix]')
+	d1.color='#000090'
+	d2.color='#200010'
+
+	puts 'dict 1 is ' << d1.inspect << ", color: " << d1.color
+	puts 'dict 2 is ' << d2.inspect << ", color: " << d2.color
+end

data/lib/file_checking.rb

@@ -0,0 +1,103 @@
+#encoding: UTF-8
+=begin
+/***************************************************************************
+ *   ©2011-2017 Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 3 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
+ *   GNU General Public License for more details.                          *
+ *                                                                         *
+ *   You should have received a copy of the GNU General Public License     *
+ *   along with this program; if not, write to the                         *
+ *   Free Software Foundation, Inc.,                                       *
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
+ ***************************************************************************/
+=end
+require 'filemagic'
+
+=begin
+A module to facilitate frequently occuring checks on
+file-system objects
+=end
+module File_Checking
+
+  @@text_messages  = {
+    :exist? => "does not exist!",
+    :exist => "does not exist!",
+    :readable? => "is not readable!",
+    :readable => "is not readable!",
+    :executable? => "is not executable!",
+    :executable => "is not executable!",
+    :writable? => "is not writable!",
+    :writable => "is not writable!",
+    :directory? => "is not a directory!",
+    :directory => "is not a directory!",
+    :file? => "is not a file!",
+    :file => "is not a file!",
+    :type => "is not of the right file-type!",
+    :mime => "does not have the right mime-type!",
+  }
+
+  # find the file-type for a given file name
+  def self::file_type(file, mime = false)
+    fm = FileMagic.fm
+    file_magic = fm.file(file)
+    file_mime = nil
+    if mime
+    fm.flags = [:mime_type] 
+    file_mime = fm.file(file)	
+    end
+    return file_magic, file_mime
+  end
+
+
+  # Checks if the file with the name from the first
+  # parameter has the properties, listed in the second. 
+  # The messages parameter is an array of one or several
+  # of :exist?, :readable?, :writable?, :directory?  or 
+  # their string-representations, respectively. 
+  # Returns nil in case of success, otherwise an 
+  # informative message, describing the first negative
+  # test-result.
+  def file_check(file, *messages)
+    File_Checking.file_check(file, *messages)
+  end
+
+  # Checks if the file with the name from the first
+  # parameter has the properties, listed in the second. 
+  # The messages parameter is an array of one or all
+  # of :exist?, :readable?, :writable?, :directory? or
+  # their string-representations, respectively. 
+  # Returns nil in case of success, otherwise an 
+  # informative message, describing the first negative
+  # test-result.
+  def self.file_check(file, *messages)
+    msg = nil
+    if(file && messages.respond_to?(:to_ary) && !messages.empty?)
+      messages.each do |k|
+        if(! k.to_s.end_with?('?'))	
+          k = (k.to_s << '?').to_sym
+        end
+        @log.debug ('checking ' << k.to_s) if @log
+        if(msg == nil && File.respond_to?(k) && ! File.send(k, file.to_s))
+          msg = "#{file} #{@@text_messages[k.to_sym]}"
+        end
+      end
+    end
+    msg
+  end
+end
+
+=begin
+# example
+
+include File_Checking
+msg = file_check('some_file.txt', [:exist?, :readable?, 'writable'])
+puts msg if msg
+=end

data/lib/html2index.rb

@@ -0,0 +1,277 @@
+#encoding: UTF-8 
+=begin
+/***************************************************************************
+ *   ©2015-201/ Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 3 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
+ *   GNU General Public License for more details.                          *
+ *                                                                         *
+ *   You should have received a copy of the GNU General Public License     *
+ *   along with this program; if not, write to the                         *
+ *   Free Software Foundation, Inc.,                                       *
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
+ ***************************************************************************/
+=end
+
+require_relative 'argparser'
+require_relative 'logging'
+require_relative 'translating'
+require_relative 'user_input'
+require_relative 'constants'
+require_relative 'version'
+require_relative 'definition'
+require_relative 'dictionary'
+require_relative 'template'
+require 'open-uri'
+require 'nokogiri'
+require 'filemagic'
+require_relative 'file_checking'
+require_relative 'configuration'
+
+class HTML2Index
+  include Logging
+  include Nokogiri
+
+=begin
+        initialize the HTML2Index-object
+=end
+
+  def initialize(*argv)
+    init_logger()
+    # ensure the configuration-file exist.
+    options = ArgParser.parse(argv)
+    $configuration = Configuration.new(options)
+    $configuration.user_conf
+    @log.level = $log_level
+    @file = $configuration.source
+
+    @log.debug("read #{@file}, write #{$configuration.target}")
+    msg = File_Checking::file_check(@file, :exist?, :readable?, :file?)
+    if(msg) 
+      @log.error("Error: unsuitable file " <<@file << ": " << msg)
+      exit false
+    end
+
+    ftype = File_Checking::file_type(@file)
+    @log.debug('ftype for ' << @file << ' is ' << ftype.to_s)
+    if(ftype && !ftype.empty? && !ftype[0].downcase.match("(html|xml) .*document") )
+      @log.error(@file.dup << ' does not look like a valid file to scan: ' << ftype)
+      exit false
+    end
+
+    @problem_log = File.join(temp_dir, PROBLEM_LOG)
+    @log.debug('calling generate() ') 
+    generate()
+  end
+
+  # create an unnumbered list of the consulted dictionaries.
+  def dict_list
+    list = '<ul>'
+    $configuration.dicts.each do |d|
+      list << '<li>' << d.name << ' : ' << '<a href="' << d.url << '">' << d.url << '</a></li>' << "\n"
+    end
+    @log.debug('list is ' << list.to_s)
+    list << "</ul>\n"
+  end
+
+=begin
+        Parses the html-file and generates the glossary.
+=end
+  def generate()
+    @log.info('Generating... PSE wait')
+    begin
+      @doc = HTML::Document.parse(File.open(@file ) )
+      write_html()
+    rescue SystemExit => ir
+      @log.info "Bye"
+      exit true
+    rescue Exception => ex
+      @log.error "line #{__LINE__}: " << ex.message
+      exit false
+    end
+    if(File.exist?(@problem_log ))
+      @log.info "Some expressions caused problems and are listed in " << @problem_log
+    end
+  end
+  private
+=begin
+        Searches the expression in online-dictionaries and
+        creates a Definition-object for each definition, found.
+        Returns an array of all Definition-objects for the 
+        expression.
+=end
+  def dict_definition(expression)
+    definitions = Array.new
+    $configuration.dicts.each do |d|
+      text_array = Array.new
+      page = nil
+      begin
+        url = d.url.dup << URI.encode_www_form_component(expression.gsub(/\s/,'_').gsub("'", ''))
+        page = Nokogiri::HTML(URI.open(url))
+      rescue Exception => ex
+        url_b = d.url.dup << URI.encode_www_form_component(expression.gsub(/\s/,'_'))
+        @log.warn('WARNING, accessing ' << url << ': ' << ex.message)
+        if(url_b != url)
+          @log.warn("\twill try " << url_b)
+          begin
+            page = Nokogiri::HTML(URI.open(url_b, "User-Agent"=>"#{APPNAME} #{VERSION}", "From"=>"Function Test <bat.guano@don.blech.e4ward.com>"))
+          rescue Exception => ex
+            @log.warn("\tWARNING, accessing " << url_b << ': ' << ex.message)
+          end
+        end
+      end
+      if(page)
+        nodes = page.xpath(d.xpath)
+        if(nodes)
+          text_array = nodes.collect {|n| n.inner_text.to_s.strip}
+          if(!text_array.empty?)
+            definition = Definition.new(d.name, expression.gsub('_', ' '), text_array)
+            definition.color = d.color
+            definitions.push(definition)
+          end
+        end
+      else
+        File.open(@problem_log, 'a'){|f| f.write(expression + "\n") }
+      end
+    end
+    return definitions
+  end
+
+=begin
+Creates and returns an Array of Definitions.
+=end
+  def index()
+    begin
+      #TODO: Hash
+      list = Array.new
+      def_list = Array.new
+      tag = $configuration.html_tag
+      attr = $configuration.html_attribute
+      value = $configuration.attr_value
+      xpath = ".//#{tag}[contains(@#{attr}, '#{value}')]"
+      # xpath = './/' << value << '[@' << attr << '="' << value << '"]'
+      @log.debug('xpath is ' << xpath)
+      tags = @doc.xpath(xpath)
+      tags.each do |t|
+        expression = t.attribute('title').to_s
+        unless (expression && !expression.strip.empty? )
+          expression = t.text.to_s
+        end
+        if(expression)
+          expression.gsub!(/\s+/, " ")
+          expression.strip!	
+          if(!expression.empty? && !def_list.include?(expression) )
+            if(!list.include?expression.downcase) 
+              list.push expression.downcase
+              definition = dict_definition(expression)
+              if(definition && !definition.empty?)
+                # @log.debug('definition is ' << definition.to_s)
+                def_list.push(format_index(definition) )
+                def_list.sort!
+              end
+            end
+          end
+        end
+      end
+=begin
+      if(@log.level == Logger::DEBUG)
+        @log.debug('def_list is ') 
+        def_list.each_with_index{|t, i| @log.debug( "%i %s"%[i, t]) }
+      end
+=end
+    rescue Interrupt => ex
+      @log.warn "--------------\n\tIndex generation interrupted by user\n---------------"
+    end
+    return def_list
+  end
+
+=begin
+Formats the definition text.
+=end
+  def format_index(ndef)
+    item = "\n<dt>" 
+    if(ndef.respond_to?(:to_ary))
+      item << ndef[0].expression.dup
+      item << "</dt>"
+      ndef.each_with_index do |dn, i|
+        item << "\n\t<dd>(<i style='color:#" << dn.color << ";'>" << dn.origin << "</i>): " << dn.definition.join("<br/>") << "</dd>" 
+      end
+    else
+      item << ndef.expression.dup << "</dt>"
+      item << "\n\t<dd>(<i><u>" << dn.origin << "</i<): " << dn.definition << "</dd>"
+    end
+    return item
+  end
+  # def temp_dir options = {:remove => true}
+  def temp_dir
+    @temp_dir ||= begin
+                    @log.debug('creating temp_dir')
+                    require 'tmpdir'
+                    require 'fileutils'
+                    path = File.join(Dir::tmpdir, "HTML2Index_#{Time.now.to_i}_#{rand(1000)}")
+                    @log.debug('temp-dir path is ' << path)
+                    Dir.mkdir(path)
+                    # at_exit {FileUtils.rm_rf(path) if File.exists?(path)} if options[:remove]
+                    File.new path
+                  end
+  end
+
+  def write_html()
+    tempfile = nil
+    out = nil
+    out_file = $configuration.target
+    template = Template.new
+    if out_file
+      if(File.exist?(out_file))
+        if(File.writable?(out_file))
+          puts "\nWARNING! File " << out_file << " exists!"
+          print "Do you want to overwrite it? (Y/n) "
+          res = wait_for_user
+          puts
+          unless(['y', 'Y'].include?(res.chr) )
+            puts "Okay, doing nothing."
+            exit false
+          end
+        else
+          puts "ERROR! File " << out_file << " is not writable! Aborting, bye." 
+          exit false
+        end
+      end
+    else
+      @log.debug('out_file is STDOUT')
+    end
+    begin
+      html = template.to_s
+
+      # create the definitions in index()
+      def_list = "<dl>\n" << index().join("\n") << "\n</dl\n>"
+      # associate content with fields
+      data = {:dict_list => dict_list(), :glossary => def_list }
+      placeholders = $configuration.placeholders
+      fdelim = $configuration.fdelim
+      placeholders.each_pair do |k, v|
+        repl = fdelim.dup << v << fdelim.reverse
+        @log.debug('try to replace ' << repl)
+        html.gsub!(repl, data[k])
+      end
+      @log.debug('html is now ' << html)
+      html.gsub!(/\<head\>/, "<head>\n\t#{GeneratorMeta}")
+      File.write(out_file, html) if out_file
+      puts html if !out_file
+    rescue Exception => ex
+      @log.error( "line #{__LINE__} " << ex.message << ' (' << ex.class.name << ')')
+    ensure
+      out.close if out && !out.closed?
+      File.unlink(out) if out
+    end
+  end
+end
+
+

data/lib/log.conf

@@ -0,0 +1,56 @@
+#encoding: UTF-8 
+=begin
+/***************************************************************************
+ *   ©2013-2015 Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 3 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
+ *   GNU General Public License for more details.                          *
+ *                                                                         *
+ *   You should have received a copy of the GNU General Public License     *
+ *   along with this program; if not, write to the                         *
+ *   Free Software Foundation, Inc.,                                       *
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
+ ***************************************************************************/
+
+A simplified logger configuration. Set the level for each individual logger
+below. Choose a different log-device or log-file if you like. Keep the 
+formatting intact.  Do not change other sections of this file.
+=end
+
+# Do not touch from here ----->
+require 'logger'
+
+debug = Logger::DEBUG
+info = Logger::INFO
+error = Logger::ERROR
+fatal = Logger::FATAL
+warn = Logger::WARN
+unknown = Logger::UNKNOWN
+{
+# <---------------- to here !
+
+# Enter your settings here, but take into consideration that not all 
+# the named classes will really produce readable output. Well, you can 
+# always try... Either name just the log-level or make the log-level
+# precede the output-device or output-file like in the examples.
+
+# Example: naming a log-file
+# :HtmlBuilder => [info, 'C:\temp\htmlbuilder.log']
+# :HtmlBuilder => [debug, '/tmp/htmlbuilder.log'],
+
+:HTML2Index => debug,
+:Template => info,
+:Configuration => info,
+:ArgParser => info,
+
+# And ignore the remainder, too.
+}
+
+#eof

data/lib/logging.rb

@@ -0,0 +1,206 @@
+#encoding: UTF-8 
+=begin
+/***************************************************************************
+ *   ©2011-2017 Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 3 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ *   This program is distributed in the hope that it will be useful,       *
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
+ *   GNU General Public License for more details.                          *
+ *                                                                         *
+ *   You should have received a copy of the GNU General Public License     *
+ *   along with this program; if not, write to the                         *
+ *   Free Software Foundation, Inc.,                                       *
+ *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
+ ***************************************************************************/
+=end
+require 'logger'
+require_relative 'file_checking'
+
+=begin Creates a member @log and precede its output with the name of the class
+of the object. 
+Example for a class-level logger:
+# --------------------
+class TClass
+        self.extend(Logging)
+        @@log = init_logger(STDOUT)
+        def test_log
+                @@log.info('class-level logger called from instance: ' << @@log.to_s)
+                @log = @@log
+                @log.info('AGAIN: class-level logger called from instance: ' << @log.to_s)
+        end
+        def self::test_log
+                @log.info('class-level logger called from class: ' << @log.to_s)
+                @@log.info('AGAIN: class-level logger called from class: ' << @@log.to_s)
+        end
+end
+#---------------------
+Example for a object-level logger:
+ATTN! This means 1 logger per object.
+# --------------------
+class TClass
+        include Logging
+        def initialize
+                init_logger(STDOUT, Logger::DEBUG)
+        end
+        def test_log
+                @log.debug('called test_log() ')
+        end
+end
+=end
+module Logging
+  @@LEVELS = {:debug => Logger::DEBUG, :info => Logger::INFO, :error => Logger::ERROR, :warn => Logger::WARN, :unknown => Logger::UNKNOWN, :fatal => Logger::FATAL}
+
+  @@have_log = false
+  @@LOG_CONF = File.dirname(File.absolute_path(__FILE__)) << File::Separator << 'log.conf'
+
+  # Call this method in an instance-method (e.g. initialize() ) to define the
+  # object-level logger; i.e. an object-specific member @log.
+  # Call this method within the class-definition for a class-level logger; i.e.
+  # a member @log for class-level acces.
+  # The method returns the logger, so you can actually do what you want with it.
+  def init_logger(target = STDOUT, level = Logger::INFO)
+    # Prepare for a class-level logger. This is actually quite cool.
+    # ---> Ingeniuous code starts here
+    cn = (self.class == Class ? name : self.class.name)
+    # <--- Ingeniuous code ends here
+
+    # allow to override the set log-levels with an
+    # external configuration (log.conf).
+    log_conf(cn)
+    # Or use the defaults as set here or elsewhere...
+
+    @level ||= level 
+    @target ||= target
+    @log = Logger.new(@target)
+    @log.level = @level
+    @log.formatter = formatter(cn)
+    if ! @@have_log
+      @log.debug cn.dup << ' reading logging-configuration from ' << @@LOG_CONF	
+      @@have_log = true
+      @log.debug('level is ' << level.to_s)
+    end
+    return @log
+  end
+
+  def formatter(classname)
+    proc do |severity, datetime, progname, msg|
+      t = Time.now
+      "#{classname} (#{__LINE__}): #{severity} #{t.hour}-#{t.min}-#{t.sec}: #{msg}\n"
+    end
+  end
+
+  # Set the log-target to an IO object.
+  def log_target=(target)
+    @target = target
+    @log = Logger.new(@target)
+    @log.level = @level
+  end
+
+  # set the log-level
+
+  def log_level=(level)
+    @level = level
+    @log.level = @level
+  end
+
+  # verify log level
+  def log_level?(level)
+    return @log.level == level
+  end
+
+  # brutal log-function: print message, than exit.
+  def debout(str)
+    puts self.class.name.dup << " DEBOUT: " << str 
+    exit true
+  end
+
+  private
+
+  # Override or set the log-level and target-device, as set in a file 'log.conf'.
+  # I do not like the look of this, but it works just the way I want it to.
+  # "HEAVANS! Isn't there a standard way to do this in Ruby, for Christ's sake?", you say.
+  # Heck, I don't care. <= Read that again, I say.
+  def log_conf(cn = nil)
+    config = level = target = nil
+    if(File::exist?(@@LOG_CONF) )
+      begin
+        conf = File.read(@@LOG_CONF)
+        config = instance_eval(conf)
+      rescue Exception => ex
+        STDERR.puts "WARNING! Cannot evaluate the logger-configuration!" << ' ' << ex.message
+        STDERR.puts "Default log-levels apply."
+      end
+    else
+      puts "Default log-levels apply."
+    end
+
+    if(config && config.respond_to?(:to_hash) )
+      config.default = nil
+      if cn
+        config = config[cn.to_sym]
+      else
+        config = config[self.class.name.to_sym]
+      end
+
+      if(config )
+        if(config.respond_to?(:to_ary) && config.size == 2)
+          @level, @target = config
+          @target.downcase!
+          logdir = File.dirname(@target)
+          msg = File_Checking::file_check(logdir, :exist?, :directory?, :writable?)
+          if(msg) 
+            STDERR.puts "WARNING! A logfile for '%s' cannot be written to %s (%s)!" %[self.class.name, logdir, msg] 
+            @target = nil
+          end
+        else
+          @level = config
+        end	
+      end
+    end
+  end
+  attr_reader :level
+end
+
+######### test
+if __FILE__ == $0
+  class TClass
+    # class level ---->
+    self.extend(Logging)
+    @@log = init_logger(STDOUT, Logger::INFO)
+    # <------
+    # object-level ---->
+    include Logging
+    # <---------
+
+    def test_log
+      @@log.info('class-level logger called from instance: ' << @@log.to_s)
+      #@log = @@log # works too
+      @log = TClass.class_eval{@log}
+      @log.info('AGAIN: class-level logger called from instance: ' << @log.to_s)
+      @log.debug("you won't see this on log-level INFO")
+
+      # object-level ---->
+      init_logger
+      # <-----------
+      @log.info("That's a different thing: " << @log.to_s << " - object-level logger!")
+
+    end
+    def self::test_log
+      @log.info('class-level logger called from class: ' << @log.to_s)
+      @@log.info('AGAIN: class-level logger called from class: ' << @log.to_s)
+    end
+  end
+
+  TClass.new.test_log # class-logger + 1st object-logger
+  TClass.new.test_log # same class-logger + 2nd object-logger
+
+  TClass::test_log # same class-logger
+  puts 'And just say it once clearly: THIS IS COOOL!!'
+end
+#EOF