html2index 1.2.1 → 1.32

data/lib/definition.rb

@@ -1,24 +1,19 @@
 #encoding: UTF-8 
 =begin
 /***************************************************************************
- *   ©2015-2017 Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   ©2015-2026, Michael Uplawski <michael.uplawski@uplawski.eu>           *
  *                                                                         *
  *   This program is free software; you can redistribute it and/or modify  *
- *   it under the terms of the GNU General Public License as published by  *
- *   the Free Software Foundation; either version 3 of the License, or     *
- *   (at your option) any later version.                                   *
+ *   it under the terms of the WTFPL, version 2 or later, as published on  *
+ *   https://www.wtfpl.net/about/.                                         *
  *                                                                         *
  *   This program is distributed in the hope that it will be useful,       *
  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
- *   GNU General Public License for more details.                          *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
  *                                                                         *
- *   You should have received a copy of the GNU General Public License     *
- *   along with this program; if not, write to the                         *
- *   Free Software Foundation, Inc.,                                       *
- *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
  ***************************************************************************/
 =end
+
 require 'constants'
 require_relative 'translating'
 

data/lib/dictionary.rb

@@ -1,24 +1,19 @@
 #encoding: UTF-8 
 =begin
 /***************************************************************************
- *   ©2015-2017 Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   ©2015-2026, Michael Uplawski <michael.uplawski@uplawski.eu>           *
  *                                                                         *
  *   This program is free software; you can redistribute it and/or modify  *
- *   it under the terms of the GNU General Public License as published by  *
- *   the Free Software Foundation; either version 3 of the License, or     *
- *   (at your option) any later version.                                   *
+ *   it under the terms of the WTFPL, version 2 or later, as published on  *
+ *   https://www.wtfpl.net/about/.                                         *
  *                                                                         *
  *   This program is distributed in the hope that it will be useful,       *
  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
- *   GNU General Public License for more details.                          *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
  *                                                                         *
- *   You should have received a copy of the GNU General Public License     *
- *   along with this program; if not, write to the                         *
- *   Free Software Foundation, Inc.,                                       *
- *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
  ***************************************************************************/
 =end
+
 require 'constants'
 require_relative 'translating'
 
@@ -32,6 +27,13 @@ class Dictionary
 		@xpath = xpath
 		@color = color
 	end
+
+    def valid?
+      return (@name && ! @name.empty?
+              && @url && ! @url.empty?
+              && @xpath && ! @xpath.empty?
+              && @color && ! @color.empty?)
+    end
 end
 
 ##### TEST

data/lib/html2index.rb

@@ -1,27 +1,20 @@
 #encoding: UTF-8 
 =begin
 /***************************************************************************
- *   ©2015-201/ Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   ©2015-2026, Michael Uplawski <michael.uplawski@uplawski.eu>           *
  *                                                                         *
  *   This program is free software; you can redistribute it and/or modify  *
- *   it under the terms of the GNU General Public License as published by  *
- *   the Free Software Foundation; either version 3 of the License, or     *
- *   (at your option) any later version.                                   *
+ *   it under the terms of the WTFPL, version 2 or later, as published on  *
+ *   https://www.wtfpl.net/about/.                                         *
  *                                                                         *
  *   This program is distributed in the hope that it will be useful,       *
  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
- *   GNU General Public License for more details.                          *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
  *                                                                         *
- *   You should have received a copy of the GNU General Public License     *
- *   along with this program; if not, write to the                         *
- *   Free Software Foundation, Inc.,                                       *
- *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
  ***************************************************************************/
 =end
-
 require_relative 'argparser'
-require_relative 'logging'
+require_relative 'basic_logging'
 require_relative 'translating'
 require_relative 'user_input'
 require_relative 'constants'
@@ -31,43 +24,46 @@ require_relative 'dictionary'
 require_relative 'template'
 require 'open-uri'
 require 'nokogiri'
-require 'filemagic'
-require_relative 'file_checking'
+require 'marcel'
 require_relative 'configuration'
 
 class HTML2Index
-  include Logging
+  include BasicLogging
   include Nokogiri
 
+
+  @@log_level = :debug
 =begin
         initialize the HTML2Index-object
 =end
 
   def initialize(*argv)
-    init_logger()
     # ensure the configuration-file exist.
     options = ArgParser.parse(argv)
     $configuration = Configuration.new(options)
     $configuration.user_conf
-    @log.level = $log_level
+    set_level $log_level
     @file = $configuration.source
 
-    @log.debug("read #{@file}, write #{$configuration.target}")
-    msg = File_Checking::file_check(@file, :exist?, :readable?, :file?)
-    if(msg) 
-      @log.error("Error: unsuitable file " <<@file << ": " << msg)
-      exit false
+    debug("read #{@file}, write #{$configuration.target}")
+    msg = ''
+    [:exist?, :readable?, :file?].each do |cond|
+      if !File.send(cond, @file)
+        error("Error: unsuitable file " <<@file << ": NOT " << cond.to_s.split('?')[0] << '!')
+        exit false
+      end
     end
 
-    ftype = File_Checking::file_type(@file)
-    @log.debug('ftype for ' << @file << ' is ' << ftype.to_s)
-    if(ftype && !ftype.empty? && !ftype[0].downcase.match("(html|xml) .*document") )
-      @log.error(@file.dup << ' does not look like a valid file to scan: ' << ftype)
+    ftype = Marcel::Magic.by_path(@file).subtype
+
+    debug('ftype for ' << @file << ' is ' << ftype.to_s)
+    if(ftype && !ftype.empty? && !ftype.downcase.match(/(html|xml)/) )
+      error(@file.dup << ' does not look like a valid file to scan: ' << ftype)
       exit false
     end
 
     @problem_log = File.join(temp_dir, PROBLEM_LOG)
-    @log.debug('calling generate() ') 
+    debug('calling generate() ') 
     generate()
   end
 
@@ -77,7 +73,7 @@ class HTML2Index
     $configuration.dicts.each do |d|
       list << '<li>' << d.name << ' : ' << '<a href="' << d.url << '">' << d.url << '</a></li>' << "\n"
     end
-    @log.debug('list is ' << list.to_s)
+    debug('list is ' << list.to_s)
     list << "</ul>\n"
   end
 
@@ -85,19 +81,19 @@ class HTML2Index
         Parses the html-file and generates the glossary.
 =end
   def generate()
-    @log.info('Generating... PSE wait')
+    info('Generating... PSE wait')
     begin
       @doc = HTML::Document.parse(File.open(@file ) )
       write_html()
     rescue SystemExit => ir
-      @log.info "Bye"
+      info "Bye"
       exit true
     rescue Exception => ex
-      @log.error "line #{__LINE__}: " << ex.message
+      error "line #{__LINE__}: " << ex.message
       exit false
     end
     if(File.exist?(@problem_log ))
-      @log.info "Some expressions caused problems and are listed in " << @problem_log
+      info "Some expressions caused problems and are listed in " << @problem_log
     end
   end
   private
@@ -109,41 +105,62 @@ class HTML2Index
 =end
   def dict_definition(expression)
     definitions = Array.new
-    $configuration.dicts.each do |d|
-      text_array = Array.new
-      page = nil
-      begin
-        url = d.url.dup << URI.encode_www_form_component(expression.gsub(/\s/,'_').gsub("'", ''))
-        page = Nokogiri::HTML(URI.open(url))
-      rescue Exception => ex
-        url_b = d.url.dup << URI.encode_www_form_component(expression.gsub(/\s/,'_'))
-        @log.warn('WARNING, accessing ' << url << ': ' << ex.message)
-        if(url_b != url)
-          @log.warn("\twill try " << url_b)
+    if $configuration.dicts && ! $configuration.dicts.empty?  
+      $configuration.dicts.each do |d|
+        if d && d.valid?
+          debug 'dict: ' << (d ? d.inspect : ' N I L ')
+          text_array = Array.new
+          page = nil
           begin
-            page = Nokogiri::HTML(URI.open(url_b, "User-Agent"=>"#{APPNAME} #{VERSION}", "From"=>"Function Test <bat.guano@don.blech.e4ward.com>"))
+            url = d.url.dup << URI.encode_www_form_component(expression.gsub(/\s/,'_').gsub("'", ''))
+            page = Nokogiri::HTML(URI.open(url))
           rescue Exception => ex
-            @log.warn("\tWARNING, accessing " << url_b << ': ' << ex.message)
+            url_b = d.url.dup << URI.encode_www_form_component(expression.gsub(/\s/,'_'))
+            warn('WARNING, accessing ' << url << ': ' << ex.message)
+            if(url_b != url)
+              warn("\twill try " << url_b)
+              begin
+                page = Nokogiri::HTML(URI.open(url_b, "User-Agent"=>"#{APPNAME} #{VERSION}", "From"=>"Function Test <bat.guano@don.blech.e4ward.com>"))
+              rescue Exception => ex
+                warn("\tWARNING, accessing " << url_b << ': ' << ex.message)
+              end
+            end
+          else
+            warn  'invalid dictionary definition'
+
           end
         end
-      end
-      if(page)
-        nodes = page.xpath(d.xpath)
-        if(nodes)
-          text_array = nodes.collect {|n| n.inner_text.to_s.strip}
-          if(!text_array.empty?)
-            definition = Definition.new(d.name, expression.gsub('_', ' '), text_array)
-            definition.color = d.color
-            definitions.push(definition)
+        if(page)
+          nodes = page.xpath(d.xpath)
+          if(nodes)
+            text_array = nodes.collect {|n| n.inner_text.to_s.strip}
+            if(!text_array.empty?)
+              definition = Definition.new(d.name, expression.gsub('_', ' '), text_array)
+              definition.color = d.color
+              definitions.push(definition)
+            end
           end
+        else
+          File.open(@problem_log, 'a'){|f| f.write(expression + "\n") }
         end
-      else
-        File.open(@problem_log, 'a'){|f| f.write(expression + "\n") }
       end
+    else
+      error 'no dictionaries defined!'
+      exit false
     end
     return definitions
   end
 
+  def verify_strings(strs)
+    debug 'strings to verify: ' << strs.to_s
+    if strs.respond_to?(:to_ary) 
+      strs.each {|s| return false if(!s || s.strip.empty?) }
+    else
+      return strs && !(strs.strip).empy?()
+    end
+    return true
+  end
+
 =begin
 Creates and returns an Array of Definitions.
 =end
@@ -152,43 +169,57 @@ Creates and returns an Array of Definitions.
       #TODO: Hash
       list = Array.new
       def_list = Array.new
+
       tag = $configuration.html_tag
       attr = $configuration.html_attribute
       value = $configuration.attr_value
-      xpath = ".//#{tag}[contains(@#{attr}, '#{value}')]"
-      # xpath = './/' << value << '[@' << attr << '="' << value << '"]'
-      @log.debug('xpath is ' << xpath)
-      tags = @doc.xpath(xpath)
-      tags.each do |t|
-        expression = t.attribute('title').to_s
-        unless (expression && !expression.strip.empty? )
-          expression = t.text.to_s
-        end
-        if(expression)
-          expression.gsub!(/\s+/, " ")
-          expression.strip!	
-          if(!expression.empty? && !def_list.include?(expression) )
-            if(!list.include?expression.downcase) 
-              list.push expression.downcase
-              definition = dict_definition(expression)
-              if(definition && !definition.empty?)
-                # @log.debug('definition is ' << definition.to_s)
-                def_list.push(format_index(definition) )
-                def_list.sort!
+
+      if verify_strings([tag,attr,value]) 
+        xpath = ".//#{tag}[contains(@#{attr}, '#{value}')]"
+        # xpath = './/' << value << '[@' << attr << '="' << value << '"]'
+        debug('xpath is ' << xpath)
+        tags = @doc.xpath(xpath)
+        if(tags && !tags.empty?)
+          tags.each do |t|
+            expression = t.attribute('title').to_s
+            unless (expression && !expression.strip.empty? )
+              expression = t.text.to_s
+            end
+            if(expression)
+              expression.gsub!(/\s+/, " ")
+              expression.strip!	
+              if(!expression.empty? && !def_list.include?(expression) )
+                if(!list.include?expression.downcase) 
+                  list.push expression.downcase
+                  definition = dict_definition(expression)
+                  if(definition && !definition.empty?)
+                    # debug('definition is ' << definition.to_s)
+                    def_list.push(format_index(definition) )
+                    def_list.sort!
+                  end
+                end
               end
             end
           end
+        else
+          error 'Cannot find any marked tags in the source-file. Aborting!'
+          raise SystemExit
+          #exit false
         end
+      else
+        error 'Configuration incomplete. Verify tag, attribute and attribute value! Aborting!'
+        raise SystemExit
+        #exit false
       end
-=begin
-      if(@log.level == Logger::DEBUG)
-        @log.debug('def_list is ') 
-        def_list.each_with_index{|t, i| @log.debug( "%i %s"%[i, t]) }
-      end
-=end
     rescue Interrupt => ex
-      @log.warn "--------------\n\tIndex generation interrupted by user\n---------------"
+      warn "--------------\n\tIndex generation interrupted by user\n---------------"
+    rescue SystemExit => sx
+      raise sx
+    rescue Exception => ex 
+      error( "line #{__LINE__} " << ex.message << ' (' << ex.class.name << ')')
+      exit false
     end
+
     return def_list
   end
 
@@ -212,11 +243,11 @@ Formats the definition text.
   # def temp_dir options = {:remove => true}
   def temp_dir
     @temp_dir ||= begin
-                    @log.debug('creating temp_dir')
+                    debug('creating temp_dir')
                     require 'tmpdir'
                     require 'fileutils'
                     path = File.join(Dir::tmpdir, "HTML2Index_#{Time.now.to_i}_#{rand(1000)}")
-                    @log.debug('temp-dir path is ' << path)
+                    debug('temp-dir path is ' << path)
                     Dir.mkdir(path)
                     # at_exit {FileUtils.rm_rf(path) if File.exists?(path)} if options[:remove]
                     File.new path
@@ -227,10 +258,12 @@ Formats the definition text.
     tempfile = nil
     out = nil
     out_file = $configuration.target
+    debug 'out_file is ' << out_file
     template = Template.new
+    debug 'template is ' << template.to_s
     if out_file
-      if(File.exist?(out_file))
-        if(File.writable?(out_file))
+      if(out_file != '<STDOUT>' && File.exist?(out_file) )
+        if( File.writable?(out_file) )
           puts "\nWARNING! File " << out_file << " exists!"
           print "Do you want to overwrite it? (Y/n) "
           res = wait_for_user
@@ -245,9 +278,10 @@ Formats the definition text.
         end
       end
     else
-      @log.debug('out_file is STDOUT')
+      info('Writing to ' << out_file)
     end
     begin
+      debug 'begin'
       html = template.to_s
 
       # create the definitions in index()
@@ -258,15 +292,15 @@ Formats the definition text.
       fdelim = $configuration.fdelim
       placeholders.each_pair do |k, v|
         repl = fdelim.dup << v << fdelim.reverse
-        @log.debug('try to replace ' << repl)
+        debug('try to replace ' << repl)
         html.gsub!(repl, data[k])
       end
-      @log.debug('html is now ' << html)
+      debug('html is now ' << html)
       html.gsub!(/\<head\>/, "<head>\n\t#{GeneratorMeta}")
       File.write(out_file, html) if out_file
       puts html if !out_file
     rescue Exception => ex
-      @log.error( "line #{__LINE__} " << ex.message << ' (' << ex.class.name << ')')
+      error( "line #{__LINE__} " << ex.message << ' (' << ex.class.name << ')')
     ensure
       out.close if out && !out.closed?
       File.unlink(out) if out

data/lib/template.rb

@@ -1,53 +1,44 @@
 #encoding: UTF-8 
+
 =begin
 /***************************************************************************
- *   ©2017-2017 Michael Uplawski <michael.uplawski@uplawski.eu>            *
+ *   ©2017-2026, Michael Uplawski <michael.uplawski@uplawski.eu>           *
  *                                                                         *
  *   This program is free software; you can redistribute it and/or modify  *
- *   it under the terms of the GNU General Public License as published by  *
- *   the Free Software Foundation; either version 3 of the License, or     *
- *   (at your option) any later version.                                   *
+ *   it under the terms of the WTFPL, version 2 or later, as published on  *
+ *   https://www.wtfpl.net/about/.                                         *
  *                                                                         *
  *   This program is distributed in the hope that it will be useful,       *
  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
- *   GNU General Public License for more details.                          *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
  *                                                                         *
- *   You should have received a copy of the GNU General Public License     *
- *   along with this program; if not, write to the                         *
- *   Free Software Foundation, Inc.,                                       *
- *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
  ***************************************************************************/
 =end
-require 'logger'
+
 require 'date'
-require_relative 'logging'
+require_relative 'basic_logging'
 require_relative 'dictionary'
 require_relative 'constants'
-require_relative 'file_checking'
 
 class Template
-  self.extend(Logging)
-  @@log = init_logger
+  include BasicLogging
 
   def initialize()
-    @log = @@log
-    @log.level = $log_level
+    set_level = $log_level
     template_file = $configuration.template
     @template = nil
     if template_file
-      msg = File_Checking::file_check(template_file, :file, :readable )
-      if !msg
-        ftype = File_Checking::file_type(template_file)
-        if ftype && !ftype.empty? && !ftype[0].downcase.match("(html|xml) .*document")
-          msg = template_file.dup << ' does not look like a valid template file for (x)html: ' << ftype.join('; ') 
+      error = nil 
+      [:file?, :readable?].each do |cond|
+        if(!File.send(cond, template_file) )
+           error = true
+           warn('Cannot use template ' << template_file << ': NOT ' << cond.to_s.split('?')[0] << ',' << "\n\tusing default template")
         end
       end
-      if(!msg)
+
+      if(!error)
         @template = File.read(template_file)
-        @log.debug('using template ' << template_file)
-      else
-        @log.warn('Cannot use template ' << template_file << ': ' << msg << "\n\tusing default template")
+        debug('using template ' << template_file)
       end
     end
   end
@@ -57,7 +48,7 @@ class Template
     if @template && @template.respond_to?(:to_str)
       str = @template
     else
-      @log.warn('Empty template, using default!')
+      warn('Empty template, using default!')
       str = @@def_template
     end
     str
@@ -74,8 +65,8 @@ class Template
     when :def_template
       @@def_template
     else
-      @log.error('Do not know ' << name.to_s)
-      @log.error('Aborting. Bye!')
+      error('Do not know ' << name.to_s)
+      error('Aborting. Bye!')
     end
   end
 

data/lib/translating.rb

@@ -1,23 +1,16 @@
 #encoding: UTF-8
 =begin
 /***************************************************************************
- *   ©2011-2017, Michael Uplawski				           * 
- *   <michael.uplawski@uplawski.eu>  					   *
+ *   ©2011-2026, Michael Uplawski <michael.uplawski@uplawski.eu>           *
  *                                                                         *
  *   This program is free software; you can redistribute it and/or modify  *
- *   it under the terms of the GNU General Public License as published by  *
- *   the Free Software Foundation; either version 3 of the License, or     *
- *   (at your option) any later version.                                   *
- *                                                                         * 
+ *   it under the terms of the WTFPL, version 2 or later, as published on  *
+ *   https://www.wtfpl.net/about/.                                         *
+ *                                                                         *
  *   This program is distributed in the hope that it will be useful,       *
  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
- *   GNU General Public License for more details.                          *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
  *                                                                         *
- *   You should have received a copy of the GNU General Public License     *
- *   along with this program; if not, write to the                         *
- *   Free Software Foundation, Inc.,                                       *
- *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
  ***************************************************************************/
 =end
 
@@ -25,7 +18,6 @@
 RD = File.expand_path(File.dirname(__FILE__) ) +  File::SEPARATOR if !defined?(RD)
 
 require 'yaml'
-require_relative 'file_checking'
 
 
 # A way to produce translated text in a ruby-program.
@@ -38,21 +30,32 @@ module Translating
 
 	@@lang = nil 
 	@@lang_file = format("%s%s", RD, 'LANG')  
-	@@tr = YAML::load_file("#{RD}translations")
-	
-	# find the current language-setting and return it.
-	def self.language()
-		if @@lang == nil
-			r = ENV['LANG']
-			if(r)
-				@@lang = r[0, 2]
-			elsif( !File_Checking::file_check(@@lang_file, [:exist?, :readable?]) && File::size(@@lang_file) >= 2)
-				File::open(@@lang_file, 'r') {|f| @@lang = f.readline}
-				@@lang.chomp!.downcase! if @@lang
-			end
-		end
-		@@lang = 'en' if !@@lang
-	end
+    @@tr = YAML::load_file("#{RD}translations")
+
+    # find the current language-setting and return it.
+    def self.language()
+      if @@lang == nil
+        r = ENV['LANG']
+        if(r)
+          @@lang = r[0, 2]
+        else
+          error = nil 
+          [:exist?, :readable?].each do |cond|
+            if(! File.send(cond, @@lang_File))
+              error = cond.to_s.split('?')[0]
+            end
+          end
+          if(!error && File::size(@@lang_file) >= 2)
+            File::open(@@lang_file, 'r') {|f| @@lang = f.readline}
+            @@lang.chomp!.downcase! if @@lang
+          else
+            warn 'Cannot use ' << @@lang_file << '; file unuseable'
+            info 'Language will be English'
+          end
+        end
+      end
+      @@lang = 'en' if !@@lang
+    end
 
 	# Translate a string to the currently set langage. 
 	# The args parameter may contain replacement-text which 

data/lib/user_input.rb

@@ -1,26 +1,18 @@
 #encoding: UTF-8
-
 =begin
 /***************************************************************************
- *   Copyright © 2014, Michael Uplawski <michael.uplawski@uplawski.eu>     *
+ *   ©2011-2026, Michael Uplawski <michael.uplawski@uplawski.eu>           *
  *                                                                         *
  *   This program is free software; you can redistribute it and/or modify  *
- *   it under the terms of the GNU General Public License as published by  *
- *   the Free Software Foundation; either version 3 of the License, or     *
- *   (at your option) any later version.                                   *
+ *   it under the terms of the WTFPL, version 2 or later, as published on  *
+ *   https://www.wtfpl.net/about/.                                         *
  *                                                                         *
  *   This program is distributed in the hope that it will be useful,       *
  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
- *   GNU General Public License for more details.                          *
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                  *
  *                                                                         *
- *   You should have received a copy of the GNU General Public License     *
- *   along with this program; if not, write to the                         *
- *   Free Software Foundation, Inc.,                                       *
- *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
  ***************************************************************************/
 =end
-
 require 'io/wait'
 require 'io/console'