libis-format 0.9.32 → 0.9.33

data/lib/libis/format/identification_tool.rb

@@ -0,0 +1,174 @@
+require 'csv'
+require 'tmpdir'
+
+require 'singleton'
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+require 'libis/format/type_database'
+
+module Libis
+  module Format
+
+    class IdentificationTool
+      include Singleton
+      include ::Libis::Tools::Logger
+
+      def self.bad_mimetype(mimetype)
+        self.instance.bad_mimetype(mimetype)
+      end
+
+      def self.run(file, recursive = false)
+        if file.is_a?(Array)
+          return run_list file
+        elsif file.is_a?(String) && File.exists?(file) && File.readable?(file)
+          if File.directory?(file)
+            return run_dir(file, recursive)
+          elsif File.file?(file)
+            return self.instance.run(file)
+          end
+        end
+        raise ArgumentError,
+              'IdentificationTool: file argument should be a path to an existing file or directory or a list of those'
+      end
+
+      def self.run_dir(file, recursive = true)
+        self.instance.run_dir file, recursive
+      end
+
+      def self.run_list(filelist)
+        self.instance.run_list filelist
+      end
+
+      protected
+
+      def create_list_file(filelist)
+        list_file = Dir::Tmpname.make_tmpname(%w'file .list', nil)
+        File.open(list_file, 'w') do |f|
+          filelist.each do |fname|
+            f.write "#{fname}\n"
+          end
+        end
+        yield(list_file)
+      ensure
+        File.delete(list_file)
+      end
+
+      def find_files(dir, recurse = true)
+        args = []
+        args << '-L'
+        args << dir.escape_for_string
+        args << '-maxdepth' << '1' unless recurse
+        args << '-type' << 'f'
+        args << '-print'
+        output = ::Libis::Tools::Command.run('find', *args)
+        warn "Find command errors: #{output[:err].join("\n")}" unless output[:err].empty?
+        output[:out]
+      end
+
+      # Reformat output to make it easier to post-process and decide on the preferred format
+      #
+      # input format:
+      # [
+      #   { filepath: <filename>, mimetype: <mimetype>, matchtype: <matchtype>, ... }
+      # ]
+      #
+      # output format:
+      #   { <filename> => [<result>, ...], ... }
+      #
+      # <result> is the enchanced Hash output of the identification tool:
+      #   { mimetype: <mimetype>, puid: <puid>, matchtype: <matchtype>, score: <score>, ...}
+      #
+      def process_output(output)
+        output.reduce({}) do |results, x|
+          filepath = x.delete(:filepath)
+          results[filepath] ||= []
+          results[filepath.freeze] << annotate(x)
+          results
+        end
+      end
+
+      # Enhance the output with mimetype and score
+      def annotate(result)
+        # Enhance result with mimetype if needed
+        if bad_mimetypes.include?(result[:mimetype]) && !bad_puids.include?(result[:puid])
+          result[:mimetype] = get_mimetype(result[:puid])
+        end
+
+        # Normalize the mimetype
+        Libis::Format::TypeDatabase.normalize(result, PUID: :puid, MIME: :mimetype)
+
+        # Default score is 5
+        result[:score] = 5
+
+        # Weak detection score is 1
+        result[:score] = 1 if bad_mimetypes.include? result[:mimetype]
+
+        # freeze all strings
+        result.each {|_, v| v.freeze if v.is_a?(String)}
+
+        # Adapt score based on matchtype
+        result[:matchtype] = result[:matchtype].to_s.downcase
+        case result[:matchtype]
+
+          # Signature match increases score with 2
+          when 'signature'
+            result[:score] += 2
+          # typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(result[:puid])
+          # ext = File.extname(result[:filename])
+          # result[:score] += 1 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
+
+          # Container match increases score with 4
+          when 'container'
+            result[:score] += 4
+          # typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(result[:puid])
+          # ext = File.extname(result[:filename])
+          # result[:score] += 1 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
+
+          # Extension match is the weakest identification; score is lowered by 2 points
+          when 'extension'
+            result[:score] -= 2
+
+          # Magic code (file tool) is to be trused even less
+          when 'magic'
+            result[:score] -= 3
+
+          # Or no change otherwise
+          else
+            # do nothing
+        end
+
+        # Detecting a zip file should decrease the score as it may hide one of the many zip-based formats (e.g. epub,
+        # Office OpenXML, OpenDocument, jar, maff, svx)
+        if result[:mimetype] == 'application/zip'
+          result[:score] -= 2
+        end
+
+        # Return result enhanced with mimetype and score fields
+        result
+      end
+
+      def get_mimetype(puid)
+        ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first rescue nil
+      end
+
+      def get_puid(mimetype)
+        ::Libis::Format::TypeDatabase.mime_infos(mimetype).first[:PUID].first rescue nil
+      end
+
+      attr_accessor :bad_mimetypes, :bad_puids
+
+      def initialize
+        @bad_mimetypes = [nil, '', 'None', 'application/octet-stream']
+        @bad_puids = [nil, 'fmt/unknown']
+      end
+
+      def bad_mimetype(mimetype)
+        @bad_mimetypes << mimetype
+      end
+    end
+
+  end
+end

data/lib/libis/format/identifier.rb

@@ -3,13 +3,17 @@
 require 'singleton'
 
 require 'libis-tools'
+require 'libis/tools/extend/hash'
 require 'libis/tools/extend/string'
 require 'libis/tools/extend/empty'
 
 require 'libis/format/type_database'
 
+require_relative 'config'
 require_relative 'fido'
 require_relative 'droid'
+require_relative 'file_tool'
+require_relative 'extension_identification'
 
 module Libis
   module Format
@@ -18,39 +22,8 @@ module Libis
       include ::Libis::Tools::Logger
       include Singleton
 
-      RETRY_MIMETYPES = %w(application/zip) + ::Libis::Format::Fido::BAD_MIMETYPES
-      FIDO_FAILURES = %w(application/vnd.oasis.opendocument.text application/vnd.oasis.opendocument.spreadsheet)
-
-      attr_reader :xml_validations
-
-      protected
-
-      def initialize
-        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
-        @fido_formats = [(File.join(data_dir, 'lias_formats.xml'))]
-        # noinspection RubyStringKeysInHashInspection
-        @xml_validations = {'archive/ead' => File.join(data_dir, 'ead.xsd')}
-      end
-
-      def result_ok?(result, who_is_asking = nil)
-        result = ::Libis::Format::TypeDatabase.enrich(result, PUID: :puid, MIME: :mimetype)
-        return false if result.empty?
-        return true unless result[:TYPE].empty?
-        return false if RETRY_MIMETYPES.include? result[:mimetype]
-        return false if FIDO_FAILURES.include? result[:mimetype] and who_is_asking == :DROID
-        !(result[:mimetype].empty? and result[:puid].empty?)
-      end
-
-      def get_puid(mimetype)
-        ::Libis::Format::TypeDatabase.mime_infos(mimetype).first[:PUID].first rescue nil
-      end
-
       public
 
-      def self.add_fido_format(f)
-        ::Libis::Format::Fido.add_format f
-      end
-
       def self.add_xml_validation(mimetype, xsd_file)
         instance.xml_validations[mimetype] = xsd_file
       end
@@ -59,134 +32,173 @@ module Libis
         instance.xml_validations
       end
 
-      def self.get(file_path, options = nil)
-        instance.get file_path, options
+      def self.get(file, options = {})
+        instance.get file, options
       end
 
-      def get(file, options = nil)
+      attr_reader :xml_validations
 
-        unless File.exists? file
-          error 'File %s cannot be found.', file
-          return nil
-        end
-        if File.directory? file
-          error '%s is a directory.', file
-          return nil
-        end
+      def get(file, options = {})
 
-        options ||= {}
+        options[:droid] = true unless options[:tool] and options[:tool] != :droid
+        options[:fido] = true unless options[:tool] and options[:tool] != :fido
+        options[:file] = true unless options[:tool] and options[:tool] != :file
 
-        result = {messages: []}
+        result = {messages: [], output: {}, formats: {}}
 
-        # use FIDO
-        # Note: FIDO does not always do a good job, mainly due to lacking container inspection.
-        # FIDO misses should be registered in
-        result = get_fido_identification(file, result, options[:formats]) unless options[:droid]
+        begin
+          get_droid_identification(file, options[:recursive], result) if options[:droid]
+        rescue => e
+          log_msg(result, :error, "Error running Droid: #{e.message} @ #{e.backtrace.first}")
+        end
 
-        # use DROID
-        result = get_droid_identification file, result
+        begin
+          get_fido_identification(file, options[:recursive], result) if options[:fido]
+        rescue => e
+          log_msg(result, :error, "Error running Fido: #{e.message} @ #{e.backtrace.first}")
+        end
 
-        # use FILE
-        result = get_file_identification(file, result)
+        begin
+          get_file_identification(file, options[:recursive], result) if options[:file]
+        rescue => e
+          log_msg(result, :error, "Error running File: #{e.message} @ #{e.backtrace.first}")
+        end
 
-        # Try file extension
-        result = get_extension_identification(file, result)
+        # get_extension_identification(file, options[:recursive], result)
 
         # determine XML type. Add custom types at runtime with
         # Libis::Tools::Format::Identifier.add_xml_validation('my_type', '/path/to/my_type.xsd')
-        result = validate_against_xml_schema(file, result)
+        validate_against_xml_schema(result)
 
-        result[:mimetype] ?
-            log_msg(result, :info, "Identification of '#{file}': '#{result}'") :
-            log_msg(result, :warn, "Could not identify MIME type of '#{file}'")
-      end
+        process_results(result)
 
-      def get_fido_identification(file, result = {}, xtra_formats = nil)
-        return result if result_ok? result
+        # result[:mimetype] ?
+        #     log_msg(result, :info, "Identification of '#{file}': '#{result}'") :
+        #     log_msg(result, :warn, "Could not identify MIME type of '#{file}'")
 
-        fido_result = ::Libis::Format::Fido.run(file, xtra_formats)
+        result
 
-        return result unless fido_result.is_a? Hash
+      end
 
-        result.merge! fido_result
-        result[:method] = 'fido'
+      protected
 
-        log_msg(result, :debug, "Fido MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})")
+      def initialize
+        @xml_validations = Libis::Format::Config[:xml_validations].to_h
       end
 
-      def get_droid_identification(file, result = {})
-        return result if result_ok? result, :DROID
-        droid_output = ::Libis::Format::Droid.run file
-        result[:messages] << [:debug, "DROID: #{droid_output}"]
-        warn 'Droid found multiple matches; using first match only' if droid_output.size > 1
-        result.clear
-        droid_output = droid_output.first
-        result[:mimetype] = droid_output[:mime_type].to_s.split(/[\s,]+/).find { |x| x =~ /.*\/.*/ }
-        result[:matchtype] = droid_output[:method]
-        result[:puid] = droid_output[:puid]
-        result[:format_name] = droid_output[:format_name]
-        result[:format_version] = droid_output[:format_version]
-        result[:method] = 'droid'
+      def get_file_identification(file, recursive, result)
+        output = ::Libis::Format::FileTool.run(file, recursive)
+        process_tool_output(output, result)
+        output
+      end
 
-        log_msg(result, :debug, "Droid MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})")
+      def get_fido_identification(file, recursive, result)
+        output = ::Libis::Format::Fido.run(file, recursive)
+        process_tool_output(output, result)
+        output
       end
 
-      def get_file_identification(file, result = nil)
-        return result if result_ok? result
-        begin
-          output = ::Libis::Tools::Command.run('file', '-b', '--mime-type', "\"#{file.escape_for_string}\"")[:err]
-          mimetype = output.strip.split
-          if mimetype
-            log_msg(result, :debug, "File result: '#{mimetype}'")
-            result[:mimetype] = mimetype
-            result[:puid] = get_puid(mimetype)
-          end
-          result[:method] = 'file'
-        rescue Exception
-          # ignored
-        end
-        result
+      def get_droid_identification(file, recursive, result)
+        output = ::Libis::Format::Droid.run(file, recursive)
+        process_tool_output(output, result)
+        output
+      end
+
+      def get_extension_identification(file, recursive, result)
+        output = ::Libis::Format::ExtensionIdentification.run(file, recursive)
+        process_tool_output(output, result)
+        output
       end
 
-      def get_extension_identification(file, result = nil)
-        return result if result_ok? result
-        info = ::Libis::Format::TypeDatabase.ext_infos(File.extname(file)).first
-        log_msg result, :debug, "File extension info: #{info}"
-        if info
-          result[:mimetype] = info[:MIME].first rescue nil
-          result[:puid] = info[:PUID].first rescue nil
+      def validate_against_xml_schema(result)
+        result[:output].each do |file, file_results|
+          file_results.each do |file_result|
+            xml_validate(file, file_result, result)
+          end
         end
-        result[:method] = 'extension'
-        result
       end
 
-      def validate_against_xml_schema(file, result)
-        return result unless result[:mimetype] =~ /^(text|application)\/xml$/
+      def xml_validate(file, file_result, result)
+        return unless file_result[:mimetype] =~ /^(text|application)\/xml$/
         doc = ::Libis::Tools::XmlDocument.open file
         xml_validations.each do |mime, xsd_file|
           next unless xsd_file
           begin
             if doc.validates_against?(xsd_file)
               log_msg result, :debug, "XML file validated against XML Schema: #{xsd_file}"
-              result[:mimetype] = mime
-              result[:puid] = nil
-              result = ::Libis::Format::TypeDatabase.enrich(result, PUID: :puid, MIME: :mimetype)
+              info = {mimetype: mime, tool: file_result[:source], source: :xsd_validation, match_type: 'xsd_validation', format_version: '', }
+              file_result.merge! Libis::Format::TypeDatabase.enrich(info, PUID: :puid, MIME: :mimetype, NAME: :format_name)
             end
-          rescue
-            # Do nothing - probably Nokogiri chrashed during validation.
-            # Could have many causes (remote schema: firewall, network, link rot, ...; schema syntax error; ...)
-            # so we just ignore and continue.
+          rescue => e
+            # Do nothing - probably Nokogiri chrashed during validation. Could have many causes
+            # (remote schema (firewall, network, link rot, ...), schema syntax error, corrupt XML,...)
+            # so we log and continue.
+            log_msg(result, :warn, "Error during XML validation: #{e.message}")
           end
         end
-        result
+      end
+
+      def process_results(result)
+        result[:output].map do |file, output|
+          file_result = result[:formats][file] = {}
+          if output.empty?
+            file_result = {
+                mimetype: 'application/octet-stream',
+                puid: 'fmt/unknown',
+                score: 0,
+                source: nil
+            }
+          else
+            format_matches = output.group_by {|x| [x[:mimetype], x[:puid]]}
+            format_matches.each do |match, group|
+              format_matches[match] = group.group_by {|x| x[:score]}.sort.reverse.to_h
+            end
+            case format_matches.count
+              when 0
+                # No this really cannot happen. If there are not hits, there will be at least a format [nil,nil]
+              when 1
+                # only one match, that's easy. The first of the highest score will be used
+                file_result.merge!(get_best_result(output))
+              else
+                process_multiple_formats(file_result, format_matches, output)
+            end
+          end
+        end
+      end
+
+      def process_multiple_formats(file_result, format_matches, output)
+        # multiple matches. Let's select the highest score matches
+        file_result.merge!(get_best_result(output))
+        file_result[:alternatives] = []
+        format_matches.keys.each do |mime, puid|
+          next if file_result[:mimetype] == mime && puid.nil?
+              selection = output.select {|x| x[:mimetype] == mime && x[:puid] == puid}
+          file_result[:alternatives] << get_best_result(selection)
+        end
+        file_result[:alternatives] = file_result[:alternatives].sort_by {|x| x[:score]}.reverse
+        file_result.delete(:alternatives) if file_result[:alternatives].size <= 1
       end
 
       private
 
+      def process_tool_output(output, result)
+        output.each do |file, file_output|
+          result[:output][file] ||= []
+          result[:output][file] += file_output
+        end
+      end
+
       def log_msg(result, severity, text)
-        return {} unless result.is_a?(Hash)
-        (result[:messages] ||= []) << [severity, text]
-        result
+        result[:messages] << [severity, text]
+      end
+
+      def get_mimetype(puid)
+        ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first rescue nil
+      end
+
+      def get_best_result(results)
+        score = results.map {|x| x[:score]}.max
+        results.select {|x| x[:score] == score}.reduce(:apply_defaults)
       end
 
     end

data/lib/libis/format/type_database.rb

@@ -7,6 +7,8 @@ require 'backports/rails/hash'
 require 'libis/tools/logger'
 require 'libis/tools/extend/string'
 
+require_relative 'config'
+
 module Libis
   module Format
 
@@ -19,6 +21,25 @@ module Libis
       end
 
       def self.enrich(info, map_keys = {})
+        return {} unless info.is_a? Hash
+        mapper = Hash.new {|hash,key| hash[key] = key}
+        mapper.merge! map_keys
+        unless (puid = info[mapper[:PUID]]).blank?
+          info[mapper[:TYPE]] ||= puid_infos(puid).first[:TYPE] rescue nil
+        end
+        unless (mime = info[mapper[:MIME]]).blank?
+          info[mapper[:TYPE]] ||= mime_infos(mime).first[:TYPE] rescue nil
+        end
+        unless (type_name = info[mapper[:TYPE]]).nil?
+          mapper.keys.each do |key|
+            info[mapper[key]] = get(type_name, key) || info[mapper[key]]
+          end
+          info[mapper[:GROUP]] = self.type_group(type_name)
+        end
+        info
+      end
+
+      def self.normalize(info, map_keys = {})
         return {} unless info.is_a? Hash
         mapper = Hash.new {|hash,key| hash[key] = key}
         mapper.merge! map_keys
@@ -29,14 +50,25 @@ module Libis
           info[mapper[:TYPE]] ||= self.mime_infos(mime).first[:TYPE] rescue nil
         end
         unless (type_name = info[mapper[:TYPE]]).nil?
-          info[mapper[:MIME]] = self.type_mimetypes(type_name).first if info[mapper[:MIME]].blank?
-          info[mapper[:PUID]] = self.type_puids(type_name).first if info[mapper[:PUID]].blank?
-          info[mapper[:EXTENSIONS]] = self.type_extentions(type_name)
+          info[mapper[:MIME]] = self.type_mimetypes(type_name).first if self.type_mimetypes(type_name).first
           info[mapper[:GROUP]] = self.type_group(type_name)
         end
         info
       end
 
+      def self.get(type_name, key)
+        case key
+          when :MIME
+            type_mimetypes(type_name).first
+          when :PUID
+            type_puids(type_name).first
+          when :EXTENSION
+            type_extentions(type_name).first
+          else
+            self.typeinfo(type_name)[key]
+        end
+      end
+
       def self.type_group(t)
         typeinfo(t)[:GROUP]
       end
@@ -159,8 +191,7 @@ module Libis
 
       def initialize
         @types = Hash.new
-        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
-        type_database = File.join(data_dir, 'types.yml')
+        type_database = Libis::Format::Config[:type_database]
         load_types(type_database)
       end
 

data/lib/libis/format/version.rb

@@ -1,5 +1,5 @@
 module Libis
   module Format
-    VERSION = '0.9.32'
+    VERSION = '0.9.33'
   end
 end

data/lib/libis/format.rb

@@ -5,9 +5,12 @@ module Libis
     autoload :Config, 'libis/format/config'
     autoload :TypeDatabase, 'libis/format/type_database'
     autoload :Identifier, 'libis/format/identifier'
+    autoload :Identifier, 'libis/format/identifier'
 
+    autoload :FileTool, 'libis/format/file_tool'
     autoload :Fido, 'libis/format/fido'
     autoload :Droid, 'libis/format/droid'
+    autoload :ExtensionIdentification, 'libis/format/extension_identification'
 
     autoload :OfficeToPdf, 'libis/format/office_to_pdf'
     autoload :PdfCopy, 'libis/format/pdf_copy'

data/libis-format.gemspec

@@ -26,8 +26,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.3'
   spec.add_development_dependency 'rspec', '~> 3.1'
   spec.add_development_dependency 'simplecov', '~> 0.9'
+  spec.add_development_dependency 'awesome_print'
 
-  spec.add_runtime_dependency 'libis-tools', '~> 0.9'
+  spec.add_runtime_dependency 'libis-tools', '~> 0.9.52'
   spec.add_runtime_dependency 'os', '= 0.9.6'
   spec.add_runtime_dependency 'mini_magick', '~> 4.3'
   spec.add_runtime_dependency 'deep_dive', '~> 0.3'

data/spec/converter_spec.rb

@@ -23,6 +23,8 @@ describe 'Converters' do
 
   before(:all) {
     Libis::Tools::Config.logger.level = :WARN
+    ::Libis::Format::Config[:droid_path] = '/opt/droid/droid.sh'
+    ::Libis::Format::Config[:fido_path] = '/usr/local/bin/fido'
   }
 
   context 'Repository' do
@@ -114,7 +116,7 @@ describe 'Converters' do
       expect(result).to eq tgt_file
       compare = MiniMagick::Tool::Compare.new
       compare << ref_file << tgt_file
-      compare.metric << 'AE'
+      compare.metric << 'MAE'
       compare.fuzz << '1%'
       compare << diff_file
       compare.call {|_, _, status| expect(status).to be 0}
@@ -132,7 +134,7 @@ describe 'Converters' do
       expect(result).to eq tgt_file
       compare = MiniMagick::Tool::Compare.new
       compare << ref_file << tgt_file
-      compare.metric << 'AE'
+      compare.metric << 'MAE'
       compare << diff_file
       compare.call {|_, _, status| expect(status).to be 0}
       FileUtils.rm tgt_file, force: true
@@ -167,7 +169,7 @@ describe 'Converters' do
       compare = MiniMagick::Tool::Compare.new
       compare << ref_file << tgt_file
       compare.metric << 'AE'
-      compare.fuzz << '1%'
+      compare.fuzz << '100%'
       compare << diff_file
       compare.call do |_stdin, _stdout, status|
         expect(status).to be 0
@@ -189,7 +191,7 @@ describe 'Converters' do
       expect(File.exist?(tgt_file)).to be_truthy
       compare = MiniMagick::Tool::Compare.new
       compare << ref_file << tgt_file
-      compare.metric << 'AE'
+      compare.metric << 'MAE'
       compare.fuzz << '10%'
       compare << diff_file
       compare.call {|_,_,status| expect(status).to be 0}