libis-format 0.9.1

data/lib/libis/format/fido.rb

@@ -0,0 +1,109 @@
+require 'os'
+require 'csv'
+require 'singleton'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/type_database'
+
+module Libis
+  module Format
+
+    class Fido
+      include ::Libis::Tools::Logger
+      include Singleton
+
+      BAD_MIMETYPES = [nil, '', 'None', 'application/octet-stream']
+
+      def self.run(file, formats = nil)
+        instance.run file, formats
+      end
+
+      def run(file, xtra_formats = nil)
+
+        fido_results = []
+
+        fmt_list = formats.dup
+        case xtra_formats
+          when Array
+            fmt_list += xtra_formats
+          when String
+            fmt_list << xtra_formats
+          else
+            # do nothing
+        end
+
+        bin_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'tools', 'fido'))
+        cmd = File.join(bin_dir, OS.windows? ? 'fido.bat' : 'fido.sh')
+        args = []
+        args << '-loadformats' << "#{fmt_list.join(',')}" unless fmt_list.empty?
+        args << "#{file.escape_for_string}"
+        fido = ::Libis::Tools::Command.run(cmd, *args)
+        warn "Fido errors: #{fido[:err].join("\n")}" unless fido[:err].empty?
+
+        keys = [:status, :time, :puid, :format_name, :signature_name, :filesize, :filename, :mimetype, :matchtype]
+        fido_output = CSV.parse(fido[:out].join("\n")).map { |a| Hash[keys.zip(a)] }
+        debug "Fido output: #{fido_output}"
+
+        fido_output.each do |x|
+          if x[:status] == 'OK'
+            x[:mimetype] = get_mimetype(x[:puid]) if x[:mimetype] == 'None'
+            next if BAD_MIMETYPES.include? x[:mimetype]
+            x[:score] = 5
+            case x[:matchtype]
+              when 'signature'
+                x[:score] += 5
+              when 'container'
+                typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(x[:puid])
+                ext = File.extname(file)
+                x[:score] += 2 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
+              else
+                # do nothing
+            end
+            fido_results << x
+          end
+        end
+
+        fido_results = fido_results.inject({}) do |result, value|
+          result[value[:score]] ||= []
+          result[value[:score]] << value
+          result
+        end
+
+        debug "Fido results: #{fido_results}"
+
+        max_score = fido_results.keys.max
+
+        # Only if we find a single hit of type 'signature' or 'container', we are confident enough to return a result
+        return {} unless max_score and max_score >= 5 && fido_results[max_score].size == 1
+
+        fido_results[max_score].first
+      end
+
+      def self.add_format(f)
+        instance.formats << f
+      end
+
+      def self.formats
+        instance.formats
+      end
+
+      protected
+
+      attr_reader :formats
+
+      def initialize
+        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
+        @formats = [(File.join(data_dir, 'lias_formats.xml'))]
+      end
+
+      def get_mimetype(puid)
+        ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first rescue nil
+      end
+
+    end
+
+  end
+end

data/lib/libis/format/identifier.rb

@@ -0,0 +1,185 @@
+# encoding: utf-8
+
+require 'singleton'
+
+require 'libis-tools'
+require 'libis/tools/extend/string'
+require 'libis/tools/extend/empty'
+
+require 'libis/format/type_database'
+
+require_relative 'fido'
+require_relative 'droid'
+
+module Libis
+  module Format
+
+    class Identifier
+      include ::Libis::Tools::Logger
+      include Singleton
+
+      RETRY_MIMETYPES = %w(application/zip) + ::Libis::Format::Fido::BAD_MIMETYPES
+      FIDO_FAILURES = %w(application/vnd.oasis.opendocument.text application/vnd.oasis.opendocument.spreadsheet)
+
+      attr_reader :xml_validations
+
+      protected
+
+      def initialize
+        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
+        @fido_formats = [(File.join(data_dir, 'lias_formats.xml'))]
+        # noinspection RubyStringKeysInHashInspection
+        @xml_validations = {'archive/ead' => File.join(data_dir, 'ead.xsd')}
+      end
+
+      def result_ok?(result, who_is_asking = nil)
+        result = ::Libis::Format::TypeDatabase.enrich(result, PUID: :puid, MIME: :mimetype)
+        return false if result.empty?
+        return true unless result[:TYPE].empty?
+        return false if RETRY_MIMETYPES.include? result[:mimetype]
+        return false if FIDO_FAILURES.include? result[:mimetype] and who_is_asking == :DROID
+        !(result[:mimetype].empty? and result[:puid].empty?)
+      end
+
+      def get_puid(mimetype)
+        ::Libis::Format::TypeDatabase.mime_infos(mimetype).first[:PUID].first rescue nil
+      end
+
+      public
+
+      def self.add_fido_format(f)
+        ::Libis::Format::Fido.add_format f
+      end
+
+      def self.add_xml_validation(mimetype, xsd_file)
+        instance.xml_validations[mimetype] = xsd_file
+      end
+
+      def self.xml_validations
+        instance.xml_validations
+      end
+
+      def self.get(file_path, options = nil)
+        instance.get file_path, options
+      end
+
+      def get(file, options = nil)
+
+        unless File.exists? file
+          error 'File %s cannot be found.', file
+          return nil
+        end
+        if File.directory? file
+          error '%s is a directory.', file
+          return nil
+        end
+
+        options ||= {}
+
+        result = {}
+
+        # use FIDO
+        # Note: FIDO does not always do a good job, mainly due to lacking container inspection.
+        # FIDO misses should be registered in
+        result = get_fido_identification(file, result, options[:formats]) unless options[:droid]
+
+        # use DROID
+        result = get_droid_identification file, result
+
+        # use FILE
+        result = get_file_identification(file, result)
+
+        # Try file extension
+        result = get_extension_identification(file, result)
+
+        # determine XML type. Add custom types at runtime with
+        # Libis::Tools::Format::Identifier.add_xml_validation('my_type', '/path/to/my_type.xsd')
+        result = validate_against_xml_schema(file, result)
+
+        result ? info("Identification of '#{file}': '#{result}'") : warn("Could not identify MIME type of '#{file}'")
+
+        result
+      end
+
+      def get_fido_identification(file, result = {}, xtra_formats = nil)
+        return result if result_ok? result
+
+        fido_result = ::Libis::Format::Fido.run(file, xtra_formats)
+
+        return result unless fido_result.is_a? Hash
+
+        result.merge! fido_result
+        result[:method] = 'fido'
+
+        debug "Fido MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})" unless result.empty?
+        result
+      end
+
+      def get_droid_identification(file, result = {})
+        return result if result_ok? result, :DROID
+        droid_output = ::Libis::Format::Droid.run file
+        debug "DROID: #{droid_output}"
+        warn 'Droid found multiple matches; using first match only' if droid_output.size > 1
+        result.clear
+        droid_output = droid_output.first
+        result[:mimetype] = droid_output[:mime_type].to_s.split(/[\s,]+/).find {|x| x =~ /.*\/.*/}
+        result[:matchtype] = droid_output[:method]
+        result[:puid] = droid_output[:puid]
+        result[:format_name] = droid_output[:format_name]
+        result[:format_version] = droid_output[:format_version]
+        result[:method] = 'droid'
+
+        debug "Droid MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})" if result
+        result
+      end
+
+      def get_file_identification(file, result = nil)
+        return result if result_ok? result
+        result = {}
+        begin
+          output = ::Libis::Tools::Command.run('file', '-b', '--mime-type', "\"#{file.escape_for_string}\"")[:err]
+          mimetype = output.strip.split
+          if mimetype
+            debug "File result: '#{mimetype}'"
+            result[:mimetype] = mimetype
+            result[:puid] = get_puid(mimetype)
+          end
+          result[:method] = 'file'
+        rescue Exception
+          # ignored
+        end
+        result
+      end
+
+      def get_extension_identification(file, result = nil)
+        return result if result_ok? result
+        result = {}
+        info = ::Libis::Format::TypeDatabase.ext_infos(File.extname(file)).first
+        debug "File extension info: #{info}"
+        if info
+          result[:mimetype] = info[:MIME].first rescue nil
+          result[:puid] = info[:PUID].first rescue nil
+        end
+        result[:method] = 'extension'
+        result
+      end
+
+      def validate_against_xml_schema(file, result)
+        return result unless result[:mimetype] =~ /^(text|application)\/xml$/
+        doc = ::Libis::Tools::XmlDocument.open file
+        xml_validations.each do |mime, xsd_file|
+          next unless xsd_file
+          if doc.validates_against?(xsd_file)
+            debug "XML file validated against XML Schema: #{xsd_file}"
+            result[:mimetype] = mime
+            result[:puid] = nil
+            result = ::Libis::Format::TypeDatabase.enrich(result, PUID: :puid, MIME: :mimetype)
+          end
+        end
+        result
+      end
+
+    end
+
+  end
+end

data/lib/libis/format/type_database.rb

@@ -0,0 +1,170 @@
+# coding: utf-8
+
+require 'singleton'
+require 'yaml'
+
+require 'backports/rails/hash'
+require 'libis/tools/logger'
+require 'libis/tools/extend/string'
+
+module Libis
+  module Format
+
+    class TypeDatabase
+      include Singleton
+      include ::Libis::Tools::Logger
+
+      def self.typeinfo(t)
+        self.instance.types[t] || {}
+      end
+
+      def self.enrich(info, map_keys = {})
+        return {} unless info.is_a? Hash
+        mapper = Hash.new {|hash,key| hash[key] = key}
+        mapper.merge! map_keys
+        unless (puid = info[mapper[:PUID]]).blank?
+          info[mapper[:TYPE]] ||= self.puid_infos(puid).first[:TYPE] rescue nil
+        end
+        unless (mime = info[mapper[:MIME]]).blank?
+          info[mapper[:TYPE]] ||= self.mime_infos(mime).first[:TYPE] rescue nil
+        end
+        unless (type_name = info[mapper[:TYPE]]).nil?
+          info[mapper[:MIME]] = self.type_mimetypes(type_name).first if info[mapper[:MIME]].blank?
+          info[mapper[:PUID]] = self.type_puids(type_name).first if info[mapper[:PUID]].blank?
+          info[mapper[:EXTENSIONS]] = self.type_extentions(type_name)
+          info[mapper[:GROUP]] = self.type_group(type_name)
+        end
+        info
+      end
+
+      def self.type_group(t)
+        typeinfo(t)[:GROUP]
+      end
+
+      def self.type_mimetypes(t)
+        typeinfo(t)[:MIME] || []
+      end
+
+      def self.type_puids(t)
+        typeinfo(t)[:PUID] || []
+      end
+
+      def self.type_extentions(t)
+        typeinfo(t)[:EXTENSIONS] || []
+      end
+
+      def self.group_types(group)
+        self.instance.types.select do |_, v|
+          v[:GROUP] == group.to_sym
+        end.keys
+      end
+
+      def self.puid_infos(puid)
+        self.instance.types.select do |_, v|
+          v[:PUID].include? puid rescue false
+        end.values
+      end
+
+      def self.puid_types(puid)
+        self.instance.types.select do |_, v|
+          v[:PUID].include? puid rescue false
+        end.keys
+      end
+
+      def self.puid_groups(puid)
+        puid_types(puid).map do |t|
+          type_group t
+        end
+      end
+
+      def self.mime_infos(mime)
+        self.instance.types.select do |_, v|
+          v[:MIME].include? mime rescue false
+        end.values
+      end
+
+      def self.mime_types(mime)
+        self.instance.types.select do |_, v|
+          v[:MIME].include? mime rescue false
+        end.keys
+      end
+
+      def self.mime_groups(mime)
+        mime_types(mime).map do |t|
+          type_group t
+        end
+      end
+
+      def self.ext_infos(ext)
+        ext = ext.gsub /^\./, ''
+        self.instance.types.select do |_, v|
+          v[:EXTENSIONS].include?(ext) rescue false
+        end.values
+      end
+
+      def self.ext_types(ext)
+        ext = ext.gsub /^\./, ''
+        self.instance.types.select do |_, v|
+          v[:EXTENSIONS].include?(ext) rescue false
+        end.keys
+      end
+
+      def self.puid_typeinfo(puid)
+        self.instance.types.each do |_, v|
+          return v if v[:PUID] and v[:PUID].include?(puid)
+        end
+        nil
+      end
+
+      def self.known_mime?(mime)
+        self.instance.types.each do |_, v|
+          return true if v[:MIME].include? mime
+        end
+        false
+      end
+
+      attr_reader :types
+
+      def load_types(file_or_hash = {}, append = true)
+        hash = file_or_hash.is_a?(Hash) ? file_or_hash : YAML::load_file(file_or_hash)
+        # noinspection RubyResolve
+        hash.each do |group, type_info|
+          type_info.each do |type_name, info|
+            type_key = type_name.to_sym
+            info.symbolize_keys!
+            info[:TYPE] = type_key
+            info[:GROUP] = group.to_sym
+            info[:MIME] = info[:MIME].strip.split(/[\s,]+/).map { |v| v.strip } rescue []
+            info[:EXTENSIONS] = info[:EXTENSIONS].strip.split(/[\s,]+/).map { |v| v.strip } rescue []
+            info[:PUID] = info[:PUID].strip.split(/[\s,]+/).map { |v| v.strip } if info[:PUID]
+            if @types.has_key?(type_key)
+              warn 'Type %s already defined; merging with info from %s.', type_name.to_s, file_or_hash
+              info.merge!(@types[type_key]) do |_,v_new,v_old|
+                case v_old
+                  when Array
+                    append ? v_old + v_new : v_new + v_old
+                  when Hash
+                    append ? v_new.merge(v_old) : v_old.merge(v_new)
+                  else
+                    append ? v_old : v_new
+                end
+              end
+            end
+            @types[type_key] = info
+          end
+        end
+      end
+
+      protected
+
+      def initialize
+        @types = Hash.new
+        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
+        type_database = File.join(data_dir, 'types.yml')
+        load_types(type_database)
+      end
+
+    end
+
+  end
+end

data/lib/libis/format/version.rb

@@ -0,0 +1,5 @@
+module Libis
+  module Format
+    VERSION = '0.9.1'
+  end
+end

data/lib/libis/format.rb

@@ -0,0 +1,12 @@
+require 'libis/format/version'
+
+module Libis
+  module Format
+    autoload :TypeDatabase, 'libis/format/type_database'
+    autoload :Identifier, 'libis/format/identifier'
+    autoload :Fido, 'libis/format/fido'
+    autoload :Droid, 'libis/format/droid'
+
+    autoload :Converter, 'libis/format/converter'
+  end
+end

data/lib/libis-format.rb

@@ -0,0 +1 @@
+require 'libis/format'

data/libis-format.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'libis/format/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'libis-format'
+  spec.version       = Libis::Format::VERSION
+  spec.authors       = ['Kris Dekeyser']
+  spec.email         = ['kris.dekeyser@libis.be']
+  spec.summary       = %q{LIBIS File format format services.}
+  spec.description   = %q{Collection of tools and classes that help to identify formats of binary files and create derivative copies (e.g. PDF from Word).}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/[^/]+$}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake', '~> 10.3'
+  spec.add_development_dependency 'rspec', '~> 3.1'
+  spec.add_development_dependency 'simplecov', '~> 0.9'
+
+  spec.add_runtime_dependency 'libis-tools', '~> 0.9'
+  spec.add_runtime_dependency 'os', '= 0.9.6'
+end