libis-format 1.3.0 → 2.0.0

data/lib/libis/format/identifier.rb

@@ -1,4 +1,4 @@
-# frozen_string_literal: true
+# encoding: utf-8
 
 require 'singleton'
 require 'pathname'
@@ -7,8 +7,9 @@ require 'libis-tools'
 require 'libis/tools/extend/hash'
 require 'libis/tools/extend/string'
 require 'libis/tools/extend/empty'
+require 'nori/core_ext/object'
 
-require 'libis/format/type_database'
+require 'libis/format/library'
 
 require_relative 'config'
 require_relative 'tool/fido'
@@ -18,10 +19,13 @@ require_relative 'tool/extension_identification'
 
 module Libis
   module Format
+
     class Identifier
       include ::Libis::Tools::Logger
       include Singleton
 
+      public
+
       def self.add_xml_validation(mimetype, xsd_file)
         instance.xml_validations[mimetype] = xsd_file
       end
@@ -37,28 +41,29 @@ module Libis
       attr_reader :xml_validations
 
       def get(file, options = {})
-        options[:droid] = true unless options.keys.include?(:droid) || (options[:tool] && (options[:tool] != :droid))
-        options[:fido] = true unless options.keys.include?(:fido) || (options[:tool] && (options[:tool] != :fido))
-        options[:file] = true unless options.keys.include?(:file) || (options[:tool] && (options[:tool] != :file))
+
+        options[:droid] = true unless options.keys.include?(:droid) or (options[:tool] and options[:tool] != :droid)
+        options[:fido] = true unless options.keys.include?(:fido) or (options[:tool] and options[:tool] != :fido)
+        options[:file] = true unless options.keys.include?(:file) or (options[:tool] and options[:tool] != :file)
         options[:xml_validation] = true if options[:xml_validation].nil?
 
-        result = { messages: [], output: {}, formats: {} }
+        result = {messages: [], output: {}, formats: {}}
 
         begin
           get_droid_identification(file, result, options) if options[:droid]
-        rescue StandardError => e
+        rescue => e
           log_msg(result, :error, "Error running Droid: #{e.message} @ #{e.backtrace.first}")
         end
 
         begin
           get_fido_identification(file, result, options) if options[:fido]
-        rescue StandardError => e
+        rescue => e
           log_msg(result, :error, "Error running Fido: #{e.message} @ #{e.backtrace.first}")
         end
 
         begin
           get_file_identification(file, result, options) if options[:file]
-        rescue StandardError => e
+        rescue => e
           log_msg(result, :error, "Error running File: #{e.message} @ #{e.backtrace.first}")
         end
 
@@ -69,13 +74,14 @@ module Libis
         # Libis::Tools::Format::Identifier.add_xml_validation('my_type', '/path/to/my_type.xsd')
         begin
           validate_against_xml_schema(result, options[:base_dir]) if options[:xml_validation]
-        rescue StandardError => e
+        rescue => e
           log_msg(result, :error, "Error validating XML files: #{e.message} @ #{e.backtrace.first}")
         end
 
         process_results(result, !options[:keep_output])
 
         result
+
       end
 
       protected
@@ -91,7 +97,7 @@ module Libis
       end
 
       def get_fido_identification(file, result, options)
-        output = ::Libis::Format::Tool::Fido.run(file, options[:recursive], **(options[:fido_options] || {}))
+        output = ::Libis::Format::Tool::Fido.run(file, options[:recursive], options[:fido_options])
         process_tool_output(output, result, options[:base_dir])
         output
       end
@@ -117,21 +123,18 @@ module Libis
       end
 
       def xml_validate(file, file_result, result, base_dir)
-        return unless file_result[:mimetype] =~ %r{^(text|application)/xml$}
-
+        return unless file_result[:mimetype] =~ /^(text|application)\/xml$/
         filepath = base_dir ? File.join(base_dir, file) : file
         doc = ::Libis::Tools::XmlDocument.open filepath
         xml_validations.each do |mime, xsd_file|
           next unless xsd_file
-
           begin
             if doc.validates_against?(xsd_file)
               log_msg result, :debug, "XML file validated against XML Schema: #{xsd_file}"
-              info = { mimetype: mime, tool_raw: file_result[:tool], tool: :xsd_validation, match_type: 'xsd_validation',
-                       format_version: '' }
-              file_result.merge! Libis::Format::TypeDatabase.enrich(info, PUID: :puid, MIME: :mimetype, NAME: :format_name)
+              info = {mimetype: mime, tool_raw: file_result[:tool], tool: :xsd_validation, match_type: 'xsd_validation', format_version: '', }
+              file_result.merge! Libis::Format::Library.enrich(info, puid: :puid, mimetype: :mimetype, name: :format_name)
             end
-          rescue StandardError => e
+          rescue => e
             # Do nothing - probably Nokogiri chrashed during validation. Could have many causes
             # (remote schema (firewall, network, link rot, ...), schema syntax error, corrupt XML,...)
             # so we log and continue.
@@ -139,7 +142,7 @@ module Libis
                     "Error during XML validation of file #{file} against #{File.basename(xsd_file)}: #{e.message}")
           end
         end
-      rescue StandardError => e
+      rescue => e
         # Not much we can do. probably Nokogiri chrashed opening the XML file. What caused this?
         # (XML not parsable, false XML identification, ???)
         # so we log and continue.
@@ -147,30 +150,30 @@ module Libis
       end
 
       def process_results(result, delete_output = true)
-        result[:output].each_key do |file|
+        result[:output].keys.each do |file|
           output = result[:output][file]
           file_result = result[:formats][file] = {}
           if output.empty?
             log_msg(result, :warn, "Could not identify format of '#{file}'.")
-            file_result.merge!(
-              mimetype: 'application/octet-stream',
-              puid: 'fmt/unknown',
-              score: 0,
-              tool: nil
-            )
+            file_result = {
+                mimetype: 'application/octet-stream',
+                puid: 'fmt/unknown',
+                score: 0,
+                tool: nil
+            }
           else
-            format_matches = output.group_by { |x| [x[:mimetype], x[:puid]] }
+            format_matches = output.group_by {|x| [x[:mimetype], x[:puid]]}
             format_matches.each do |match, group|
-              format_matches[match] = group.group_by { |x| x[:score] }.sort.reverse.to_h
+              format_matches[match] = group.group_by {|x| x[:score]}.sort.reverse.to_h
             end
             case format_matches.count
-            when 0
-              # No this really cannot happen. If there are no hits, there will be at least a format [nil,nil]
-            when 1
-              # only one match, that's easy. The first of the highest score will be used
-              file_result.merge!(get_best_result(output))
-            else
-              process_multiple_formats(file_result, format_matches, output)
+              when 0
+                # No this really cannot happen. If there are no hits, there will be at least a format [nil,nil]
+              when 1
+                # only one match, that's easy. The first of the highest score will be used
+                file_result.merge!(get_best_result(output))
+              else
+                process_multiple_formats(file_result, format_matches, output)
             end
           end
         end
@@ -181,13 +184,12 @@ module Libis
         # multiple matches. Let's select the highest score matches
         file_result.merge!(get_best_result(output))
         file_result[:alternatives] = []
-        format_matches.each_key do |mime, puid|
+        format_matches.keys.each do |mime, puid|
           next if file_result[:mimetype] == mime && puid.nil?
-
-          selection = output.select { |x| x[:mimetype] == mime && x[:puid] == puid }
+          selection = output.select {|x| x[:mimetype] == mime && x[:puid] == puid}
           file_result[:alternatives] << get_best_result(selection)
         end
-        file_result[:alternatives] = file_result[:alternatives].sort_by { |x| x[:score] }.reverse
+        file_result[:alternatives] = file_result[:alternatives].sort_by {|x| x[:score]}.reverse
         file_result.delete(:alternatives) if file_result[:alternatives].size <= 1
       end
 
@@ -206,15 +208,15 @@ module Libis
       end
 
       def get_mimetype(puid)
-        ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first
-      rescue StandardError
-        nil
+        ::Libis::Format::Library.get_field_by(:puid, puid, :mimetypes)
       end
 
       def get_best_result(results)
-        score = results.map { |x| x[:score] }.max
-        results.select { |x| x[:score] == score }.reduce(:apply_defaults)
+        score = results.map {|x| x[:score]}.max
+        results.select {|x| x[:score] == score}.reduce(:apply_defaults)
       end
+
     end
+
   end
 end

data/lib/libis/format/info.rb

@@ -0,0 +1,27 @@
+module Libis
+  module Format
+    class Info
+      attr_reader :name, :category, :description, :puids, :mimetypes, :extensions
+
+      def initialize(name:, category:, description: '', puids: [], mimetypes: [], extensions: [])
+        @name = name
+        @category = category
+        @description = description
+        @puids = puids
+        @mimetypes = mimetypes
+        @extensions = extensions
+      end
+
+      def to_hash
+        {
+            name: name,
+            description: description.dup,
+            category: category,
+            puids: puids.dup,
+            mimetypes: mimetypes.dup,
+            extensions: extensions.dup
+        }
+      end
+    end
+  end
+end

data/lib/libis/format/library.rb

@@ -0,0 +1,147 @@
+# froze_string_litteral: true
+# coding: utf-8
+require 'singleton'
+
+module Libis
+  module Format
+
+    class Library
+      include Singleton
+
+      class << self
+        def implementation=(impl)
+          instance.implementation = impl
+        end
+
+        def get_info(format)
+          instance.get_info(format)
+        end
+
+        def get_info_by(key, value)
+          instance.get_info_by(key, value)
+        end
+
+        def get_infos_by(key, value)
+          instance.get_infos_by(key, value)
+        end
+
+        def get_field(format, field)
+          instance.get_field(format, field)
+        end
+
+        def get_field_by(key, value, field)
+          instance.get_field_by(key, value, field)
+        end
+
+        def get_fields_by(key, value, field)
+          instance.get_fields_by(key, value, field)
+        end
+
+        def known?(key, value)
+          instance.known?(key, value)
+        end
+
+        def enrich(info, map_keys = {})
+          instance.enrich(info, map_keys)
+        end
+
+        def normalize(info, map_keys = {})
+          instance.normalize(info, map_keys)
+        end
+      end
+
+      def implementation=(impl)
+        @implementation = impl
+      end
+
+      def get_field(format, field)
+        get_field_by(:name, format, field)
+      end
+
+      def get_field_by(key, value, field)
+        info = get_info_by(key, value)
+        return nil unless info
+        case field
+        when :mimetype
+          info[:mimetypes]&.first
+        when :puid
+          info[:puids]&.first
+        when :extension
+          info[:extensions]&.first
+        else
+          info[field]
+        end
+      end
+
+      def get_fields_by(key, value, field)
+        get_infos_by(key, value)&.map { |info| info[field] }.compact
+      end
+
+      def get_info(format)
+        get_info_by(:name, format)
+      end
+
+      def get_info_by(key, value)
+        get_infos_by(key, value)&.first
+      end
+
+      def get_infos_by(key, value)
+        result = @implementation.query(key, value)
+        result.map(&:to_hash)
+      end
+
+      def known?(key, value)
+        !get_info_by(key, value).nil?
+      end
+
+      def enrich(info, map_keys = {})
+        info = normalize(info, map_keys)
+        mapper = Hash.new { |hash, key| hash[key] = key }.merge(map_keys)
+        unless (format = info[mapper[:name]]).nil?
+          lib_info = get_info(format)
+          mapper.keys.each do |key|
+            case key
+            when :mimetype
+              info[mapper[key]] = lib_info[:mimetypes].first if lib_info[:mimetypes].first
+            when :puid
+              info[mapper[key]] = lib_info[:puids].first if lib_info[:puids].first
+            when :extension
+              info[mapper[key]] = lib_info[:extensions].first if lib_info[:extensions].first
+            else
+              info[mapper[key]] = lib_info[key] if lib_info[key]
+            end
+          end
+        end
+        info
+      end
+
+      # Derive name from the available info
+      def normalize(info, map_keys = {})
+        return {} unless info.is_a? Hash
+        mapper = Hash.new { |hash, key| hash[key] = key }.merge(map_keys)
+        # fill format from looking up by puid
+        unless (puid = info[mapper[:puid]]).blank?
+          info[mapper[:name]] ||= get_field_by(:puid, puid, :name)
+        end
+        # fill format from looking up by mimetype
+        unless (mime = info[mapper[:mimetype]]).blank?
+          info[mapper[:name]] ||= get_field_by(:mimetype, mime, :name)
+        end
+        # finally complete the information from looking up by format name
+        unless (format = info[mapper[:name]]).nil?
+          info[mapper[:mimetype]] = get_field(format, :mimetype)
+          info[mapper[:category]] = get_field(format, :category)
+        end
+        info
+      end
+
+      private
+
+      def initialize
+        @implementation = eval(Libis::Format::Config[:format_library_implementation])
+      end
+
+    end
+
+  end
+end

data/lib/libis/format/tool/droid.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'singleton'
 
 require 'tempfile'
@@ -7,10 +5,10 @@ require 'csv'
 
 require 'libis/format/config'
 
-unless CSV::HeaderConverters.key?(:droid_headers)
-  CSV::HeaderConverters[:droid_headers] = lambda { |h|
-    h.encode(ConverterEncoding).downcase.strip
-     .gsub(/\W+/, '').to_sym
+unless CSV::HeaderConverters.has_key?(:droid_headers)
+  CSV::HeaderConverters[:droid_headers] = lambda {|h|
+    h.encode(ConverterEncoding).downcase.strip.
+        gsub(/\W+/, "").to_sym
   }
 end
 
@@ -19,7 +17,9 @@ require_relative 'identification_tool'
 module Libis
   module Format
     module Tool
+
       class Droid < Libis::Format::Tool::IdentificationTool
+
         def run_list(filelist, _options = {})
           runner(filelist)
         end
@@ -43,13 +43,12 @@ module Libis
         end
 
         def parse_report(report)
-          keys = %i[
-            id parent_id uri filepath filename matchtype status filesize type extension
-            mod_time ext_mismatch hash format_count puid mimetype format_name format_version
-          ]
+          keys = [
+              :id, :parent_id, :uri, :filepath, :filename, :matchtype, :status, :filesize, :type, :extension,
+              :mod_time, :ext_mismatch, :hash, :format_count, :puid, :mimetype, :format_name, :format_version]
           result = CSV.parse(File.readlines(report).join)
-                      .map { |a| Hash[keys.zip(a)] }
-                      .select { |a| a[:type] == 'File' }
+                       .map {|a| Hash[keys.zip(a)]}
+                       .select {|a| a[:type] == 'File'}
           # File.delete report
           result.each do |r|
             r.delete(:id)
@@ -71,39 +70,39 @@ module Libis
 
         def create_report(profile, report)
           args = [
-            '-e', report,
-            '-p', profile,
-            '-q'
+              '-e', report,
+              '-p', profile,
+              '-q'
           ]
           timeout = Libis::Format::Config[:timeouts][:droid]
           result = Libis::Tools::Command.run(
-            Libis::Format::Config[:droid_cmd], *args,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:droid_cmd], *args,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
-          result[:err].select! { |x| x =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} ERROR / }
-          raise "#{self.class} report took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} report errors: #{result[:err].join("\n")}" unless result[:err].empty?
+          result[:err].select! {|x| x =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} ERROR /}
+          raise RuntimeError, "#{self.class} report took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} report errors: #{result[:err].join("\n")}" unless result[:err].empty?
 
           File.delete profile
         end
 
         def create_profile(file_or_list, profile, recursive = false)
           args = []
-          files = file_or_list.is_a?(Array) ? file_or_list.map(&:escape_for_string) : [file_or_list.escape_for_string]
-          files.each { |file| args << '-a' << file }
+          files = (file_or_list.is_a?(Array)) ? file_or_list.map(&:escape_for_string) : [file_or_list.escape_for_string]
+          files.each {|file| args << '-a' << file}
           args << '-q'
           args << '-p' << profile
           args << '-R' if recursive
           timeout = Libis::Format::Config[:timeouts][:droid]
           result = Libis::Tools::Command.run(
-            Libis::Format::Config[:droid_cmd], *args,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:droid_cmd], *args,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
-          result[:err].select! { |x| x =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} ERROR / }
-          raise "#{self.class} profile took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} profile errors: #{result[:err].join("\n")}" unless result[:err].empty?
+          result[:err].select! {|x| x =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} ERROR /}
+          raise RuntimeError, "#{self.class} profile took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} profile errors: #{result[:err].join("\n")}" unless result[:err].empty?
         end
 
         def profile_file_name
@@ -113,7 +112,9 @@ module Libis
         def result_file_name
           Tools::TempFile.name('droid', '.csv')
         end
+
       end
+
     end
   end
 end

data/lib/libis/format/tool/extension_identification.rb

@@ -1,56 +1,58 @@
-# frozen_string_literal: true
-
 require_relative 'identification_tool'
 
 module Libis
   module Format
     module Tool
+
       class ExtensionIdentification < Libis::Format::Tool::IdentificationTool
-        def run_list(filelist, **_options)
+
+        def run_list(filelist, _options = {})
+
           output = runner(nil, filelist)
 
           process_output(output)
+
         end
 
-        def run_dir(dir, recursive = true, **_options)
+        def run_dir(dir, recursive = true, _options = {})
+
           filelist = find_files(dir, recursive)
 
           output = runner(nil, filelist)
 
           process_output(output)
+
         end
 
-        def run(file, **_options)
+        def run(file, _options)
+
           output = runner(file)
 
           process_output(output)
+
         end
 
         protected
 
         def runner(*args)
+
           args.map do |file|
-            info = ::Libis::Format::TypeDatabase.ext_infos(File.extname(file)).first
-            next unless info
-
-            {
-              filepath: file,
-              mimetype: begin
-                info[:MIME].first
-              rescue StandardError
-                nil
-              end,
-              puid: begin
-                info[:PUID].first
-              rescue StandardError
-                nil
-              end,
-              matchtype: 'extension',
-              tool: :type_database
-            }
+            info = ::Libis::Format::Library.get_info_by(:extension, File.extname(file))
+            if info
+              {
+                  filepath: file,
+                  mimetype: (info[:mimetypes].first rescue nil),
+                  puid: (info[:puids].first rescue nil),
+                  matchtype: 'extension',
+                  tool: :format_library
+              }
+            end
           end.cleanup
+
         end
+
       end
+
     end
   end
-end
+end

data/lib/libis/format/tool/{ff_mpeg.rb → ffmpeg.rb}

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'libis/tools/extend/string'
 require 'libis/tools/extend/empty'
 require 'libis/tools/command'
@@ -10,17 +8,13 @@ require 'libis/format/config'
 module Libis
   module Format
     module Tool
+
       class FFMpeg
         include Singleton
         include ::Libis::Tools::Logger
 
-        def self.installed?
-          result = Libis::Tools::Command.run(Libis::Format::Config[:ffmpeg_cmd], '-h')
-          (result[:status]).zero?
-        end
-
         def self.run(source, target, options = {})
-          instance.run source, target, options
+          self.instance.run source, target, options
         end
 
         def run(source, target, options = {})
@@ -34,22 +28,21 @@ module Libis
 
           timeout = Libis::Format::Config[:timeouts][:ffmpeg]
           result = Libis::Tools::Command.run(
-            Libis::Format::Config[:ffmpeg_cmd], *opts,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:ffmpeg_cmd], *opts,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
 
-          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} errors: #{result[:err].join("\n")}" unless (result[:status]).zero?
+          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
           warn "FFMpeg warnings: #{(result[:err] + result[:out]).join("\n")}" unless result[:err].empty?
 
-          {
-            command: result,
-            files: [target]
-          }
+          result[:out]
         end
+
       end
+
     end
   end
 end