libis-format 1.3.7.1 → 2.0.0

data/lib/libis/format/library.rb

@@ -0,0 +1,147 @@
+# froze_string_litteral: true
+# coding: utf-8
+require 'singleton'
+
+module Libis
+  module Format
+
+    class Library
+      include Singleton
+
+      class << self
+        def implementation=(impl)
+          instance.implementation = impl
+        end
+
+        def get_info(format)
+          instance.get_info(format)
+        end
+
+        def get_info_by(key, value)
+          instance.get_info_by(key, value)
+        end
+
+        def get_infos_by(key, value)
+          instance.get_infos_by(key, value)
+        end
+
+        def get_field(format, field)
+          instance.get_field(format, field)
+        end
+
+        def get_field_by(key, value, field)
+          instance.get_field_by(key, value, field)
+        end
+
+        def get_fields_by(key, value, field)
+          instance.get_fields_by(key, value, field)
+        end
+
+        def known?(key, value)
+          instance.known?(key, value)
+        end
+
+        def enrich(info, map_keys = {})
+          instance.enrich(info, map_keys)
+        end
+
+        def normalize(info, map_keys = {})
+          instance.normalize(info, map_keys)
+        end
+      end
+
+      def implementation=(impl)
+        @implementation = impl
+      end
+
+      def get_field(format, field)
+        get_field_by(:name, format, field)
+      end
+
+      def get_field_by(key, value, field)
+        info = get_info_by(key, value)
+        return nil unless info
+        case field
+        when :mimetype
+          info[:mimetypes]&.first
+        when :puid
+          info[:puids]&.first
+        when :extension
+          info[:extensions]&.first
+        else
+          info[field]
+        end
+      end
+
+      def get_fields_by(key, value, field)
+        get_infos_by(key, value)&.map { |info| info[field] }.compact
+      end
+
+      def get_info(format)
+        get_info_by(:name, format)
+      end
+
+      def get_info_by(key, value)
+        get_infos_by(key, value)&.first
+      end
+
+      def get_infos_by(key, value)
+        result = @implementation.query(key, value)
+        result.map(&:to_hash)
+      end
+
+      def known?(key, value)
+        !get_info_by(key, value).nil?
+      end
+
+      def enrich(info, map_keys = {})
+        info = normalize(info, map_keys)
+        mapper = Hash.new { |hash, key| hash[key] = key }.merge(map_keys)
+        unless (format = info[mapper[:name]]).nil?
+          lib_info = get_info(format)
+          mapper.keys.each do |key|
+            case key
+            when :mimetype
+              info[mapper[key]] = lib_info[:mimetypes].first if lib_info[:mimetypes].first
+            when :puid
+              info[mapper[key]] = lib_info[:puids].first if lib_info[:puids].first
+            when :extension
+              info[mapper[key]] = lib_info[:extensions].first if lib_info[:extensions].first
+            else
+              info[mapper[key]] = lib_info[key] if lib_info[key]
+            end
+          end
+        end
+        info
+      end
+
+      # Derive name from the available info
+      def normalize(info, map_keys = {})
+        return {} unless info.is_a? Hash
+        mapper = Hash.new { |hash, key| hash[key] = key }.merge(map_keys)
+        # fill format from looking up by puid
+        unless (puid = info[mapper[:puid]]).blank?
+          info[mapper[:name]] ||= get_field_by(:puid, puid, :name)
+        end
+        # fill format from looking up by mimetype
+        unless (mime = info[mapper[:mimetype]]).blank?
+          info[mapper[:name]] ||= get_field_by(:mimetype, mime, :name)
+        end
+        # finally complete the information from looking up by format name
+        unless (format = info[mapper[:name]]).nil?
+          info[mapper[:mimetype]] = get_field(format, :mimetype)
+          info[mapper[:category]] = get_field(format, :category)
+        end
+        info
+      end
+
+      private
+
+      def initialize
+        @implementation = eval(Libis::Format::Config[:format_library_implementation])
+      end
+
+    end
+
+  end
+end

data/lib/libis/format/tool/droid.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'singleton'
 
 require 'tempfile'
@@ -7,10 +5,10 @@ require 'csv'
 
 require 'libis/format/config'
 
-unless CSV::HeaderConverters.key?(:droid_headers)
-  CSV::HeaderConverters[:droid_headers] = lambda { |h|
-    h.encode(ConverterEncoding).downcase.strip
-     .gsub(/\W+/, '').to_sym
+unless CSV::HeaderConverters.has_key?(:droid_headers)
+  CSV::HeaderConverters[:droid_headers] = lambda {|h|
+    h.encode(ConverterEncoding).downcase.strip.
+        gsub(/\W+/, "").to_sym
   }
 end
 
@@ -19,7 +17,9 @@ require_relative 'identification_tool'
 module Libis
   module Format
     module Tool
+
       class Droid < Libis::Format::Tool::IdentificationTool
+
         def run_list(filelist, _options = {})
           runner(filelist)
         end
@@ -43,13 +43,12 @@ module Libis
         end
 
         def parse_report(report)
-          keys = %i[
-            id parent_id uri filepath filename matchtype status filesize type extension
-            mod_time ext_mismatch hash format_count puid mimetype format_name format_version
-          ]
+          keys = [
+              :id, :parent_id, :uri, :filepath, :filename, :matchtype, :status, :filesize, :type, :extension,
+              :mod_time, :ext_mismatch, :hash, :format_count, :puid, :mimetype, :format_name, :format_version]
           result = CSV.parse(File.readlines(report).join)
-                      .map { |a| Hash[keys.zip(a)] }
-                      .select { |a| a[:type] == 'File' }
+                       .map {|a| Hash[keys.zip(a)]}
+                       .select {|a| a[:type] == 'File'}
           # File.delete report
           result.each do |r|
             r.delete(:id)
@@ -71,39 +70,39 @@ module Libis
 
         def create_report(profile, report)
           args = [
-            '-e', report,
-            '-p', profile,
-            '-q'
+              '-e', report,
+              '-p', profile,
+              '-q'
           ]
           timeout = Libis::Format::Config[:timeouts][:droid]
           result = Libis::Tools::Command.run(
-            Libis::Format::Config[:droid_cmd], *args,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:droid_cmd], *args,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
-          result[:err].select! { |x| x =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} ERROR / }
-          raise "#{self.class} report took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} report errors: #{result[:err].join("\n")}" unless result[:err].empty?
+          result[:err].select! {|x| x =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} ERROR /}
+          raise RuntimeError, "#{self.class} report took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} report errors: #{result[:err].join("\n")}" unless result[:err].empty?
 
           File.delete profile
         end
 
         def create_profile(file_or_list, profile, recursive = false)
           args = []
-          files = file_or_list.is_a?(Array) ? file_or_list.map(&:escape_for_string) : [file_or_list.escape_for_string]
-          files.each { |file| args << '-a' << file }
+          files = (file_or_list.is_a?(Array)) ? file_or_list.map(&:escape_for_string) : [file_or_list.escape_for_string]
+          files.each {|file| args << '-a' << file}
           args << '-q'
           args << '-p' << profile
           args << '-R' if recursive
           timeout = Libis::Format::Config[:timeouts][:droid]
           result = Libis::Tools::Command.run(
-            Libis::Format::Config[:droid_cmd], *args,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:droid_cmd], *args,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
-          result[:err].select! { |x| x =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} ERROR / }
-          raise "#{self.class} profile took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} profile errors: #{result[:err].join("\n")}" unless result[:err].empty?
+          result[:err].select! {|x| x =~ /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} ERROR /}
+          raise RuntimeError, "#{self.class} profile took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} profile errors: #{result[:err].join("\n")}" unless result[:err].empty?
         end
 
         def profile_file_name
@@ -113,7 +112,9 @@ module Libis
         def result_file_name
           Tools::TempFile.name('droid', '.csv')
         end
+
       end
+
     end
   end
 end

data/lib/libis/format/tool/extension_identification.rb

@@ -1,56 +1,58 @@
-# frozen_string_literal: true
-
 require_relative 'identification_tool'
 
 module Libis
   module Format
     module Tool
+
       class ExtensionIdentification < Libis::Format::Tool::IdentificationTool
-        def run_list(filelist, **_options)
+
+        def run_list(filelist, _options = {})
+
           output = runner(nil, filelist)
 
           process_output(output)
+
         end
 
-        def run_dir(dir, recursive = true, **_options)
+        def run_dir(dir, recursive = true, _options = {})
+
           filelist = find_files(dir, recursive)
 
           output = runner(nil, filelist)
 
           process_output(output)
+
         end
 
-        def run(file, **_options)
+        def run(file, _options)
+
           output = runner(file)
 
           process_output(output)
+
         end
 
         protected
 
         def runner(*args)
+
           args.map do |file|
-            info = ::Libis::Format::TypeDatabase.ext_infos(File.extname(file)).first
-            next unless info
-
-            {
-              filepath: file,
-              mimetype: begin
-                info[:MIME].first
-              rescue StandardError
-                nil
-              end,
-              puid: begin
-                info[:PUID].first
-              rescue StandardError
-                nil
-              end,
-              matchtype: 'extension',
-              tool: :type_database
-            }
+            info = ::Libis::Format::Library.get_info_by(:extension, File.extname(file))
+            if info
+              {
+                  filepath: file,
+                  mimetype: (info[:mimetypes].first rescue nil),
+                  puid: (info[:puids].first rescue nil),
+                  matchtype: 'extension',
+                  tool: :format_library
+              }
+            end
           end.cleanup
+
         end
+
       end
+
     end
   end
-end
+end

data/lib/libis/format/tool/{ff_mpeg.rb → ffmpeg.rb}

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'libis/tools/extend/string'
 require 'libis/tools/extend/empty'
 require 'libis/tools/command'
@@ -10,17 +8,13 @@ require 'libis/format/config'
 module Libis
   module Format
     module Tool
+
       class FFMpeg
         include Singleton
         include ::Libis::Tools::Logger
 
-        def self.installed?
-          result = Libis::Tools::Command.run(Libis::Format::Config[:ffmpeg_cmd], '-h')
-          (result[:status]).zero?
-        end
-
         def self.run(source, target, options = {})
-          instance.run source, target, options
+          self.instance.run source, target, options
         end
 
         def run(source, target, options = {})
@@ -34,22 +28,21 @@ module Libis
 
           timeout = Libis::Format::Config[:timeouts][:ffmpeg]
           result = Libis::Tools::Command.run(
-            Libis::Format::Config[:ffmpeg_cmd], *opts,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:ffmpeg_cmd], *opts,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
 
-          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
+          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
           warn "FFMpeg warnings: #{(result[:err] + result[:out]).join("\n")}" unless result[:err].empty?
 
-          {
-            command: result,
-            files: [target]
-          }
+          result[:out]
         end
+
       end
+
     end
   end
 end

data/lib/libis/format/tool/fido.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'libis/tools/extend/string'
 require 'libis/tools/command'
 
@@ -11,33 +9,36 @@ require_relative 'identification_tool'
 module Libis
   module Format
     module Tool
+
       class Fido < Libis::Format::Tool::IdentificationTool
+
         def self.add_formats(formats_file)
-          instance.formats << formats_file unless instance.formats.include?(formats_file)
+          self.instance.formats << formats_file unless self.instance.formats.include?(formats_file)
         end
 
         def self.del_formats(formats_file)
-          instance.formats.delete(formats_file)
+          self.instance.formats.delete(formats_file)
         end
 
         attr_reader :formats
 
-        def run_list(filelist, **options)
+        def run_list(filelist, options = {})
           create_list_file(filelist) do |list_file|
-            output = runner(nil, '-input', list_file.escape_for_string, **options)
+            output = runner(nil, '-input', list_file.escape_for_string, options)
             process_output(output)
           end
         end
 
-        def run_dir(dir, recursive = true, **options)
+        def run_dir(dir, recursive = true, options = {})
           args = []
           args << '-recurse' if recursive
-          output = runner(dir, *args, **options)
+          args << options
+          output = runner(dir, *args)
           process_output(output)
         end
 
-        def run(file, **options)
-          output = runner(file, **options)
+        def run(file, options = {})
+          output = runner(file, options)
           process_output(output)
         end
 
@@ -52,9 +53,11 @@ module Libis
 
         attr_writer :formats
 
-        def runner(filename, *args, **options)
-          # Load custom format definitions if present
-          args << '-loadformats' << formats.join(',').to_s unless formats.empty?
+        def runner(filename, *args)
+          options = {}
+          options = args.pop if args.last.is_a?(Hash)
+          # Load custome format definitions if present
+          args << '-loadformats' << "#{formats.join(',')}" unless formats.empty?
 
           # Workaround for Fido performance bug
           args << '-bufsize' << (options[:bufsize] || 1000).to_s
@@ -65,7 +68,7 @@ module Libis
           args << '-nocontainer' if options[:nocontainer]
 
           # Add filename to argument list (optional)
-          args << filename.escape_for_string.to_s if filename
+          args << "#{filename.escape_for_string}" if filename
 
           # No header output
           args << '-q'
@@ -73,20 +76,20 @@ module Libis
           # Run command and capture results
           timeout = Libis::Format::Config[:timeouts][:fido]
           result = ::Libis::Tools::Command.run(
-            Libis::Format::Config[:fido_cmd], *args,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:fido_cmd], *args,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
 
           # Log warning if needed
-          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} errors: #{result[:err].join("\n")}" unless (result[:status])&.zero? && result[:err].empty?
+          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
           # Parse output (CSV) text into array and return result
-          keys = %i[status time puid format_name format_version filesize filepath mimetype matchtype]
+          keys = [:status, :time, :puid, :format_name, :format_version, :filesize, :filepath, :mimetype, :matchtype]
           data = CSV.parse(result[:out].join("\n"))
-                    .map { |a| Hash[keys.zip(a)] }
-                    .select { |a| a[:status] == 'OK' }
+                       .map {|a| Hash[keys.zip(a)]}
+                       .select {|a| a[:status] == 'OK'}
           data.each do |r|
             r.delete(:time)
             r.delete(:status)
@@ -94,7 +97,9 @@ module Libis
             r[:tool] = :fido
           end
         end
+
       end
+
     end
   end
 end

data/lib/libis/format/tool/file_tool.rb

@@ -1,38 +1,49 @@
-# frozen_string_literal: true
-
 require_relative 'identification_tool'
 
 module Libis
   module Format
     module Tool
+
       class FileTool < Libis::Format::Tool::IdentificationTool
-        def run_list(filelist, **_options)
+
+        def run_list(filelist, _options = {})
+
           create_list_file(filelist) do |list_file|
+
             output = runner(nil, '--files-from', list_file)
 
             process_output(output)
+
           end
+
         end
 
-        def run_dir(dir, recursive = true, **_options)
+        def run_dir(dir, recursive = true, _options = {})
+
           filelist = find_files(dir, recursive)
 
           create_list_file(filelist) do |list_file|
+
             output = runner(nil, '--files-from', list_file)
 
             process_output(output)
+
           end
+
         end
 
-        def run(file, **_options)
+        def run(file, _options = {})
+
           output = runner(file)
 
           process_output(output)
+
         end
 
         protected
 
         def runner(filename, *args)
+
           # Create new argument list
           opts = []
 
@@ -50,21 +61,23 @@ module Libis
           # Run the UNIX file command and capture the results
           timeout = Libis::Format::Config[:timeouts][:file_tool]
           result = ::Libis::Tools::Command.run(
-            'file', *opts,
-            timeout:,
-            kill_after: timeout * 2
+              'file', *opts,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
 
-          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} errors: #{result[:err].join("\n")}" unless (result[:status]).zero? && result[:err].empty?
+          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
           # Parse output text into array and return result
           result[:out].map do |line|
             r = line.split(/:\s+/)
-            { filepath: r[0], mimetype: r[1], matchtype: 'magic', tool: :file }
+            {filepath: r[0], mimetype: r[1], matchtype: 'magic', tool: :file}
           end
         end
+
       end
+
     end
   end
 end

data/lib/libis/format/tool/fop_pdf.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'os'
 
 require 'libis/tools/extend/string'
@@ -11,46 +9,37 @@ require 'libis/format/config'
 module Libis
   module Format
     module Tool
+
       class FopPdf
         include ::Libis::Tools::Logger
 
-        def self.installed?
-          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], '-version')
-          return false unless (result[:status]).zero?
-
-          File.exist?(Libis::Format::Config[:fop_jar])
+        def self.run(xml, target, options = [])
+          self.new.run xml, target, options
         end
 
-        def self.run(xml, target)
-          new.run xml, target
-        end
+        def run(xml, target, options = [])
 
-        def run(xml, target)
           if OS.java?
             # TODO: import library and execute in current VM. For now do exactly as in MRI.
           end
 
           timeout = Libis::Format::Config[:timeouts][:fop]
           result = Libis::Tools::Command.run(
-            Libis::Format::Config[:java_cmd],
-            "-Dfop.home=#{File.dirname(Libis::Format::Config[:fop_jar])}",
-            '-Djava.awt.headless=true',
-            '-jar', Libis::Format::Config[:fop_jar],
-            '-fo', xml,
-            '-pdf', target,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:java_cmd],
+              "-Dfop.home=#{File.dirname(Libis::Format::Config[:fop_jar])}",
+              '-jar', Libis::Format::Config[:fop_jar],
+              '-fo', xml,
+              '-pdf', target,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
 
-          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} errors: #{result[:err].join("\n")}" unless (result[:status]).zero?
+          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
-          {
-            command: result,
-            files: [target]
-          }
         end
       end
+
     end
   end
 end