libis-format 0.9.41 → 0.9.44

data/lib/libis/format/identifier.rb

@@ -12,10 +12,10 @@ require 'nori/core_ext/object'
 require 'libis/format/type_database'
 
 require_relative 'config'
-require_relative 'fido'
-require_relative 'droid'
-require_relative 'file_tool'
-require_relative 'extension_identification'
+require_relative 'tool/fido'
+require_relative 'tool/droid'
+require_relative 'tool/file_tool'
+require_relative 'tool/extension_identification'
 
 module Libis
   module Format
@@ -78,7 +78,7 @@ module Libis
           log_msg(result, :error, "Error validating XML files: #{e.message} @ #{e.backtrace.first}")
         end
 
-        process_results(result)
+        process_results(result, !options[:keep_output])
 
         result
 
@@ -91,25 +91,25 @@ module Libis
       end
 
       def get_file_identification(file, result, options)
-        output = ::Libis::Format::FileTool.run(file, options[:recursive])
+        output = ::Libis::Format::Tool::FileTool.run(file, options[:recursive])
         process_tool_output(output, result, options[:base_dir])
         output
       end
 
       def get_fido_identification(file, result, options)
-        output = ::Libis::Format::Fido.run(file, options[:recursive])
+        output = ::Libis::Format::Tool::Fido.run(file, options[:recursive])
         process_tool_output(output, result, options[:base_dir])
         output
       end
 
       def get_droid_identification(file, result, options)
-        output = ::Libis::Format::Droid.run(file, options[:recursive])
+        output = ::Libis::Format::Tool::Droid.run(file, options[:recursive])
         process_tool_output(output, result, options[:base_dir])
         output
       end
 
       def get_extension_identification(file, result, options)
-        output = ::Libis::Format::ExtensionIdentification.run(file, options[:recursive])
+        output = ::Libis::Format::Tool::ExtensionIdentification.run(file, options[:recursive])
         process_tool_output(output, result, options[:base_dir])
         output
       end
@@ -149,9 +149,9 @@ module Libis
         log_msg(result, :warn, "Error parsing XML file #{file}: #{e.message} @ #{e.backtrace.first}")
       end
 
-      def process_results(result)
+      def process_results(result, delete_output = true)
         result[:output].keys.each do |file|
-          output = result[:output].delete(file)
+          output = result[:output][file]
           file_result = result[:formats][file] = {}
           if output.empty?
             log_msg(result, :warn, "Could not identify format of '#{file}'.")
@@ -177,7 +177,7 @@ module Libis
             end
           end
         end
-        result.delete(:output)
+        result.delete(:output) if delete_output
       end
 
       def process_multiple_formats(file_result, format_matches, output)

data/lib/libis/format/tool/droid.rb

@@ -0,0 +1,108 @@
+require 'singleton'
+
+require 'tempfile'
+require 'csv'
+
+require 'libis/format/config'
+
+unless CSV::HeaderConverters.has_key?(:droid_headers)
+  CSV::HeaderConverters[:droid_headers] = lambda {|h|
+    h.encode(ConverterEncoding).downcase.strip.
+        gsub(/\W+/, "").to_sym
+  }
+end
+
+require_relative 'identification_tool'
+
+module Libis
+  module Format
+    module Tool
+
+      class Droid < Libis::Format::Tool::IdentificationTool
+
+        def run_list(filelist)
+          runner(filelist)
+        end
+
+        def run_dir(dir, recursive = true)
+          profile = profile_file_name
+          report = result_file_name
+          create_profile(dir, profile, recursive)
+          create_report(profile, report)
+          parse_report(report)
+        end
+
+        def run(file)
+          runner(file)
+        end
+
+        protected
+
+        def runner(file_or_list)
+          profile = profile_file_name
+          report = result_file_name
+          create_profile(file_or_list, profile)
+          create_report(profile, report)
+          parse_report(report)
+        end
+
+        def parse_report(report)
+          keys = [
+              :id, :parent_id, :uri, :filepath, :filename, :matchtype, :status, :filesize, :type, :extension,
+              :mod_time, :ext_mismatch, :hash, :format_count, :puid, :mimetype, :format_name, :format_version]
+          result = CSV.parse(File.readlines(report).join)
+                       .map {|a| Hash[keys.zip(a)]}
+                       .select {|a| a[:type] == 'File'}
+          # File.delete report
+          result.each do |r|
+            r.delete(:id)
+            r.delete(:parent_id)
+            r.delete(:uri)
+            r.delete(:filename)
+            r.delete(:status)
+            r.delete(:filesize)
+            r.delete(:type)
+            r.delete(:extension)
+            r.delete(:mod_time)
+            r.delete(:hash)
+            r.delete(:format_count)
+            r[:source] = :droid
+          end
+          File.delete report
+          process_output(result)
+        end
+
+        def create_report(profile, report)
+          args = [
+              '-e', report,
+              '-p', profile,
+              '-q'
+          ]
+          result = Libis::Tools::Command.run(Libis::Format::Config[:droid_path], *args)
+          raise RuntimeError, "DROID report errors: #{result[:err].join("\n")}" unless result[:status] == 0
+          File.delete profile
+        end
+
+        def create_profile(file_or_list, profile, recursive = false)
+          args = []
+          files = (file_or_list.is_a?(Array)) ? file_or_list.map(&:escape_for_string) : [file_or_list.escape_for_string]
+          files.each {|file| args << '-a' << file}
+          args << '-p' << profile << '-q'
+          args << '-R' if recursive
+          result = Libis::Tools::Command.run(Libis::Format::Config[:droid_path], *args)
+          raise RuntimeError, "DROID profile errors: #{result[:err].join("\n")}" unless result[:status] == 0
+        end
+
+        def profile_file_name
+          Tools::TempFile.name('droid', '.profile')
+        end
+
+        def result_file_name
+          Tools::TempFile.name('droid', '.csv')
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/tool/extension_identification.rb

@@ -0,0 +1,58 @@
+require_relative 'identification_tool'
+
+module Libis
+  module Format
+    module Tool
+
+      class ExtensionIdentification < Libis::Format::Tool::IdentificationTool
+
+        def run_list(filelist)
+
+          output = runner(nil, filelist)
+
+          process_output(output)
+
+        end
+
+        def run_dir(dir, recursive = true)
+
+          filelist = find_files(dir, recursive)
+
+          output = runner(nil, filelist)
+
+          process_output(output)
+
+        end
+
+        def run(file)
+
+          output = runner(file)
+
+          process_output(output)
+
+        end
+
+        protected
+
+        def runner(*args)
+
+          args.map do |file|
+            info = ::Libis::Format::TypeDatabase.ext_infos(File.extname(file)).first
+            if info
+              {
+                  filepath: file,
+                  mimetype: (info[:MIME].first rescue nil),
+                  puid: (info[:PUID].first rescue nil),
+                  matchtype: 'extension',
+                  source: :type_database
+              }
+            end
+          end.cleanup
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/tool/ffmpeg.rb

@@ -0,0 +1,43 @@
+require 'libis/tools/extend/string'
+require 'libis/tools/extend/empty'
+require 'libis/tools/command'
+
+require 'csv'
+require 'libis/format/config'
+
+module Libis
+  module Format
+    module Tool
+
+      class FFMpeg
+        include Singleton
+        include ::Libis::Tools::Logger
+
+        def self.run(source, target, options = {})
+          self.instance.run source, target, options
+        end
+
+        def run(source, target, options = {})
+          opts = []
+          opts += options[:global] unless options[:global].empty?
+          opts += options[:input] unless options[:input].empty?
+          opts << '-i' << source
+          opts += options[:filter] unless options[:filter].empty?
+          opts += options[:output] unless options[:output].empty?
+          opts << target
+          result = Libis::Tools::Command.run(Libis::Format::Config[:ffmpeg_path], *opts)
+
+          unless result[:status] == 0
+            error "FFMpeg errors: #{(result[:err] + result[:out]).join("\n")}"
+            return false
+          end
+          warn "FFMpeg warnings: #{(result[:err] + result[:out]).join("\n")}" unless result[:err].empty?
+
+          result[:out]
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/tool/fido.rb

@@ -0,0 +1,91 @@
+require 'libis/tools/extend/string'
+require 'libis/tools/command'
+
+require 'csv'
+require 'libis/format/config'
+
+require_relative 'identification_tool'
+
+module Libis
+  module Format
+    module Tool
+
+      class Fido < Libis::Format::Tool::IdentificationTool
+
+        def self.add_formats(formats_file)
+          self.instance.formats << formats_file unless self.instance.formats.include?(formats_file)
+        end
+
+        def self.del_formats(formats_file)
+          self.instance.formats.delete(formats_file)
+        end
+
+        attr_reader :formats
+
+        def run_list(filelist)
+          create_list_file(filelist) do |list_file|
+            output = runner(nil, '-input', list_file.escape_for_string)
+            process_output(output)
+          end
+        end
+
+        def run_dir(dir, recursive = true)
+          args = []
+          args << '-recurse' if recursive
+          output = runner(dir, *args)
+          process_output(output)
+        end
+
+        def run(file)
+          output = runner(file)
+          process_output(output)
+        end
+
+        protected
+
+        def initialize
+          super
+          @formats = Libis::Format::Config[:fido_formats].dup
+          bad_mimetype('application/vnd.oasis.opendocument.text')
+          bad_mimetype('application/vnd.oasis.opendocument.spreadsheet')
+        end
+
+        attr_writer :formats
+
+        def runner(filename, *args)
+          # Load custome format definitions if present
+          args << '-loadformats' << "#{formats.join(',')}" unless formats.empty?
+
+          # Workaround for Fido performance bug
+          args << '-bufsize' << '1000'
+
+          # Add filename to argument list (optional)
+          args << "#{filename.escape_for_string}" if filename
+
+          # No header output
+          args << '-q'
+
+          # Run command and capture results
+          fido = ::Libis::Tools::Command.run(Libis::Format::Config[:fido_path], *args)
+
+          # Log warning if needed
+          raise RuntimeError, "Fido errors: #{fido[:err].join("\n")}" unless fido[:err].empty?
+
+          # Parse output (CSV) text into array and return result
+          keys = [:status, :time, :puid, :format_name, :format_version, :filesize, :filepath, :mimetype, :matchtype]
+          result = CSV.parse(fido[:out].join("\n"))
+                       .map {|a| Hash[keys.zip(a)]}
+                       .select {|a| a[:status] == 'OK'}
+          result.each do |r|
+            r.delete(:time)
+            r.delete(:status)
+            r.delete(:filesize)
+            r[:source] = :fido
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/tool/file_tool.rb

@@ -0,0 +1,78 @@
+require_relative 'identification_tool'
+
+module Libis
+  module Format
+    module Tool
+
+      class FileTool < Libis::Format::Tool::IdentificationTool
+
+        def run_list(filelist)
+
+          create_list_file(filelist) do |list_file|
+
+            output = runnerIdentificationTool(nil, '--files-from', list_file)
+
+            process_output(output)
+
+          end
+
+        end
+
+        def run_dir(dir, recursive = true)
+
+          filelist = find_files(dir, recursive)
+
+          create_list_file(filelist) do |list_file|
+
+            output = runner(nil, '--files-from', list_file)
+
+            process_output(output)
+
+          end
+
+        end
+
+        def run(file)
+
+          output = runner(file)
+
+          process_output(output)
+
+        end
+
+        protected
+
+        def runner(filename, *args)
+
+          # Create new argument list
+          opts = []
+
+          # Add fixed options
+          # -L : follow symlinks
+          # --mime-type : only print MIME type
+          opts << '-L' << '--mime-type'
+
+          # Append passed arguments
+          opts += args
+
+          # Finally add the filename to process
+          opts << filename.escape_for_string if filename
+
+          # Run the UNIX file command and capture the results
+          file_tool = ::Libis::Tools::Command.run('file', *opts)
+
+          raise RuntimeError, "File command errors: #{file_tool[:err].join("\n")}" unless file_tool[:err].empty?
+
+
+          # Parse output text into array and return result
+          file_tool[:out].map do |line|
+            r = line.split(/:\s+/)
+            {filepath: r[0], mimetype: r[1], matchtype: 'magic', source: :file}
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/tool/identification_tool.rb

@@ -0,0 +1,175 @@
+require 'csv'
+require 'tmpdir'
+
+require 'singleton'
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+require 'libis/format/type_database'
+
+module Libis
+  module Format
+    module Tool
+
+        class IdentificationTool
+        include Singleton
+        include ::Libis::Tools::Logger
+
+        def self.bad_mimetype(mimetype)
+          self.instance.bad_mimetype(mimetype)
+        end
+
+        def self.run(file, recursive = false)
+          if file.is_a?(Array)
+            return run_list file
+          elsif file.is_a?(String) && File.exists?(file) && File.readable?(file)
+            if File.directory?(file)
+              return run_dir(file, recursive)
+            elsif File.file?(file)
+              return self.instance.run(file)
+            end
+          end
+          raise ArgumentError,
+                'IdentificationTool: file argument should be a path to an existing file or directory or a list of those'
+        end
+
+        def self.run_dir(file, recursive = true)
+          self.instance.run_dir file, recursive
+        end
+
+        def self.run_list(filelist)
+          self.instance.run_list filelist
+        end
+
+        protected
+
+        def create_list_file(filelist)
+          list_file = Tempfile.new(%w'file .list')
+          filelist.each do |fname|
+            list_file.write "#{fname}\n"
+          end
+          list_file.close
+          yield(list_file.path)
+        ensure
+          list_file.unlink
+        end
+
+        def find_files(dir, recurse = true)
+          args = []
+          args << '-L'
+          args << dir.escape_for_string
+          args << '-maxdepth' << '1' unless recurse
+          args << '-type' << 'f'
+          args << '-print'
+          output = ::Libis::Tools::Command.run('find', *args)
+          warn "Find command errors: #{output[:err].join("\n")}" unless output[:err].empty?
+          output[:out]
+        end
+
+        # Reformat output to make it easier to post-process and decide on the preferred format
+        #
+        # input format:
+        # [
+        #   { filepath: <filename>, mimetype: <mimetype>, matchtype: <matchtype>, ... }
+        # ]
+        #
+        # output format:
+        #   { <filename> => [<result>, ...], ... }
+        #
+        # <result> is the enchanced Hash output of the identification tool:
+        #   { mimetype: <mimetype>, puid: <puid>, matchtype: <matchtype>, score: <score>, ...}
+        #
+        def process_output(output)
+          output.reduce({}) do |results, x|
+            filepath = x.delete(:filepath)
+            results[filepath] ||= []
+            results[filepath.freeze] << annotate(x)
+            results
+          end
+        end
+
+        # Enhance the output with mimetype and score
+        def annotate(result)
+          # Enhance result with mimetype if needed
+          if bad_mimetypes.include?(result[:mimetype]) && !bad_puids.include?(result[:puid])
+            result[:mimetype] = get_mimetype(result[:puid])
+          end
+
+          # Normalize the mimetype
+          Libis::Format::TypeDatabase.normalize(result, PUID: :puid, MIME: :mimetype)
+
+          # Default score is 5
+          result[:score] = 5
+
+          # Weak detection score is 1
+          result[:score] = 1 if bad_mimetypes.include? result[:mimetype]
+
+          # freeze all strings
+          result.each {|_, v| v.freeze if v.is_a?(String)}
+
+          # Adapt score based on matchtype
+          result[:matchtype] = result[:matchtype].to_s.downcase
+          case result[:matchtype]
+
+            # Signature match increases score with 2
+            when 'signature'
+              result[:score] += 2
+            # typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(result[:puid])
+            # ext = File.extname(result[:filename])
+            # result[:score] += 1 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
+
+            # Container match increases score with 4
+            when 'container'
+              result[:score] += 4
+            # typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(result[:puid])
+            # ext = File.extname(result[:filename])
+            # result[:score] += 1 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
+
+            # Extension match is the weakest identification; score is lowered by 2 points
+            when 'extension'
+              result[:score] -= 2
+
+            # Magic code (file tool) is to be trused even less
+            when 'magic'
+              result[:score] -= 3
+
+            # Or no change otherwise
+            else
+              # do nothing
+          end
+
+          # Detecting a zip file should decrease the score as it may hide one of the many zip-based formats (e.g. epub,
+          # Office OpenXML, OpenDocument, jar, maff, svx)
+          if result[:mimetype] == 'application/zip'
+            result[:score] -= 2
+          end
+
+          # Return result enhanced with mimetype and score fields
+          result
+        end
+
+        def get_mimetype(puid)
+          ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first rescue nil
+        end
+
+        def get_puid(mimetype)
+          ::Libis::Format::TypeDatabase.mime_infos(mimetype).first[:PUID].first rescue nil
+        end
+
+        attr_accessor :bad_mimetypes, :bad_puids
+
+        def initialize
+          @bad_mimetypes = [nil, '', 'None', 'application/octet-stream']
+          @bad_puids = [nil, 'fmt/unknown']
+        end
+
+        def bad_mimetype(mimetype)
+          @bad_mimetypes << mimetype
+        end
+      end
+
+    end
+  end
+end

data/lib/libis/format/tool/office_to_pdf.rb

@@ -0,0 +1,54 @@
+require 'fileutils'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+    module Tool
+
+      class OfficeToPdf
+        include ::Libis::Tools::Logger
+
+        def self.run(source, target, options = {})
+          self.new.run source, target, options
+        end
+
+        def run(source, target, options = {})
+          workdir = '/...'
+          workdir = Dir.tmpdir unless Dir.exist? workdir
+
+          workdir = File.join(workdir, rand(1000000).to_s)
+          FileUtils.mkpath(workdir)
+
+          src_file = File.join(workdir, File.basename(source))
+          FileUtils.symlink source, src_file
+
+          tgt_file = File.join(workdir, File.basename(source, '.*') + '.pdf')
+
+          export_filter = options[:export_filter] || 'pdf'
+
+          result = Libis::Tools::Command.run(
+              Libis::Format::Config[:soffice_path], '--headless',
+              '--convert-to', export_filter,
+              '--outdir', workdir, src_file
+          )
+
+          unless result[:status] == 0
+            warn "PdfConvert errors: #{(result[:err] + result[:out]).join("\n")}"
+            return false
+          end
+
+          FileUtils.copy tgt_file, target, preserve: true
+          FileUtils.rmtree workdir
+
+          result[:out]
+        end
+      end
+
+    end
+  end
+end