libis-format 0.9.32 → 0.9.33

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 11f5b2cb05e4748b97fef65f805e8222e9857354
-  data.tar.gz: 49c667a039c6174b41aaabbb2800955989884fa0
+  metadata.gz: 5a13d9df8f5e85ccc58758e46e556066d16935b0
+  data.tar.gz: 3a80e5cfc6d7ea70f4c0091420012446ec801b74
 SHA512:
-  metadata.gz: 9bbb1f1f21742ece1b4e34ce22a295327436b0c59b996a018bfba9c828377e5a79a1cefb9f4c2a01e080dbdc15e8c6586dc15b5ab66eca936550c840eeba380d
-  data.tar.gz: 9214406a8a4b638801d8c4258e14401a3b65fb1283299e76664115148514476f3738832be3718b7d124583a3ea21e53b9a949f7557c7a63d42203f97e4381b14
+  metadata.gz: c6a45a701e5ec6e07ebf763ee2fe3d97eb327fdccc85ca9aaca3e006e5dee3c1c058081c9a33c311aa72a7f2272661986799ab4a3f731f35e6ed5053ac1a25b3
+  data.tar.gz: 6f7b02f7ab9a9fad169920f7b2dc04fdbc63a33fb8d5e70e458de9055fa1427acf453f88ca3937883d721a15d636af53ce943a6feeb68e980e0d28a04970222d

data/data/types.yml

@@ -10,53 +10,62 @@
 IMAGE:
   TIFF:
     NAME: Tagged Image File Format (TIFF)
-    MIME: image/tiff
+    MIME: image/tiff image/x-tiff image/tif image/x-tif application/tiff application/x-tiff application/tif application/x-tif
     PUID: fmt/353 fmt/154 fmt/153 fmt/156 fmt/155 fmt/152 x-fmt/399 x-fmt/388 x-fmt/387 fmt/202
     EXTENSIONS: tif,TIF,tiff,tifx,dng,nef
 
   JP2:
     NAME: Joint Photographic Experts Group 2000 (JPEG 2000)
-    MIME: image/jp2
-    PUID: fmt/406
+    MIME: image/jp2, image/jpeg2000, image/jpeg2000-image, image/x-jpeg2000-image, image/jpx
+    PUID: fmt/463, fmt/151, x-fmt/392
     EXTENSIONS: jp2
 
   JPG:
     NAME: Joint Photographic Experts Group (JPEG)
-    MIME: image/jpeg
+    MIME: image/jpeg, image/jpg, image/jp_, application/jpg, application/x-jpg, image/pjpeg, image/pipeg, image/vnd.swiftview-jpeg
     PUID: fmt/42 fmt/43 fmt/44 fmt/41 x-fmt/398 x-fmt/390 x-fmt/391 fmt/645
     EXTENSIONS: jpg,jpe,jpeg
 
   PNG:
     NAME: Portable Network Graphics (PNG)
-    MIME: image/png
+    MIME: image/png, application/png, application/x-png
+    PUID: fmt/11, fmt/12, fmt/13,
     EXTENSIONS: png
 
+  APNG:
+    NAME: Animate Portable Network Graphics
+    MIME: image/vnd.mozilla.apng
+    PUID: fmt/935
+    EXTENSIONS: apng, png
+
   BMP:
     NAME: Device Independent Bitmap (DIP/BMP)
-    MIME: image/bmp,image/x-ms-bmp
+    MIME: image/bmp, image/x-bmp, image/x-bitmap, image/x-xbitmap, image/x-win-bitmap, image/x-windows-bmp, image/ms-bmp, image/x-ms-bmp, application/bmp, application/x-bmp, application/x-win-bitmap
+    PUID: fmt/115, fmt/118, fmt/119, fmt/114, fmt/116, fmt/117, x-fmt/270
     EXTENSIONS: bmp
 
   GIF:
     NAME: Graphics Interchange Format (GIF)
     MIME: image/gif
+    PUID: fmt/3, fmt/4
     EXTENSIONS: gif
 
   PBM:
     NAME: Portable Bitmap Format (PBM)
-    PUID: fmt/409
-    MIME: image/x‑portable‑bitmap
+    MIME: image/x-portable-bitmap, image/pbm, image/x-pbm
+    PUID: fmt/409, x-fmt/164
     EXTENSIONS: pbm
 
   PGM:
     NAME: Portable GrayMap Format (PGM)
-    PUID: fmt/406
-    MIME: image/x‑portable‑graymap
+    PUID: fmt/406, fmt/407
+    MIME: image/x-portable-graymap, image/pgm, image/x-pgm
     EXTENSIONS: pgm
 
   PPM:
     NAME: Portable Pixel Map (PPM)
-    PUID: fmt/408
-    MIME: image/x‑portable‑pixmap
+    PUID: fmt/408, x-fmt/178
+    MIME: image/x-portable-pixmap, application/ppm, application/x-ppm, image/ppm, image/x-ppm
     EXTENSIONS: ppm
 
 AUDIO:
@@ -133,7 +142,7 @@ TEXT:
   RTF:
     NAME: Rich Text Format (RTF)
     PUID: fmt/45
-    MIME: text/rtf application/rtf
+    MIME: application/rtf text/rtf
     EXTENSIONS: rtf
 
   HTML:
@@ -143,8 +152,8 @@ TEXT:
 
   MSDOC:
     NAME: Microsoft Word Document (DOC)
-    PUID: fmt/609 fmt/39 x-fmt/2 x-fmt/129 x-fmt/273 x-fmt/274 x-fmt/275 x-fmt/276 fmt/37 fmt/38 fmt/40
-    MIME: application/vnd.ms-word application/msword
+    PUID: fmt/609 fmt/39 x-fmt/273 x-fmt/274 x-fmt/275 x-fmt/276 fmt/37 fmt/38 fmt/40 fmt/754
+    MIME: application/msword application/doc appl/text application/vnd.msword application/vnd.ms-word application/winword application/word application/x-msw6 application/x-msword
     EXTENSIONS: doc
 
   MSDOCX:
@@ -171,9 +180,14 @@ TEXT:
     MIME: application/vnd.wordperfect
     EXTENSIONS: wpd
 
+  OO_WRITER:
+    NAME: OpenDocument Text
+    MIME: application/vnd.oasis.opendocument.text, application/x-vnd.oasis.opendocument.text
+    PUID: fmt/136, fmt/290, fmt/291
+
   XML:
     NAME: Extensible Markup Language (XML)
-    MIME: text/xml application/xml
+    MIME: text/xml application/xml application/x-xml
     PUID: fmt/101
     EXTENSIONS: xml
 

data/lib/libis/format/config.rb

@@ -13,24 +13,13 @@ module Libis
     Config[:j2kdriver] = 'j2kdriver'
     Config[:soffice_path] = 'soffice'
     Config[:ghostscript_path] = 'gs'
-    # Config[:pdfa_path] =
-    #     File.absolute_path(
-    #         File.join(
-    #             File.dirname(__FILE__), '..', '..', '..', 'tools', 'pdf', 'pdfa', 'pdfa'
-    #         )
-    #     )
-    Config[:droid_path] =
-        File.absolute_path(
-            File.join(
-                File.dirname(__FILE__), '..', '..', '..', 'tools', 'droid', OS.windows? ? 'droid.bat' : 'droid.sh'
-            )
-        )
-    Config[:fido_path] =
-        File.absolute_path(
-            File.join(
-                File.dirname(__FILE__), '..', '..', '..', 'tools', 'fido', OS.windows? ? 'fido.bat' : 'fido.sh'
-            )
-        )
+    Config[:droid_path] = '/opt/droid/droid.sh'
+    Config[:fido_path] = '/usr/local/bin/fido'
+    data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
+    Config[:fido_formats] = [(File.join(data_dir, 'lias_formats.xml'))]
+    # noinspection RubyStringKeysInHashInspection
+    Config[:xml_validations] = [['archive/ead', File.join(data_dir, 'ead.xsd')]]
+    Config[:type_database] = File.join(data_dir, 'types.yml')
 
   end
 end

data/lib/libis/format/converter/image_converter.rb

@@ -8,6 +8,12 @@ require 'fileutils'
 
 MiniMagick.logger.level = ::Logger::ERROR
 
+MiniMagick.configure do |config|
+  # config.cli = :graphicsmagick
+  config.validate_on_create = false
+  config.validate_on_write = false
+end
+
 module Libis
   module Format
     module Converter

data/lib/libis/format/droid.rb

@@ -3,45 +3,102 @@ require 'singleton'
 require 'tempfile'
 require 'csv'
 
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
-
 require 'libis/format/config'
 
+unless CSV::HeaderConverters.has_key?(:droid_headers)
+  CSV::HeaderConverters[:droid_headers] = lambda {|h|
+    h.encode(ConverterEncoding).downcase.strip.
+        gsub(/\W+/, "").to_sym
+  }
+end
+
+require_relative 'identification_tool'
+
 module Libis
   module Format
 
-    class Droid
-      include Singleton
-      include ::Libis::Tools::Logger
+    class Droid < Libis::Format::IdentificationTool
+
+      def run_list(filelist)
+        runner(filelist)
+      end
 
-      def self.run(file)
-        self.instance.run file
+      def run_dir(dir, recursive = true)
+        profile = profile_file_name
+        report = result_file_name
+        create_profile(dir, profile, recursive)
+        create_report(profile, report)
+        parse_report(report)
       end
 
       def run(file)
-        profile = File.join Dir.tmpdir, Dir::Tmpname.make_tmpname(%w'droid .profile', nil)
-        report = File.join Dir.tmpdir, Dir::Tmpname.make_tmpname(%w'droid .csv', nil)
-        result = Libis::Tools::Command.run(
-            Libis::Format::Config[:droid_path],
-            '-a', file.escape_for_string,
-            '-p', profile,
-            '-q',
-        )
-        warn "DROID profile errors: #{result[:err].join("\n")}" unless result[:status] == 0
-        result = Libis::Tools::Command.run(
-            Libis::Format::Config[:droid_path],
+        runner(file)
+      end
+
+      protected
+
+      def runner(file_or_list)
+        profile = profile_file_name
+        report = result_file_name
+        create_profile(file_or_list, profile)
+        create_report(profile, report)
+        parse_report(report)
+      end
+
+      def parse_report(report)
+        keys = [
+            :id, :parent_id, :uri, :filepath, :filename, :matchtype, :status, :filesize, :type, :extension,
+            :mod_time, :ext_mismatch, :hash, :format_count, :puid, :mimetype, :format_name, :format_version]
+        result = CSV.parse(File.readlines(report).join)
+                     .map {|a| Hash[keys.zip(a)]}
+                     .select {|a| a[:type] == 'File'}
+        # File.delete report
+        result.each do |r|
+          r.delete(:id)
+          r.delete(:parent_id)
+          r.delete(:uri)
+          r.delete(:filename)
+          r.delete(:status)
+          r.delete(:filesize)
+          r.delete(:type)
+          r.delete(:extension)
+          r.delete(:mod_time)
+          r.delete(:hash)
+          r.delete(:format_count)
+          r[:source] = :droid
+        end
+        process_output(result)
+      end
+
+      def create_report(profile, report)
+        args = [
             '-e', report,
             '-p', profile,
             '-q'
-        )
-        warn "DROID report errors: #{result[:err].join("\n")}" unless result[:status] == 0
+        ]
+        result = Libis::Tools::Command.run(Libis::Format::Config[:droid_path], *args)
+        raise RuntimeError, "DROID report errors: #{result[:err].join("\n")}" unless result[:status] == 0
         File.delete profile
-        result = CSV.read(report , headers: true, header_converters: [:downcase, :symbol])
-        File.delete report
-        result.map{|r|r.to_hash}
       end
+
+      def create_profile(file_or_list, profile, recursive = false)
+        args = []
+        files = (file_or_list.is_a?(Array)) ? file_or_list.map(&:escape_for_string) : [file_or_list.escape_for_string]
+        files.each { |file| args << '-a' << file}
+        args << '-p' << profile << '-q'
+        args << '-R' if recursive
+        result = Libis::Tools::Command.run(Libis::Format::Config[:droid_path], *args)
+        raise RuntimeError, "DROID profile errors: #{result[:err].join("\n")}" unless result[:status] == 0
+      end
+
+      def profile_file_name
+        File.join Dir.tmpdir, Dir::Tmpname.make_tmpname(%w'droid .profile', nil)
+      end
+
+      def result_file_name
+        File.join Dir.tmpdir, Dir::Tmpname.make_tmpname(%w'droid .csv', nil)
+      end
+
     end
 
   end

data/lib/libis/format/extension_identification.rb

@@ -0,0 +1,55 @@
+require_relative 'identification_tool'
+
+module Libis
+  module Format
+
+    class ExtensionIdentification < Libis::Format::IdentificationTool
+
+      def run_list(filelist)
+
+        output = runner(nil, filelist)
+
+        process_output(output)
+
+      end
+
+      def run_dir(dir, recursive = true)
+
+        filelist = find_files(dir, recursive)
+
+        output = runner(nil, filelist)
+
+        process_output(output)
+
+      end
+
+      def run(file)
+
+        output = runner(file)
+
+        process_output(output)
+
+      end
+
+      protected
+
+      def runner(*args)
+
+        args.map do |file|
+          info = ::Libis::Format::TypeDatabase.ext_infos(File.extname(file)).first
+          if info
+            {
+                filepath: file,
+                mimetype: (info[:MIME].first rescue nil),
+                puid: (info[:PUID].first rescue nil),
+                matchtype: 'extension',
+                source: :type_database
+            }
+          end
+        end.cleanup
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/fido.rb

@@ -1,101 +1,86 @@
-require 'csv'
-
-require 'singleton'
 require 'libis/tools/extend/string'
-require 'libis/tools/logger'
 require 'libis/tools/command'
 
+require 'csv'
 require 'libis/format/config'
-require 'libis/format/type_database'
+
+require_relative 'identification_tool'
 
 module Libis
   module Format
 
-    class Fido
-      include Singleton
-      include ::Libis::Tools::Logger
+    class Fido < Libis::Format::IdentificationTool
 
-      BAD_MIMETYPES = [nil, '', 'None', 'application/octet-stream']
-
-      def self.run(file, formats = nil)
-        self.instance.run file, formats
+      def self.add_formats(formats_file)
+        self.instance.formats << formats_file unless self.instance.formats.include?(formats_file)
       end
 
-      def run(file, xtra_formats = nil)
+      def self.del_formats(formats_file)
+        self.instance.formats.delete(formats_file)
+      end
 
-        fido_results = []
+      attr_reader :formats
 
-        fmt_list = formats.dup
-        case xtra_formats
-          when Array
-            fmt_list += xtra_formats
-          when String
-            fmt_list << xtra_formats
-          else
-            # do nothing
+      def run_list(filelist)
+        create_list_file(filelist) do |list_file|
+          output = runner(nil, '-input', list_file.escape_for_string)
+          process_output(output)
         end
+      end
 
+      def run_dir(dir, recursive = true)
         args = []
-        args << '-loadformats' << "#{fmt_list.join(',')}" unless fmt_list.empty?
-        args << "#{file.escape_for_string}"
-        fido = ::Libis::Tools::Command.run(Libis::Format::Config[:fido_path], *args)
-        warn "Fido errors: #{fido[:err].join("\n")}" unless fido[:err].empty?
-
-        keys = [:status, :time, :puid, :format_name, :signature_name, :filesize, :filename, :mimetype, :matchtype]
-        fido_output = CSV.parse(fido[:out].join("\n")).map { |a| Hash[keys.zip(a)] }
-
-        fido_output.each do |x|
-          if x[:status] == 'OK'
-            x[:mimetype] = get_mimetype(x[:puid]) if x[:mimetype] == 'None'
-            next if BAD_MIMETYPES.include? x[:mimetype]
-            x[:score] = 5
-            case x[:matchtype]
-              when 'signature'
-                x[:score] += 5
-              when 'container'
-                typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(x[:puid])
-                ext = File.extname(file)
-                x[:score] += 2 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
-              else
-                # do nothing
-            end
-            fido_results << x
-          end
-        end
-
-        fido_results = fido_results.inject({}) do |result, value|
-          result[value[:score]] ||= []
-          result[value[:score]] << value
-          result
-        end
+        args << '-recurse' if recursive
+        output = runner(dir, *args)
+        process_output(output)
+      end
 
-        max_score = fido_results.keys.max
+      def run(file)
+        output = runner(file)
+        process_output(output)
+      end
 
-        # Only if we find a single hit of type 'signature' or 'container', we are confident enough to return a result
-        return {} unless max_score and max_score >= 5 && fido_results[max_score].size == 1
+      protected
 
-        fido_results[max_score].first
+      def initialize
+        super
+        @formats = Libis::Format::Config[:fido_formats].dup
+        bad_mimetype('application/vnd.oasis.opendocument.text')
+        bad_mimetype('application/vnd.oasis.opendocument.spreadsheet')
       end
 
-      def self.add_format(f)
-        instance.formats << f
-      end
+      attr_writer :formats
 
-      def self.formats
-        instance.formats
-      end
+      def runner(filename, *args)
+        # Load custome format definitions if present
+        args << '-loadformats' << "#{formats.join(',')}" unless formats.empty?
 
-      protected
+        # Workaround for Fido performance bug
+        args << '-bufsize' << '1000'
 
-      attr_reader :formats
+        # Add filename to argument list (optional)
+        args << "#{filename.escape_for_string}" if filename
 
-      def initialize
-        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
-        @formats = [(File.join(data_dir, 'lias_formats.xml'))]
-      end
+        # No header output
+        args << '-q'
 
-      def get_mimetype(puid)
-        ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first rescue nil
+        # Run command and capture results
+        fido = ::Libis::Tools::Command.run(Libis::Format::Config[:fido_path], *args)
+
+        # Log warning if needed
+        raise RuntimeError, "Fido errors: #{fido[:err].join("\n")}" unless fido[:err].empty?
+
+        # Parse output (CSV) text into array and return result
+        keys = [:status, :time, :puid, :format_name, :format_version, :filesize, :filepath, :mimetype, :matchtype]
+        result = CSV.parse(fido[:out].join("\n"))
+                     .map {|a| Hash[keys.zip(a)]}
+                     .select {|a| a[:status] == 'OK'}
+        result.each do |r|
+          r.delete(:time)
+          r.delete(:status)
+          r.delete(:filesize)
+          r[:source] = :fido
+        end
       end
 
     end

data/lib/libis/format/file_tool.rb

@@ -0,0 +1,76 @@
+require_relative 'identification_tool'
+
+module Libis
+  module Format
+
+    class FileTool < Libis::Format::IdentificationTool
+
+      def run_list(filelist)
+
+        create_list_file(filelist) do |list_file|
+
+          output = runner(nil, '--files-from', list_file)
+
+          process_output(output)
+
+        end
+
+      end
+
+      def run_dir(dir, recursive = true)
+
+        filelist = find_files(dir, recursive)
+
+        create_list_file(filelist) do |list_file|
+
+          output = runner(nil, '--files-from', list_file)
+
+          process_output(output)
+
+        end
+
+      end
+
+      def run(file)
+
+        output = runner(file)
+
+        process_output(output)
+
+      end
+
+      protected
+
+      def runner(filename, *args)
+
+        # Create new argument list
+        opts = []
+
+        # Add fixed options
+        # -L : follow symlinks
+        # --mime-type : only print MIME type
+        opts << '-L' << '--mime-type'
+
+        # Append passed arguments
+        opts += args
+
+        # Finally add the filename to process
+        opts << filename.escape_for_string if filename
+
+        # Run the UNIX file command and capture the results
+        file_tool = ::Libis::Tools::Command.run('file', *opts)
+
+        raise RuntimeError, "File command errors: #{file_tool[:err].join("\n")}" unless file_tool[:err].empty?
+
+
+        # Parse output text into array and return result
+        file_tool[:out].map do |line|
+          r = line.split(/:\s+/)
+          {filepath: r[0], mimetype: r[1], matchtype: 'magic', source: :file}
+        end
+      end
+
+    end
+
+  end
+end