libis-format 0.9.5-java

data/lib/libis/format/identifier.rb

@@ -0,0 +1,189 @@
+# encoding: utf-8
+
+require 'singleton'
+
+require 'libis-tools'
+require 'libis/tools/extend/string'
+require 'libis/tools/extend/empty'
+
+require 'libis/format/type_database'
+
+require_relative 'fido'
+require_relative 'droid'
+
+module Libis
+  module Format
+
+    class Identifier
+      include ::Libis::Tools::Logger
+      include Singleton
+
+      RETRY_MIMETYPES = %w(application/zip) + ::Libis::Format::Fido::BAD_MIMETYPES
+      FIDO_FAILURES = %w(application/vnd.oasis.opendocument.text application/vnd.oasis.opendocument.spreadsheet)
+
+      attr_reader :xml_validations
+
+      protected
+
+      def initialize
+        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
+        @fido_formats = [(File.join(data_dir, 'lias_formats.xml'))]
+        # noinspection RubyStringKeysInHashInspection
+        @xml_validations = {'archive/ead' => File.join(data_dir, 'ead.xsd')}
+      end
+
+      def result_ok?(result, who_is_asking = nil)
+        result = ::Libis::Format::TypeDatabase.enrich(result, PUID: :puid, MIME: :mimetype)
+        return false if result.empty?
+        return true unless result[:TYPE].empty?
+        return false if RETRY_MIMETYPES.include? result[:mimetype]
+        return false if FIDO_FAILURES.include? result[:mimetype] and who_is_asking == :DROID
+        !(result[:mimetype].empty? and result[:puid].empty?)
+      end
+
+      def get_puid(mimetype)
+        ::Libis::Format::TypeDatabase.mime_infos(mimetype).first[:PUID].first rescue nil
+      end
+
+      public
+
+      def self.add_fido_format(f)
+        ::Libis::Format::Fido.add_format f
+      end
+
+      def self.add_xml_validation(mimetype, xsd_file)
+        instance.xml_validations[mimetype] = xsd_file
+      end
+
+      def self.xml_validations
+        instance.xml_validations
+      end
+
+      def self.get(file_path, options = nil)
+        instance.get file_path, options
+      end
+
+      def get(file, options = nil)
+
+        unless File.exists? file
+          error 'File %s cannot be found.', file
+          return nil
+        end
+        if File.directory? file
+          error '%s is a directory.', file
+          return nil
+        end
+
+        options ||= {}
+
+        result = { messages: [] }
+
+        # use FIDO
+        # Note: FIDO does not always do a good job, mainly due to lacking container inspection.
+        # FIDO misses should be registered in
+        result = get_fido_identification(file, result, options[:formats]) unless options[:droid]
+
+        # use DROID
+        result = get_droid_identification file, result
+
+        # use FILE
+        result = get_file_identification(file, result)
+
+        # Try file extension
+        result = get_extension_identification(file, result)
+
+        # determine XML type. Add custom types at runtime with
+        # Libis::Tools::Format::Identifier.add_xml_validation('my_type', '/path/to/my_type.xsd')
+        result = validate_against_xml_schema(file, result)
+
+        result[:mimetype] ?
+            log_msg(result, :info, "Identification of '#{file}': '#{result}'") :
+            log_msg(result, :warn, "Could not identify MIME type of '#{file}'")
+      end
+
+      def get_fido_identification(file, result = {}, xtra_formats = nil)
+        return result if result_ok? result
+
+        fido_result = ::Libis::Format::Fido.run(file, xtra_formats)
+
+        return result unless fido_result.is_a? Hash
+
+        result.merge! fido_result
+        result[:method] = 'fido'
+
+        log_msg(result, :debug, "Fido MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})")
+      end
+
+      def get_droid_identification(file, result = {})
+        return result if result_ok? result, :DROID
+        droid_output = ::Libis::Format::Droid.run file
+        result[:messages] << [:debug, "DROID: #{droid_output}"]
+        warn 'Droid found multiple matches; using first match only' if droid_output.size > 1
+        result.clear
+        droid_output = droid_output.first
+        result[:mimetype] = droid_output[:mime_type].to_s.split(/[\s,]+/).find {|x| x =~ /.*\/.*/}
+        result[:matchtype] = droid_output[:method]
+        result[:puid] = droid_output[:puid]
+        result[:format_name] = droid_output[:format_name]
+        result[:format_version] = droid_output[:format_version]
+        result[:method] = 'droid'
+
+        log_msg(result, :debug, "Droid MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})")
+      end
+
+      def get_file_identification(file, result = nil)
+        return result if result_ok? result
+        begin
+          output = ::Libis::Tools::Command.run('file', '-b', '--mime-type', "\"#{file.escape_for_string}\"")[:err]
+          mimetype = output.strip.split
+          if mimetype
+            log_msg(result, :debug, "File result: '#{mimetype}'")
+            result[:mimetype] = mimetype
+            result[:puid] = get_puid(mimetype)
+          end
+          result[:method] = 'file'
+        rescue Exception
+          # ignored
+        end
+        result
+      end
+
+      def get_extension_identification(file, result = nil)
+        return result if result_ok? result
+        info = ::Libis::Format::TypeDatabase.ext_infos(File.extname(file)).first
+        log_msg result, :debug, "File extension info: #{info}"
+        if info
+          result[:mimetype] = info[:MIME].first rescue nil
+          result[:puid] = info[:PUID].first rescue nil
+        end
+        result[:method] = 'extension'
+        result
+      end
+
+      def validate_against_xml_schema(file, result)
+        return result unless result[:mimetype] =~ /^(text|application)\/xml$/
+        doc = ::Libis::Tools::XmlDocument.open file
+        xml_validations.each do |mime, xsd_file|
+          next unless xsd_file
+          if doc.validates_against?(xsd_file)
+            log_msg result, :debug, "XML file validated against XML Schema: #{xsd_file}"
+            result[:mimetype] = mime
+            result[:puid] = nil
+            result = ::Libis::Format::TypeDatabase.enrich(result, PUID: :puid, MIME: :mimetype)
+          end
+        end
+        result
+      end
+
+      private
+
+      def log_msg(result, severity, text)
+        return {} unless result.is_a?(Hash)
+        (result[:messages] ||= []) << [severity, text]
+        result
+      end
+
+    end
+
+  end
+end

data/lib/libis/format/office_to_pdf.rb

@@ -0,0 +1,52 @@
+require 'fileutils'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+
+    class OfficeToPdf
+      include ::Libis::Tools::Logger
+
+      def self.run(source, target, options = {})
+        self.new.run source, target, options
+      end
+
+      def run(source, target, options = {})
+        workdir = '/...'
+        workdir = Dir.tmpdir unless Dir.exist? workdir
+
+        workdir = File.join(workdir, rand(1000000).to_s)
+        FileUtils.mkpath(workdir)
+
+        src_file = File.join(workdir, File.basename(source))
+        FileUtils.link source, src_file
+
+        tgt_file = File.join(workdir, File.basename(source, '.*') + '.pdf')
+
+        export_filter = options[:export_filter] || 'pdf'
+
+        result = Libis::Tools::Command.run(
+            Libis::Format::Config[:soffice_path], '--headless',
+            '--convert-to', export_filter,
+            '--outdir', workdir, src_file
+        )
+
+        unless result[:status] == 0
+          warn "PdfConvert errors: #{(result[:err] + result[:out]).join("\n")}"
+          return false
+        end
+
+        FileUtils.copy tgt_file, target, preserve: true
+        FileUtils.rmtree workdir
+
+        result[:out]
+      end
+    end
+
+  end
+end

data/lib/libis/format/pdf_copy.rb

@@ -0,0 +1,40 @@
+require 'os'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+
+    class PdfCopy
+      include ::Libis::Tools::Logger
+
+      def self.run(source, target, options = [])
+        self.new.run source, target, options
+      end
+
+      def run(source, target, options = [])
+        tool_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'tools'))
+        jar_file = File.join(tool_dir, 'PdfTool.jar')
+
+        if OS.java?
+          # TODO: import library and execute in current VM. For now do exactly as in MRI.
+        end
+
+        Libis::Tools::Command.run(
+            Libis::Format::Config[:java_path],
+            '-cp', jar_file,
+            'CopyPdf',
+            '--file_input', source,
+            '--file_output', target,
+            *options
+        )
+
+      end
+    end
+
+  end
+end

data/lib/libis/format/pdf_merge.rb

@@ -0,0 +1,41 @@
+require 'os'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+
+    class PdfMerge
+      include ::Libis::Tools::Logger
+
+      def self.run(source, target, options = [])
+        self.new.run source, target, options
+      end
+
+      def run(source, target, options = [])
+        tool_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'tools'))
+        jar_file = File.join(tool_dir, 'PdfTool.jar')
+        source = [source] unless source.is_a?(Array)
+
+        if OS.java?
+          # TODO: import library and execute in current VM. For now do exactly as in MRI.
+        end
+
+        Libis::Tools::Command.run(
+            Libis::Format::Config[:java_path],
+            '-cp', jar_file,
+            'MergePdf',
+            '--file_output', target,
+            *options,
+            *source,
+        )
+
+      end
+    end
+
+  end
+end

data/lib/libis/format/pdf_split.rb

@@ -0,0 +1,39 @@
+require 'os'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+
+    class PdfSplit
+      include ::Libis::Tools::Logger
+
+      def self.run(source, target, options = [])
+        self.new.run source, target, options
+      end
+
+      def run(source, target, options = [])
+        tool_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'tools'))
+        jar_file = File.join(tool_dir, 'PdfTool.jar')
+        if OS.java?
+          # TODO: import library and execute in current VM. For now do exactly as in MRI.
+        end
+
+        Libis::Tools::Command.run(
+            Libis::Format::Config[:java_path],
+            '-cp', jar_file,
+            'SplitPdf',
+            '--file_input', source,
+            '--file_output', target,
+            *options
+        )
+
+      end
+    end
+
+  end
+end

data/lib/libis/format/pdf_to_pdfa.rb

@@ -0,0 +1,76 @@
+require 'tempfile'
+require 'csv'
+require 'fileutils'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format'
+
+module Libis
+  module Format
+
+    class PdfToPdfa
+      include ::Libis::Tools::Logger
+
+      def self.run(source, target = nil, options = {})
+        self.new.run source, target, options
+      end
+
+      def run(source, target = nil, options = nil)
+
+        target ||= File.join(Dir.tmpdir, Dir::Tmpname.make_tmpname([File.basename(source, '.*'), '.pdf']))
+
+        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
+
+        icc_info = icc_options(options[:colorspace])
+
+        icc_file = File.join(Dir.tmpdir, "#{icc_info[:icc_name]}#{Random.new.bytes(12).unpack('H*').first}.icc")
+        FileUtils.cp(File.join(data_dir, "#{icc_info[:icc_name]}.icc"), icc_file)
+
+        def_filename = File.join(Dir.tmpdir, "PDFA_def_#{Random.new.bytes(12).unpack('H*').first}.ps")
+        File.open(def_filename, 'w') do |f|
+          f.puts File.read(File.join(Libis::Format::DATA_DIR, 'PDFA_def.ps')).
+                     gsub('[** Fill in ICC profile location **]', icc_file).
+                     gsub('[** Fill in ICC reference name **]', icc_info[:icc_ref])
+        end
+
+        result = Libis::Tools::Command.run(
+            Libis::Format::Config[:ghostscript_path],
+            '-dBATCH', '-dNOPAUSE', '-dNOOUTERSAVE',
+            '-sColorConversionStrategy=/UseDeviceIndependentColor',
+            "-sProcessColorModel=#{icc_info[:device]}",
+            '-sDEVICE=pdfwrite', '-dPDFA', '-dPDFACompatibilityPolicy=1',
+            "-sOutputICCProfile=#{icc_file}",
+            '-o', File.absolute_path(target),
+            def_filename,
+            source
+        )
+
+        FileUtils.rm [icc_file, def_filename].compact, force: true
+
+        unless PdfaValidator.run(target)
+          result[:status] = -999
+          result[:err] << 'Failed to validate generated PDF/A file.'
+        end
+
+        result
+      end
+
+
+      private
+
+      def icc_options(colorspace)
+        case colorspace.to_s.downcase
+          when 'cmyk'
+            {icc_name: 'ISOcoated_v2_eci', icc_ref: 'FOGRA39L', device: 'DeviceCMYK'}
+          else
+            {icc_name: 'eciRGB_v2', icc_ref: 'sRGB', device: 'DeviceRGB'}
+        end
+      end
+
+    end
+
+  end
+end

data/lib/libis/format/pdfa_validator.rb

@@ -0,0 +1,61 @@
+require 'fileutils'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+
+    class PdfaValidator
+      include ::Libis::Tools::Logger
+
+      def self.run(source)
+        self.new.run source
+      end
+
+      def run(source)
+
+        src_file = File.absolute_path(source)
+
+        if (pdfa = Libis::Format::Config[:pdfa_path])
+          # Keep it clean: tool generates fontconfig/ cache dir in current working dir
+          previous_wd = Dir.getwd
+          Dir.chdir(Dir.tmpdir)
+
+          result = Libis::Tools::Command.run(
+              pdfa,
+              '--noxml',
+              '--level', 'B',
+              '--verb', '0',
+              src_file
+          )
+
+          Dir.chdir(previous_wd)
+
+          unless result[:out].any? { |line| line =~ /^VLD-\[PASS\]/ }
+            warn "Validator failed to validate the PDF file '%s' against PDF/A-1B constraints:\n%s", source,
+                 result[:out].join("\n")
+            return false
+          end
+        else
+          jar = File.join(ROOT_DIR, 'tools', 'pdfbox', 'preflight-app-1.8.10.jar')
+          result = Libis::Tools::Command.run(
+              Libis::Format::Config[:java_path],
+              '-jar', jar,
+              src_file
+          )
+          unless result[:status] == 0
+            warn "Validator failed to validate the PDF file '%s' against PDF/A-1B constraints:\n%s", source,
+                 result[:out].join("\n")
+            return false
+          end
+        end
+        true
+      end
+    end
+
+  end
+end

data/lib/libis/format/type_database.rb

@@ -0,0 +1,170 @@
+# coding: utf-8
+
+require 'singleton'
+require 'yaml'
+
+require 'backports/rails/hash'
+require 'libis/tools/logger'
+require 'libis/tools/extend/string'
+
+module Libis
+  module Format
+
+    class TypeDatabase
+      include Singleton
+      include ::Libis::Tools::Logger
+
+      def self.typeinfo(t)
+        self.instance.types[t.to_sym] || {}
+      end
+
+      def self.enrich(info, map_keys = {})
+        return {} unless info.is_a? Hash
+        mapper = Hash.new {|hash,key| hash[key] = key}
+        mapper.merge! map_keys
+        unless (puid = info[mapper[:PUID]]).blank?
+          info[mapper[:TYPE]] ||= self.puid_infos(puid).first[:TYPE] rescue nil
+        end
+        unless (mime = info[mapper[:MIME]]).blank?
+          info[mapper[:TYPE]] ||= self.mime_infos(mime).first[:TYPE] rescue nil
+        end
+        unless (type_name = info[mapper[:TYPE]]).nil?
+          info[mapper[:MIME]] = self.type_mimetypes(type_name).first if info[mapper[:MIME]].blank?
+          info[mapper[:PUID]] = self.type_puids(type_name).first if info[mapper[:PUID]].blank?
+          info[mapper[:EXTENSIONS]] = self.type_extentions(type_name)
+          info[mapper[:GROUP]] = self.type_group(type_name)
+        end
+        info
+      end
+
+      def self.type_group(t)
+        typeinfo(t)[:GROUP]
+      end
+
+      def self.type_mimetypes(t)
+        typeinfo(t)[:MIME] || []
+      end
+
+      def self.type_puids(t)
+        typeinfo(t)[:PUID] || []
+      end
+
+      def self.type_extentions(t)
+        typeinfo(t)[:EXTENSIONS] || []
+      end
+
+      def self.group_types(group)
+        self.instance.types.select do |_, v|
+          v[:GROUP] == group.to_sym
+        end.keys
+      end
+
+      def self.puid_infos(puid)
+        self.instance.types.select do |_, v|
+          v[:PUID].include? puid rescue false
+        end.values
+      end
+
+      def self.puid_types(puid)
+        self.instance.types.select do |_, v|
+          v[:PUID].include? puid rescue false
+        end.keys
+      end
+
+      def self.puid_groups(puid)
+        puid_types(puid).map do |t|
+          type_group t
+        end
+      end
+
+      def self.mime_infos(mime)
+        self.instance.types.select do |_, v|
+          v[:MIME].include? mime rescue false
+        end.values
+      end
+
+      def self.mime_types(mime)
+        self.instance.types.select do |_, v|
+          v[:MIME].include? mime rescue false
+        end.keys
+      end
+
+      def self.mime_groups(mime)
+        mime_types(mime).map do |t|
+          type_group t
+        end
+      end
+
+      def self.ext_infos(ext)
+        ext = ext.gsub /^\./, ''
+        self.instance.types.select do |_, v|
+          v[:EXTENSIONS].include?(ext) rescue false
+        end.values
+      end
+
+      def self.ext_types(ext)
+        ext = ext.gsub /^\./, ''
+        self.instance.types.select do |_, v|
+          v[:EXTENSIONS].include?(ext) rescue false
+        end.keys
+      end
+
+      def self.puid_typeinfo(puid)
+        self.instance.types.each do |_, v|
+          return v if v[:PUID] and v[:PUID].include?(puid)
+        end
+        nil
+      end
+
+      def self.known_mime?(mime)
+        self.instance.types.each do |_, v|
+          return true if v[:MIME].include? mime
+        end
+        false
+      end
+
+      attr_reader :types
+
+      def load_types(file_or_hash = {}, append = true)
+        hash = file_or_hash.is_a?(Hash) ? file_or_hash : YAML::load_file(file_or_hash)
+        # noinspection RubyResolve
+        hash.each do |group, type_info|
+          type_info.each do |type_name, info|
+            type_key = type_name.to_sym
+            info.symbolize_keys!
+            info[:TYPE] = type_key
+            info[:GROUP] = group.to_sym
+            info[:MIME] = info[:MIME].strip.split(/[\s,]+/).map { |v| v.strip } rescue []
+            info[:EXTENSIONS] = info[:EXTENSIONS].strip.split(/[\s,]+/).map { |v| v.strip } rescue []
+            info[:PUID] = info[:PUID].strip.split(/[\s,]+/).map { |v| v.strip } if info[:PUID]
+            if @types.has_key?(type_key)
+              warn 'Type %s already defined; merging with info from %s.', type_name.to_s, file_or_hash
+              info.merge!(@types[type_key]) do |_,v_new,v_old|
+                case v_old
+                  when Array
+                    append ? v_old + v_new : v_new + v_old
+                  when Hash
+                    append ? v_new.merge(v_old) : v_old.merge(v_new)
+                  else
+                    append ? v_old : v_new
+                end
+              end
+            end
+            @types[type_key] = info
+          end
+        end
+      end
+
+      protected
+
+      def initialize
+        @types = Hash.new
+        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
+        type_database = File.join(data_dir, 'types.yml')
+        load_types(type_database)
+      end
+
+    end
+
+  end
+end

data/lib/libis/format/version.rb

@@ -0,0 +1,5 @@
+module Libis
+  module Format
+    VERSION = '0.9.5'
+  end
+end

data/lib/libis/format.rb

@@ -0,0 +1,23 @@
+require 'libis/format/version'
+
+module Libis
+  module Format
+    autoload :Config, 'libis/format/config'
+    autoload :TypeDatabase, 'libis/format/type_database'
+    autoload :Identifier, 'libis/format/identifier'
+    autoload :Fido, 'libis/format/fido'
+    autoload :Droid, 'libis/format/droid'
+    autoload :OfficeToPdf, 'libis/format/office_to_pdf'
+    autoload :PdfCopy, 'libis/format/pdf_copy'
+    autoload :PdfMerge, 'libis/format/pdf_merge'
+    autoload :PdfSplit, 'libis/format/pdf_split'
+    autoload :PdfToPdfa, 'libis/format/pdf_to_pdfa'
+    autoload :PdfaValidator, 'libis/format/pdfa_validator'
+
+    autoload :Converter, 'libis/format/converter'
+
+    ROOT_DIR = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..'))
+    DATA_DIR = File.join(ROOT_DIR, 'data')
+
+  end
+end