libis-format 0.9.1 → 0.9.3

data/lib/libis/format/identifier.rb

@@ -76,7 +76,7 @@ module Libis
 
         options ||= {}
 
-        result = {}
+        result = { messages: [] }
 
         # use FIDO
         # Note: FIDO does not always do a good job, mainly due to lacking container inspection.
@@ -96,9 +96,9 @@ module Libis
         # Libis::Tools::Format::Identifier.add_xml_validation('my_type', '/path/to/my_type.xsd')
         result = validate_against_xml_schema(file, result)
 
-        result ? info("Identification of '#{file}': '#{result}'") : warn("Could not identify MIME type of '#{file}'")
-
-        result
+        result[:mimetype] ?
+            log_msg(result, :info, "Identification of '#{file}': '#{result}'") :
+            log_msg(result, :warn, "Could not identify MIME type of '#{file}'")
       end
 
       def get_fido_identification(file, result = {}, xtra_formats = nil)
@@ -111,14 +111,13 @@ module Libis
         result.merge! fido_result
         result[:method] = 'fido'
 
-        debug "Fido MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})" unless result.empty?
-        result
+        log_msg(result, :debug, "Fido MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})")
       end
 
       def get_droid_identification(file, result = {})
         return result if result_ok? result, :DROID
         droid_output = ::Libis::Format::Droid.run file
-        debug "DROID: #{droid_output}"
+        result[:messages] << [:debug, "DROID: #{droid_output}"]
         warn 'Droid found multiple matches; using first match only' if droid_output.size > 1
         result.clear
         droid_output = droid_output.first
@@ -129,18 +128,16 @@ module Libis
         result[:format_version] = droid_output[:format_version]
         result[:method] = 'droid'
 
-        debug "Droid MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})" if result
-        result
+        log_msg(result, :debug, "Droid MIME-type: #{result[:mimetype]} (PRONOM UID: #{result[:puid]})")
       end
 
       def get_file_identification(file, result = nil)
         return result if result_ok? result
-        result = {}
         begin
           output = ::Libis::Tools::Command.run('file', '-b', '--mime-type', "\"#{file.escape_for_string}\"")[:err]
           mimetype = output.strip.split
           if mimetype
-            debug "File result: '#{mimetype}'"
+            log_msg(result, :debug, "File result: '#{mimetype}'")
             result[:mimetype] = mimetype
             result[:puid] = get_puid(mimetype)
           end
@@ -153,9 +150,8 @@ module Libis
 
       def get_extension_identification(file, result = nil)
         return result if result_ok? result
-        result = {}
         info = ::Libis::Format::TypeDatabase.ext_infos(File.extname(file)).first
-        debug "File extension info: #{info}"
+        log_msg result, :debug, "File extension info: #{info}"
         if info
           result[:mimetype] = info[:MIME].first rescue nil
           result[:puid] = info[:PUID].first rescue nil
@@ -170,7 +166,7 @@ module Libis
         xml_validations.each do |mime, xsd_file|
           next unless xsd_file
           if doc.validates_against?(xsd_file)
-            debug "XML file validated against XML Schema: #{xsd_file}"
+            log_msg result, :debug, "XML file validated against XML Schema: #{xsd_file}"
             result[:mimetype] = mime
             result[:puid] = nil
             result = ::Libis::Format::TypeDatabase.enrich(result, PUID: :puid, MIME: :mimetype)
@@ -179,6 +175,14 @@ module Libis
         result
       end
 
+      private
+
+      def log_msg(result, severity, text)
+        return {} unless result.is_a?(Hash)
+        (result[:messages] ||= []) << [severity, text]
+        result
+      end
+
     end
 
   end

data/lib/libis/format/office_to_pdf.rb

@@ -0,0 +1,52 @@
+require 'fileutils'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+
+    class OfficeToPdf
+      include ::Libis::Tools::Logger
+
+      def self.run(source, target, options = {})
+        self.new.run source, target, options
+      end
+
+      def run(source, target, options = {})
+        workdir = '/...'
+        workdir = Dir.tmpdir unless Dir.exist? workdir
+
+        workdir = File.join(workdir, rand(1000000).to_s)
+        FileUtils.mkpath(workdir)
+
+        src_file = File.join(workdir, File.basename(source))
+        FileUtils.link source, src_file
+
+        tgt_file = File.join(workdir, File.basename(source, '.*') + '.pdf')
+
+        export_filter = options[:export_filter] || 'pdf'
+
+        result = Libis::Tools::Command.run(
+            Libis::Format::Config[:soffice_path], '--headless',
+            '--convert-to', export_filter,
+            '--outdir', workdir, src_file
+        )
+
+        unless result[:status] == 0
+          warn "PdfConvert errors: #{(result[:err] + result[:out]).join("\n")}"
+          return false
+        end
+
+        FileUtils.copy tgt_file, target, preserve: true
+        FileUtils.rmtree workdir
+
+        result[:out]
+      end
+    end
+
+  end
+end

data/lib/libis/format/pdf_copy.rb

@@ -0,0 +1,50 @@
+require 'os'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+
+    class PdfCopy
+      include ::Libis::Tools::Logger
+
+      def self.run(source, target, options = [])
+        self.new.run source, target, options
+      end
+
+      def run(source, target, options = [])
+        tool_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'tools'))
+        jar_file = File.join(tool_dir, 'PdfTool.jar')
+        if OS.java?
+          # TODO: import library and execute in current VM. For now do exactly as in MRI
+          result = Libis::Tools::Command.run(
+              Libis::Format::Config[:java_path],
+              '-jar', jar_file,
+              'CopyPdf',
+              '--file_input', source,
+              '--file_output', target,
+              *options
+          )
+          warn "PdfCopy errors: #{result[:err].join("\n")}" unless result[:status] == 0
+          result[:out]
+        else
+          result = Libis::Tools::Command.run(
+              Libis::Format::Config[:java_path],
+              '-jar', jar_file,
+              'CopyPdf',
+              '--file_input', source,
+              '--file_output', target,
+              *options
+          )
+          warn "PdfCopy errors: #{result[:err].join("\n")}" unless result[:status] == 0
+          result[:out]
+        end
+      end
+    end
+
+  end
+end

data/lib/libis/format/pdf_to_pdfa.rb

@@ -0,0 +1,79 @@
+require 'tempfile'
+require 'csv'
+require 'fileutils'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format'
+
+module Libis
+  module Format
+
+    class PdfToPdfa
+      include ::Libis::Tools::Logger
+
+      def self.run(source, target = nil, options = {})
+        self.new.run source, target, options
+      end
+
+      def run(source, target = nil, options = nil)
+
+        target ||= File.join(Dir.tmpdir, Dir::Tmpname.make_tmpname([File.basename(source, '.*'), '.pdf']))
+
+        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
+
+        icc_info = icc_options(options[:colorspace])
+
+        icc_file = File.join(Dir.tmpdir, "#{icc_info[:icc_name]}#{Random.new.bytes(12).unpack('H*').first}.icc")
+        FileUtils.cp(File.join(data_dir, "#{icc_info[:icc_name]}.icc"), icc_file)
+
+        def_filename = File.join(Dir.tmpdir, "PDFA_def_#{Random.new.bytes(12).unpack('H*').first}.ps")
+        File.open(def_filename, 'w') do |f|
+          f.puts File.read(File.join(Libis::Format::DATA_DIR, 'PDFA_def.ps')).
+                     gsub('[** Fill in ICC profile location **]', icc_file).
+                     gsub('[** Fill in ICC reference name **]', icc_info[:icc_ref])
+        end
+
+        result = Libis::Tools::Command.run(
+            Libis::Format::Config[:ghostscript_path],
+            '-dBATCH', '-dNOPAUSE', '-dNOOUTERSAVE',
+            '-sColorConversionStrategy=/UseDeviceIndependentColor',
+            "-sProcessColorModel=#{icc_info[:device]}",
+            '-sDEVICE=pdfwrite', '-dPDFA', '-dPDFACompatibilityPolicy=1',
+            "-sOutputICCProfile=#{icc_file}",
+            '-o', File.absolute_path(target),
+            def_filename,
+            source
+        )
+
+        FileUtils.rm [icc_file, def_filename].compact, force: true
+        unless result[:status] == 0
+          warn (['Pdf2PdfA errors:'] + result[:err] + result[:out]).join("\n").gsub('%', '%%')
+        end
+
+        unless PdfaValidator.run(target)
+          error "Failed to generate correct PDF/A file from '%s'", source
+          return nil
+        end
+
+        target
+      end
+
+
+      private
+
+      def icc_options(colorspace)
+        case colorspace.to_s.downcase
+          when 'cmyk'
+            {icc_name: 'ISOcoated_v2_eci', icc_ref: 'FOGRA39L', device: 'DeviceCMYK'}
+          else
+            {icc_name: 'eciRGB_v2', icc_ref: 'sRGB', device: 'DeviceRGB'}
+        end
+      end
+
+    end
+
+  end
+end

data/lib/libis/format/pdfa_validator.rb

@@ -0,0 +1,61 @@
+require 'fileutils'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+
+    class PdfaValidator
+      include ::Libis::Tools::Logger
+
+      def self.run(source)
+        self.new.run source
+      end
+
+      def run(source)
+
+        src_file = File.absolute_path(source)
+
+        if (pdfa = Libis::Format::Config[:pdfa_path])
+          # Keep it clean: tool generates fontconfig/ cache dir in current working dir
+          previous_wd = Dir.getwd
+          Dir.chdir(Dir.tmpdir)
+
+          result = Libis::Tools::Command.run(
+              pdfa,
+              '--noxml',
+              '--level', 'B',
+              '--verb', '0',
+              src_file
+          )
+
+          Dir.chdir(previous_wd)
+
+          unless result[:out].any? { |line| line =~ /^VLD-\[PASS\]/ }
+            warn "Validator failed to validate the PDF file '%s' against PDF/A-1B constraints:\n%s", source,
+                 result[:out].join("\n")
+            return false
+          end
+        else
+          jar = File.join(ROOT_DIR, 'tools', 'pdfbox', 'preflight-app-1.8.10.jar')
+          result = Libis::Tools::Command.run(
+              Libis::Format::Config[:java_path],
+              '-jar', jar,
+              src_file
+          )
+          unless result[:status] == 0
+            warn "Validator failed to validate the PDF file '%s' against PDF/A-1B constraints:\n%s", source,
+                 result[:out].join("\n")
+            return false
+          end
+        end
+        true
+      end
+    end
+
+  end
+end

data/lib/libis/format/type_database.rb

@@ -15,7 +15,7 @@ module Libis
       include ::Libis::Tools::Logger
 
       def self.typeinfo(t)
-        self.instance.types[t] || {}
+        self.instance.types[t.to_sym] || {}
       end
 
       def self.enrich(info, map_keys = {})

data/lib/libis/format/version.rb

@@ -1,5 +1,5 @@
 module Libis
   module Format
-    VERSION = '0.9.1'
+    VERSION = '0.9.3'
   end
 end

data/lib/libis/format.rb

@@ -2,11 +2,20 @@ require 'libis/format/version'
 
 module Libis
   module Format
+    autoload :Config, 'libis/format/config'
     autoload :TypeDatabase, 'libis/format/type_database'
     autoload :Identifier, 'libis/format/identifier'
     autoload :Fido, 'libis/format/fido'
     autoload :Droid, 'libis/format/droid'
+    autoload :OfficeToPdf, 'libis/format/office_to_pdf'
+    autoload :PdfCopy, 'libis/format/pdf_copy'
+    autoload :PdfToPdfa, 'libis/format/pdf_to_pdfa'
+    autoload :PdfaValidator, 'libis/format/pdfa_validator'
 
     autoload :Converter, 'libis/format/converter'
+
+    ROOT_DIR = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..'))
+    DATA_DIR = File.join(ROOT_DIR, 'data')
+
   end
 end

data/libis-format.gemspec

@@ -27,4 +27,6 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'libis-tools', '~> 0.9'
   spec.add_runtime_dependency 'os', '= 0.9.6'
+  spec.add_runtime_dependency 'mini_magick', '~> 4.3'
+  spec.add_runtime_dependency 'deep_dive', '~> 0.3'
 end

data/spec/converter_spec.rb

@@ -0,0 +1,212 @@
+# encoding: utf-8
+require 'spec_helper'
+
+require 'libis/format/converter/image_converter'
+require 'libis/format/converter/pdf_converter'
+require 'libis/format/converter/office_converter'
+
+RSpec::Matchers.define(:be_same_file_as) do |exected_file_path|
+  match do |actual_file_path|
+    expect(md5_hash(actual_file_path)).to eq md5_hash(exected_file_path)
+  end
+
+  def md5_hash(file_path)
+    Digest::MD5.hexdigest(File.read(file_path))
+  end
+end
+
+describe 'Converters' do
+
+  let(:repository) { Libis::Format::Converter::Repository }
+  let(:file_dir) { File.dirname(__FILE__)}
+
+  before(:all) {
+    Libis::Tools::Config[:logger].level = Logger::WARN
+  }
+
+  context 'Repository' do
+
+    it 'loads all converters' do
+      expect(repository.get_converters.size).to eq 3
+      # noinspection RubyResolve
+      expect(repository.get_converters.map(&:to_s)).to include 'Libis::Format::Converter::ImageConverter'
+      # noinspection RubyResolve
+      expect(repository.get_converters.map(&:to_s)).to include 'Libis::Format::Converter::OfficeConverter'
+      # noinspection RubyResolve
+      expect(repository.get_converters.map(&:to_s)).to include 'Libis::Format::Converter::PdfConverter'
+    end
+
+    it 'creates simple converter chain' do
+      chain = repository.get_converter_chain(:TIFF, :PDF)
+      expect(chain).to_not be nil
+      expect(chain.to_array.size).to eq 1
+      expect(chain.to_array).to match [{converter: Libis::Format::Converter::ImageConverter, input: :TIFF, output: :PDF}]
+    end
+
+    it 'creates complex chain' do
+      chain = repository.get_converter_chain(:TIFF, :PDFA)
+      expect(chain).to_not be nil
+      expect(chain.to_array.size).to eq 2
+      expect(chain.to_array).to match [
+                                          {converter: Libis::Format::Converter::ImageConverter, input: :TIFF, output: :PDF},
+                                          {converter: Libis::Format::Converter::PdfConverter, input: :PDF, output: :PDFA},
+                                      ]
+    end
+
+    it 'creates converter chain with options' do
+      chain = repository.get_converter_chain(:TIFF, :PDF, {watermark: {}})
+      expect(chain).to_not be nil
+      expect(chain.to_array.size).to eq 1
+      expect(chain.to_array).to match [
+                                          {converter: Libis::Format::Converter::ImageConverter, input: :TIFF, output: :PDF, operations: [{method: :watermark, argument: {}}]}
+                                      ]
+    end
+
+    it 'perfers operations to the end of the chain' do
+      chain = repository.get_converter_chain(:TIFF, :PDFA, {watermark: {}})
+      expect(chain).to_not be nil
+      expect(chain.to_array.size).to eq 2
+      expect(chain.to_array).to match [
+                                          {converter: Libis::Format::Converter::ImageConverter, input: :TIFF, output: :PDF},
+                                          {converter: Libis::Format::Converter::PdfConverter, input: :PDF, output: :PDFA, operations: [{method: :watermark, argument: {}}]}
+                                      ]
+    end
+
+  end
+
+  context 'Image Converter' do
+
+    let(:converter) { Libis::Format::Converter::ImageConverter.new }
+
+    it 'converts TIFF to JPEG' do
+      src_file = File.join(file_dir, 'data', 'test.tif')
+      ref_file = File.join(file_dir, 'data', 'test.jpg')
+      tgt_file = File.join('', 'tmp', 'test.jpg')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :JPG)
+      expect(result).to eq tgt_file
+      expect(tgt_file).to be_same_file_as ref_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+    it 'converts TIFF to PNG' do
+      src_file = File.join(file_dir, 'data', 'test.tif')
+      ref_file = File.join(file_dir, 'data', 'test.png')
+      tgt_file = File.join('', 'tmp', 'test.png')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :PNG)
+      expect(result).to eq tgt_file
+      expect(tgt_file).to be_same_file_as ref_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+    it 'converts PDF to TIFF' do
+      src_file = File.join(file_dir, 'data', 'test.pdf')
+      ref_file = File.join(file_dir, 'data', 'test.pdf.tif')
+      tgt_file = File.join('', 'tmp', 'test.pdf.tif')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :TIFF)
+      expect(result).to eq tgt_file
+      expect(tgt_file).to be_same_file_as ref_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+    it 'converts TIFF to JPEG with many options' do
+      src_file = File.join(file_dir, 'data', 'test.tif')
+      ref_file = File.join(file_dir, 'data', 'test-options.jpg')
+      tgt_file = File.join('', 'tmp', 'test-options.jpg')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      converter.watermark(text: 'RSPEC', size: 5, opacity: 0.1, rotation: 15, gap: 0.5, composition: 'modulate')
+      result = converter.convert(src_file, tgt_file, :JPG, options: {scale: '150%', quality: '70%'})
+      expect(result).to eq tgt_file
+      expect(tgt_file).to be_same_file_as ref_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+  end
+
+  context 'Pdf Converter' do
+
+    let(:converter) { Libis::Format::Converter::PdfConverter.new }
+
+    it 'converts PDF to PDF/A' do
+      src_file = File.join(file_dir, 'data', 'test.pdf')
+      tgt_file = File.join('', 'tmp', 'test_pdfa.pdf')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :PDFA)
+      expect(result).to eq tgt_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+  end
+
+  context 'Office Converter' do
+
+    let(:converter) { Libis::Format::Converter::OfficeConverter.new }
+
+    it 'converts Word document to PDF' do
+      src_file = File.join(file_dir, 'data', 'test.doc')
+      tgt_file = File.join(file_dir, 'work', 'test_doc.pdf')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :PDF)
+      expect(result).to eq tgt_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+    it 'converts Word 2010 document to PDF' do
+      src_file = File.join(file_dir, 'data', 'test.docx')
+      tgt_file = File.join(file_dir, 'work', 'test_docx.pdf')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :PDF)
+      expect(result).to eq tgt_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+    it 'converts OpenOffice document to PDF' do
+      src_file = File.join(file_dir, 'data', 'test.odt')
+      tgt_file = File.join(file_dir, 'work', 'test_odt.pdf')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :PDF)
+      expect(result).to eq tgt_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+    it 'converts RTF document to PDF' do
+      src_file = File.join(file_dir, 'data', 'test.rtf')
+      tgt_file = File.join(file_dir, 'work', 'test_rtf.pdf')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :PDF)
+      expect(result).to eq tgt_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+    it 'converts TXT document to PDF' do
+      src_file = File.join(file_dir, 'data', 'test.txt')
+      tgt_file = File.join(file_dir, 'work', 'test_txt.pdf')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :PDF)
+      expect(result).to eq tgt_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+    it 'converts Excel to PDF' do
+      src_file = File.join(file_dir, 'data', 'test.xls')
+      tgt_file = File.join(file_dir, 'work', 'test_xls.pdf')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :PDF)
+      expect(result).to eq tgt_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+    it 'converts Excel 2011 to PDF' do
+      src_file = File.join(file_dir, 'data', 'test.xlsx')
+      tgt_file = File.join(file_dir, 'work', 'test_xlsx.pdf')
+      FileUtils.mkdir_p File.dirname(tgt_file)
+      result = converter.convert(src_file, tgt_file, :PDF)
+      expect(result).to eq tgt_file
+      FileUtils.rm tgt_file, force: true
+    end
+
+  end
+
+end

data/spec/identifier_spec.rb

@@ -47,6 +47,7 @@ describe 'Identfier' do
         'test-lzw.tif' => {mimetype: 'image/tiff', puid: 'fmt/353'},
         'test.ps' => {mimetype: 'application/postscript', puid: 'x-fmt/408'},
         'test.png' => {mimetype: 'image/png', puid: 'fmt/11'},
+        'test.jpg' => {mimetype: 'image/jpeg', puid: 'fmt/43'},
         'test.gif' => {mimetype: 'image/gif', puid: 'fmt/4'},
         'test.xml' => {mimetype: 'application/xml', puid: 'fmt/101'},
         'test-ead.xml' => {mimetype: 'archive/ead', puid: 'fmt/101'},