libis-format 1.0.8 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4de569e2d7055cf0d08602b481e3e7e2eb981a3dfc81968b2ebb19239244ffb
-  data.tar.gz: 402ec6e4ab234d75e8dadc3bf862398cb81dd7563d7c316279aad82f098ab3e4
+  metadata.gz: c8f2304a5f189140ad867cb1d7d9d41f280f796992c84514316b7e584628a64b
+  data.tar.gz: ee097826d8e73fa7fa72167b4a3eaebef5992bba8fd78a6e0d454fe552730d6c
 SHA512:
-  metadata.gz: 38bb64081e41ca8abe3e43b26610989162f5b6cc9f3f4e0123d524d5519dbb879dd0a1634920ceb78690cd06ee81dc8bfdf8a7cf0e6719832f399c716d8a4c7c
-  data.tar.gz: 70c249c6b0814614e7408bb9fa7b448af8db3ea54bda3028f7a49ac79987e254b309a53effa3334713e3e853c09942809bb26d2b4ee09e79823ee50c88661c12
+  metadata.gz: 9fdb85d3847ba3ed08c96cc409b5e7f2c471d80edf61ab6581b28cb46380e1b48e8c65daf182717dd35cf216d91e50f1680d31a352d9aab557d54f7f3af61dc8
+  data.tar.gz: 0d8d4f7c29c0fa5da26a85eb82554796cb1ac18f98e00e3f1e700bd4eb79f647b8311f56e2eb7a46cc1679a2809fc140b1cc702856da1baacafd5127a65ea66f

data/.gitignore

@@ -18,3 +18,4 @@ mkmf.log
 *.pyc
 tools/pdf/
 /test*
+.rake_tasks~

data/Gemfile

@@ -3,3 +3,5 @@ source 'https://rubygems.org'
 gem 'coveralls', group: :test, require: false
 
 gemspec name: 'libis-format', development_group: :test
+
+gem 'standard'

data/data/PDFA_def.ps

@@ -5,10 +5,10 @@
 % unless the user modifies the corresponding line below.
 
 % Define entries in the document Info dictionary :
-/ICCProfile ([** Fill in ICC profile location **])
+/ICCProfile ([**ICC profile**])
 def
 
-[ /Title (Title)
+[ [**METADATA**]
   /DOCINFO pdfmark
 
 % Define an ICC profile :
@@ -35,6 +35,6 @@ def
   /Type /OutputIntent             % Must be so (the standard requires).
   /S /GTS_PDFA1                   % Must be so (the standard requires).
   /DestOutputProfile {icc_PDFA}            % Must be so (see above).
-  /OutputConditionIdentifier ([** Fill in ICC reference name **])
+  /OutputConditionIdentifier ([**ICC reference**])
 >> /PUT pdfmark
 [{Catalog} <</OutputIntents [ {OutputIntent_PDFA} ]>> /PUT pdfmark

data/lib/libis/format/config.rb

@@ -12,7 +12,6 @@ module Libis
     Config[:java_cmd] = 'java'
     Config[:j2k_cmd] = 'j2kdriver'
     Config[:soffice_cmd] = 'soffice'
-    Config[:emai2pdf_cmd] = 'convert_email'
     Config[:ghostscript_cmd] = 'gs'
     Config[:droid_cmd] = '/opt/droid/droid.sh'
     Config[:droid_temp_path] = '/tmp'
@@ -22,6 +21,7 @@ module Libis
     Config[:fido_formats] = [(File.join(Libis::Format::DATA_DIR, 'lias_formats.xml'))]
     Config[:pdf_tool] = File.join(Libis::Format::TOOL_DIR, 'PdfTool.jar')
     Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
+    Config[:email2pdf_jar] = File.join(Libis::Format::TOOL_DIR, 'emailconverter.jar')
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
     Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')

data/lib/libis/format/converter/audio_converter.rb

@@ -97,19 +97,18 @@ module Libis
 
           end
 
-          target
-
         end
 
         def assemble_and_convert(sources, target)
+          result = {}
           Tempfile.create(%w(list .txt)) do |f|
             sources.each {|src| f.puts src}
             opts[:global] ||= []
             opts[:global] += %w(-f concat)
             f.close
-            target = convert_file(f.to_path, target)
+            result = convert_file(f.to_path, target)
           end
-          target
+          result
         end
 
         def self.sounds_like(file1, file2, threshold, rate, channels)
@@ -155,6 +154,7 @@ module Libis
 
         def convert_file(source, target)
           opts = {global: [], input: [], filter: [], output: []}
+
           opts[:global] << '-hide_banner'
           opts[:global] << '-loglevel' << (@options[:quiet] ? 'fatal' : 'warning')
           opts[:output] << '-vn' # disable input video stream in case it exists
@@ -168,10 +168,8 @@ module Libis
           opts[:output] << '-ar' << @options[:sampling_freq] if @options[:sampling_freq]
           opts[:output] << '-ac' << @options[:channels] if @options[:channels]
           opts[:output] << '-f' << @options[:format] if @options[:format]
-          result = Libis::Format::Tool::FFMpeg.run(source, target, opts)
-          info "FFMpeg output: #{result}"
-          result
-          target
+
+          Libis::Format::Tool::FFMpeg.run(source, target, opts)
         end
 
       end

data/lib/libis/format/converter/chain.rb

@@ -68,14 +68,18 @@ module Libis
           end
 
           temp_files = []
+          xtra_files = []
+
+          result = { commands: [] }
 
           # noinspection RubyParenthesesAroundConditionInspection
-          result = @converter_chain.each_with_index do |node, i|
+          conversion_success = converter_chain.each_with_index do |node, i|
 
             target_type = node[:output]
             converter_class = node[:converter]
             converter = converter_class.new
 
+
             node[:operations].each do |operation|
               converter.send operation[:method], operation[:argument]
             end if node[:operations]
@@ -90,17 +94,25 @@ module Libis
 
             FileUtils.mkdir_p File.dirname(target)
 
-            src_file = converter.convert(src_file, target, target_type)
+            r = converter.convert(src_file, target, target_type)
+
+            src_file = r[:files].first
+            xtra_files += r[:files][1..]
+            break :failed unless src_file1
 
-            break :failed unless src_file
+            result[:commands] << r.merge(converter: converter_class.name)
+
+            :success
 
           end
 
+          result[:files] = [src_file] + xtra_files
+
           temp_files.each do |f|
             FileUtils.rm(f, force: true)
           end
 
-          result == :failed ? nil : target_file
+          conversion_success == :failed ? nil : result
 
         end
 

data/lib/libis/format/converter/email_converter.rb

@@ -4,6 +4,7 @@ require_relative 'base'
 
 require 'libis/format/tool/email_to_pdf'
 require 'libis/format/type_database'
+require 'rexml/document'
 
 module Libis
   module Format
@@ -27,10 +28,7 @@ module Libis
         def convert(source, target, format, opts = {})
           super
 
-          return nil unless Format::Tool::EmailToPdf.run(source, target)
-
-          target
-
+          Format::Tool::EmailToPdf.run(source, target)
         end
 
       end

data/lib/libis/format/converter/fop_pdf_converter.rb

@@ -31,8 +31,6 @@ module Libis
 
           Libis::Format::Tool::FopPdf.run(source, target)
 
-          target
-
         end
 
       end

data/lib/libis/format/converter/image_converter.rb

@@ -180,9 +180,11 @@ module Libis
               convert_image(source, target, format)
             end
           end
-
-          target
-
+          
+          { 
+            files: [target],
+            converter: self.class.name
+          }
         end
 
         def assemble_and_convert(sources, target, format)

data/lib/libis/format/converter/jp2_converter.rb

@@ -90,16 +90,16 @@ module Libis
           end
 
 
-          Libis::Tools::Command.run(
+          result = Libis::Tools::Command.run(
               Libis::Format::Config[:j2k_cmd],
               '--input-file-name', source,
               '--set-output-type', 'JP2',
               *options,
               '--output-file-name', target,
-
           )
 
-          target
+          result.merge(files: [target], converter: self.class.name)
+
         end
       end
     end

data/lib/libis/format/converter/office_converter.rb

@@ -43,9 +43,7 @@ module Libis
         def convert(source, target, format, opts = {})
           super
 
-          return nil unless Format::Tool::OfficeToPdf.run(source, target)
-
-          target
+          Format::Tool::OfficeToPdf.run(source, target)
 
         end
 

data/lib/libis/format/converter/pdf_converter.rb

@@ -5,6 +5,7 @@ require_relative 'base'
 require 'libis/tools/extend/hash'
 require 'libis/format/tool/pdf_copy'
 require 'libis/format/tool/pdf_to_pdfa'
+require 'libis/format/tool/pdfa_validator'
 require 'libis/format/tool/pdf_optimizer'
 
 module Libis
@@ -119,7 +120,10 @@ module Libis
             result = pdf_to_pdfa(source, target)
           end
 
-          result
+          { 
+            files: [result],
+            converter: self.class.name
+          }
 
         end
 
@@ -160,11 +164,16 @@ module Libis
 
           using_temp(target) do |tmpname|
             result = Libis::Format::Tool::PdfToPdfa.run source, tmpname
-            if result[:status] != 0
-              error("Pdf/A conversion encountered errors:\n%s", result[:err].join("\n"))
+
+            if result[:command][:status] != 0
+              error("Pdf/A conversion encountered errors:\n%s", (result[:command][:out] + result[:command][:err]).join("\n"))
               next nil
             else
-              warn("Pdf/A conversion warnings:\n%s", result[:err].join("\n")) unless result[:err].empty?
+              r = Libis::Format::Tool::PdfaValidator.run tmpname
+              if r[:status] != 0
+                error "Pdf/A file failed to validate with following errors:\n%s", (r[:err] || r[:out] || []).join("\n")
+                next nil
+              end
             end
             tmpname
           end

data/lib/libis/format/converter/spreadsheet_converter.rb

@@ -31,9 +31,7 @@ module Libis
         def convert(source, target, format, opts = {})
           super
 
-          return nil unless Format::Tool::SpreadsheetToOds.run(source, target)
-
-          target
+          Format::Tool::SpreadsheetToOds.run(source, target)
 
         end
 

data/lib/libis/format/converter/video_converter.rb

@@ -1,5 +1,5 @@
 require_relative 'base'
-require 'libis/format/tool/ffmpeg'
+require 'libis/format/tool/ff_mpeg'
 
 require 'fileutils'
 
@@ -179,7 +179,10 @@ module Libis
 
           end
 
-          target
+          {
+            files: [target],
+            converter: self.class.name
+          }
 
         end
 

data/lib/libis/format/converter/xslt_converter.rb

@@ -1,17 +1,17 @@
-require_relative 'base'
+require_relative "base"
 
 module Libis
   module Format
     module Converter
-
       class XsltConverter < Libis::Format::Converter::Base
-
         def self.input_types
           [:XML]
         end
 
         def self.output_types(format = nil)
-          return [] unless input_types.include?(format) if format
+          if format
+            return [] unless input_types.include?(format)
+          end
           [:XML, :HTML, :TXT]
         end
 
@@ -28,20 +28,20 @@ module Libis
           end
 
           unless @options[:xsl_file]
-            error 'No xsl_file supplied'
+            error "No xsl_file supplied"
             return nil
           end
 
           FileUtils.mkpath(File.dirname(target))
 
           if RUBY_PLATFORM == "java"
-            require 'saxon-xslt'
+            require "saxon-xslt"
             xsl = Saxon.XSLT(File.open(@options[:xsl_file]))
             xml = Saxon.XML(File.open(source))
             result = xsl.transform(xml)
-            File.open(target, 'w') {|f| f.write(result.to_s)}
+            File.write(target, result.to_s)
           else
-            require 'nokogiri'
+            require "nokogiri"
 
             doc = nil
             begin
@@ -65,7 +65,7 @@ module Libis
             xsl = nil
 
             begin
-              fp = File.open(file, 'r')
+              fp = File.open(file, "r")
               xsl = Nokogiri::XSLT(fp) do |config|
                 config.options = Nokogiri::XML::ParseOptions::STRICT | Nokogiri::XML::ParseOptions::NOBLANKS
               end
@@ -80,7 +80,7 @@ module Libis
 
             begin
               target_xml = xsl.transform(doc)
-              fp = File.open(target, 'w')
+              fp = File.open(target, "w")
               fp.write(target_xml)
             rescue Exception => e
               error "Error transforming '#{source}' with '#{file}': #{e.message} @ #{e.backtrace[0]}"
@@ -89,13 +89,14 @@ module Libis
               fp.close unless fp.nil? or fp.closed?
             end
 
-            target
-          end
+            {
+              command: {status: 0},
+              files: [target]
+            }
 
+          end
         end
-
       end
-
     end
   end
 end

data/lib/libis/format/tool/email_to_pdf.rb

@@ -1,40 +1,75 @@
-require 'fileutils'
+require "fileutils"
 
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
+require "libis/tools/extend/string"
+require "libis/tools/logger"
+require "libis/tools/command"
 
-require 'libis/format/config'
+require "libis/format/config"
+require "rexml/document"
 
 module Libis
   module Format
     module Tool
-
       class EmailToPdf
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:email2pdf_cmd], "-v")
+          result[:status] == 0
+        end
+
         def self.run(source, target, options = {})
-          self.new.run source, target, options
+          new.run source, target, options
         end
 
         def run(source, target, _ = {})
           timeout = Libis::Format::Config[:timeouts][:email2pdf] || 120
           result = Libis::Tools::Command.run(
-              Libis::Format::Config[:email2pdf_cmd],
-              source,
-              target,
-              timeout: timeout,
-              kill_after: timeout * 2
+            Libis::Format::Config[:java_cmd],
+            "-Duser.timezone=Europe/Brussels", "-Duser.language=nl", "-Duser.country=BE",
+            "-jar", Libis::Format::Config[:email2pdf_jar],
+            "-e", "-hd", "-a",
+            "-o", target,
+            source,
+            timeout: timeout,
+            kill_after: timeout * 2
           )
 
-          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          warn "EmailToPdf conversion messages: \n\t#{result[:out].join("\n\t")}" unless result[:err].empty?
-          raise RuntimeError, "#{self.class} failed to generate target file #{target}" unless File.exist?(target)
+          warn "EmailToPdf conversion messages: \n\t#{result[:err].join("\n\t")}" unless result[:err].empty?
+
+          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise "#{self.class} failed to generate target file #{target}" unless File.exist?(target)
+          raise "#{self.class} command failed with status code #{result[:status]}" unless result[:status] == 0
+
+          base_path = File.join(File.dirname(target), File.basename(target, ".*"))
+          headers_file = "#{base_path}.headers.xml"
+          headers = read_header(headers_file)
 
-          nil
+          {
+            command: result,
+            files: [target, headers_file] + headers[:attachments].map { |a| File.join("#{base_path}-attachments", a) },
+            headers: headers
+          }
         end
-      end
 
+        private
+
+        def read_header(headers_file)
+          headers = {}
+          return headers unless File.exist?(headers_file)
+          doc = REXML::Document.new(File.new(headers_file))
+          root = doc.root
+          root.children.each do |element|
+            case element.name
+            when "attachments"
+              headers[:attachments] = element.elements.map { |e| e.text }
+            else
+              headers[element.name.to_sym] = element.text
+            end
+          end
+          headers
+        end
+      end
     end
   end
 end

data/lib/libis/format/tool/{ffmpeg.rb → ff_mpeg.rb}

@@ -13,6 +13,11 @@ module Libis
         include Singleton
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:ffmpeg_cmd], "-h")
+          result[:status] == 0
+        end
+
         def self.run(source, target, options = {})
           self.instance.run source, target, options
         end
@@ -38,7 +43,11 @@ module Libis
 
           warn "FFMpeg warnings: #{(result[:err] + result[:out]).join("\n")}" unless result[:err].empty?
 
-          result[:out]
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
 
       end

data/lib/libis/format/tool/fop_pdf.rb

@@ -13,6 +13,12 @@ module Libis
       class FopPdf
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:fop_jar])
+        end
+
         def self.run(xml, target, options = [])
           self.new.run xml, target, options
         end
@@ -27,6 +33,7 @@ module Libis
           result = Libis::Tools::Command.run(
               Libis::Format::Config[:java_cmd],
               "-Dfop.home=#{File.dirname(Libis::Format::Config[:fop_jar])}",
+              '-Djava.awt.headless=true',
               '-jar', Libis::Format::Config[:fop_jar],
               '-fo', xml,
               '-pdf', target,
@@ -37,6 +44,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/office_to_pdf.rb

@@ -13,6 +13,11 @@ module Libis
       class OfficeToPdf
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:soffice_cmd], "--version")
+          result[:status] == 0
+        end
+
         def self.run(source, target, options = {})
           self.new.run source, target, options
         end
@@ -47,10 +52,14 @@ module Libis
 
           FileUtils.copy tgt_file, target, preserve: true
 
+          {
+            command: result,
+            files: [ target ]
+          }
+
         ensure
           FileUtils.rmtree workdir rescue nil
 
-          result[:out]
         end
       end
 

data/lib/libis/format/tool/pdf_copy.rb

@@ -13,6 +13,12 @@ module Libis
       class PdfCopy
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:pdf_tool])
+        end
+
         def self.run(source, target, options = [])
           self.new.run source, target, options
         end
@@ -38,7 +44,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_merge.rb

@@ -13,6 +13,12 @@ module Libis
       class PdfMerge
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:pdf_tool])
+        end
+
         def self.run(source, target, options = [])
           self.new.run source, target, options
         end
@@ -39,7 +45,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_optimizer.rb

@@ -13,6 +13,11 @@ module Libis
       class PdfOptimizer
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:ghostscript_cmd], "--version")
+          result[:status] == 0
+        end
+
         def self.run(source, target, quality)
           self.new.run source, target, quality
         end
@@ -21,7 +26,7 @@ module Libis
 
           timeout = Libis::Format::Config[:timeouts][:pdf_optimizer]
           result = Libis::Tools::Command.run(
-              'gs',
+              Libis::Format::Config[:ghostscript_cmd],
               '-sDEVICE=pdfwrite',
               '-dCompatibilityLevel=1.4',
               "-dPDFSETTINGS=/#{quality}",
@@ -36,7 +41,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end