libis-format 1.0.8 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4de569e2d7055cf0d08602b481e3e7e2eb981a3dfc81968b2ebb19239244ffb
-  data.tar.gz: 402ec6e4ab234d75e8dadc3bf862398cb81dd7563d7c316279aad82f098ab3e4
+  metadata.gz: 40c91211db466867a86d905e707302783d77f8ce96bce970229294fc3506f4dd
+  data.tar.gz: 3254048984ba29c9c6a1d556f25e7dc346ba1c0fc2082b80562c3ae64f41dc8f
 SHA512:
-  metadata.gz: 38bb64081e41ca8abe3e43b26610989162f5b6cc9f3f4e0123d524d5519dbb879dd0a1634920ceb78690cd06ee81dc8bfdf8a7cf0e6719832f399c716d8a4c7c
-  data.tar.gz: 70c249c6b0814614e7408bb9fa7b448af8db3ea54bda3028f7a49ac79987e254b309a53effa3334713e3e853c09942809bb26d2b4ee09e79823ee50c88661c12
+  metadata.gz: caa9a7a5cbc6e99c7d1afb8af83274b2db4de847fde394d3d9cdc1129019044353bdbe6527fb4aabbcb8817be175e99f6bac1eeef0b79eb3e7ab94b7594db4ce
+  data.tar.gz: f3ade6093b38aab4b8c529c0d80e5a822aa2042264efc8d076077f74ee0cd20ebd91dbe5b2d6d030394607fcde196b28c8dabc2659788308e926981cca880905

data/Gemfile

@@ -3,3 +3,5 @@ source 'https://rubygems.org'
 gem 'coveralls', group: :test, require: false
 
 gemspec name: 'libis-format', development_group: :test
+
+gem 'standard'

data/lib/libis/format/config.rb

@@ -12,7 +12,6 @@ module Libis
     Config[:java_cmd] = 'java'
     Config[:j2k_cmd] = 'j2kdriver'
     Config[:soffice_cmd] = 'soffice'
-    Config[:emai2pdf_cmd] = 'convert_email'
     Config[:ghostscript_cmd] = 'gs'
     Config[:droid_cmd] = '/opt/droid/droid.sh'
     Config[:droid_temp_path] = '/tmp'
@@ -22,6 +21,7 @@ module Libis
     Config[:fido_formats] = [(File.join(Libis::Format::DATA_DIR, 'lias_formats.xml'))]
     Config[:pdf_tool] = File.join(Libis::Format::TOOL_DIR, 'PdfTool.jar')
     Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
+    Config[:emai2pdf_jar] = File.join(Libis::Format::TOOL_DIR, 'emailconverter.jar')
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
     Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')

data/lib/libis/format/converter/audio_converter.rb

@@ -97,19 +97,18 @@ module Libis
 
           end
 
-          target
-
         end
 
         def assemble_and_convert(sources, target)
+          result = {}
           Tempfile.create(%w(list .txt)) do |f|
             sources.each {|src| f.puts src}
             opts[:global] ||= []
             opts[:global] += %w(-f concat)
             f.close
-            target = convert_file(f.to_path, target)
+            result = convert_file(f.to_path, target)
           end
-          target
+          result
         end
 
         def self.sounds_like(file1, file2, threshold, rate, channels)
@@ -155,6 +154,7 @@ module Libis
 
         def convert_file(source, target)
           opts = {global: [], input: [], filter: [], output: []}
+
           opts[:global] << '-hide_banner'
           opts[:global] << '-loglevel' << (@options[:quiet] ? 'fatal' : 'warning')
           opts[:output] << '-vn' # disable input video stream in case it exists
@@ -168,10 +168,8 @@ module Libis
           opts[:output] << '-ar' << @options[:sampling_freq] if @options[:sampling_freq]
           opts[:output] << '-ac' << @options[:channels] if @options[:channels]
           opts[:output] << '-f' << @options[:format] if @options[:format]
-          result = Libis::Format::Tool::FFMpeg.run(source, target, opts)
-          info "FFMpeg output: #{result}"
-          result
-          target
+
+          Libis::Format::Tool::FFMpeg.run(source, target, opts)
         end
 
       end

data/lib/libis/format/converter/chain.rb

@@ -68,14 +68,18 @@ module Libis
           end
 
           temp_files = []
+          xtra_files = []
+
+          result = { commands: [] }
 
           # noinspection RubyParenthesesAroundConditionInspection
-          result = @converter_chain.each_with_index do |node, i|
+          conversion_success = converter_chain.each_with_index do |node, i|
 
             target_type = node[:output]
             converter_class = node[:converter]
             converter = converter_class.new
 
+
             node[:operations].each do |operation|
               converter.send operation[:method], operation[:argument]
             end if node[:operations]
@@ -90,17 +94,25 @@ module Libis
 
             FileUtils.mkdir_p File.dirname(target)
 
-            src_file = converter.convert(src_file, target, target_type)
+            r = converter.convert(src_file, target, target_type)
+
+            src_file = r[:files].first
+            xtra_files += r[:files][1..]
+            break :failed unless src_file1
 
-            break :failed unless src_file
+            result[:commands] << r.merge(converter: converter_class.name)
+
+            :success
 
           end
 
+          result[:files] = [src_file] + xtra_files
+
           temp_files.each do |f|
             FileUtils.rm(f, force: true)
           end
 
-          result == :failed ? nil : target_file
+          conversion_success == :failed ? nil : result
 
         end
 

data/lib/libis/format/converter/email_converter.rb

@@ -4,6 +4,7 @@ require_relative 'base'
 
 require 'libis/format/tool/email_to_pdf'
 require 'libis/format/type_database'
+require 'rexml/document'
 
 module Libis
   module Format
@@ -27,10 +28,7 @@ module Libis
         def convert(source, target, format, opts = {})
           super
 
-          return nil unless Format::Tool::EmailToPdf.run(source, target)
-
-          target
-
+          Format::Tool::EmailToPdf.run(source, target)
         end
 
       end

data/lib/libis/format/converter/fop_pdf_converter.rb

@@ -31,8 +31,6 @@ module Libis
 
           Libis::Format::Tool::FopPdf.run(source, target)
 
-          target
-
         end
 
       end

data/lib/libis/format/converter/image_converter.rb

@@ -180,9 +180,11 @@ module Libis
               convert_image(source, target, format)
             end
           end
-
-          target
-
+          
+          { 
+            files: [target],
+            converter: self.class.name
+          }
         end
 
         def assemble_and_convert(sources, target, format)

data/lib/libis/format/converter/jp2_converter.rb

@@ -90,16 +90,16 @@ module Libis
           end
 
 
-          Libis::Tools::Command.run(
+          result = Libis::Tools::Command.run(
               Libis::Format::Config[:j2k_cmd],
               '--input-file-name', source,
               '--set-output-type', 'JP2',
               *options,
               '--output-file-name', target,
-
           )
 
-          target
+          result.merge(files: [target], converter: self.class.name)
+
         end
       end
     end

data/lib/libis/format/converter/office_converter.rb

@@ -43,9 +43,7 @@ module Libis
         def convert(source, target, format, opts = {})
           super
 
-          return nil unless Format::Tool::OfficeToPdf.run(source, target)
-
-          target
+          Format::Tool::OfficeToPdf.run(source, target)
 
         end
 

data/lib/libis/format/converter/pdf_converter.rb

@@ -119,7 +119,10 @@ module Libis
             result = pdf_to_pdfa(source, target)
           end
 
-          result
+          { 
+            files: [result]
+            converter: self.class.name
+          }
 
         end
 

data/lib/libis/format/converter/spreadsheet_converter.rb

@@ -31,9 +31,7 @@ module Libis
         def convert(source, target, format, opts = {})
           super
 
-          return nil unless Format::Tool::SpreadsheetToOds.run(source, target)
-
-          target
+          Format::Tool::SpreadsheetToOds.run(source, target)
 
         end
 

data/lib/libis/format/converter/video_converter.rb

@@ -1,5 +1,5 @@
 require_relative 'base'
-require 'libis/format/tool/ffmpeg'
+require 'libis/format/tool/ff_mpeg'
 
 require 'fileutils'
 
@@ -179,7 +179,10 @@ module Libis
 
           end
 
-          target
+          {
+            files: [target],
+            converter: self.class.name
+          }
 
         end
 

data/lib/libis/format/tool/email_to_pdf.rb

@@ -1,40 +1,75 @@
-require 'fileutils'
+require "fileutils"
 
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
+require "libis/tools/extend/string"
+require "libis/tools/logger"
+require "libis/tools/command"
 
-require 'libis/format/config'
+require "libis/format/config"
+require "rexml/document"
 
 module Libis
   module Format
     module Tool
-
       class EmailToPdf
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:email2pdf_cmd], "-v")
+          result[:status] == 0
+        end
+
         def self.run(source, target, options = {})
-          self.new.run source, target, options
+          new.run source, target, options
         end
 
         def run(source, target, _ = {})
           timeout = Libis::Format::Config[:timeouts][:email2pdf] || 120
           result = Libis::Tools::Command.run(
-              Libis::Format::Config[:email2pdf_cmd],
-              source,
-              target,
-              timeout: timeout,
-              kill_after: timeout * 2
+            Libis::Format::Config[:java_cmd],
+            "-Duser.timezone=Europe/Brussels -Duser.language=nl -Duser.country=BE",
+            "jar", Libis::Format::Config[:email2pdf_cmd],
+            "-e", "-hd", "-a",
+            "-o", target,
+            source,
+            timeout: timeout,
+            kill_after: timeout * 2
           )
 
-          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          warn "EmailToPdf conversion messages: \n\t#{result[:out].join("\n\t")}" unless result[:err].empty?
-          raise RuntimeError, "#{self.class} failed to generate target file #{target}" unless File.exist?(target)
+          warn "EmailToPdf conversion messages: \n\t#{result[:out].join("\n\t")}" unless result[:out].empty?
+
+          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise "#{self.class} failed to generate target file #{target}" unless File.exist?(target)
+          raise "#{self.class} command failed with status code #{result[:status]}" unless result[:status] == 0
+
+          base_path = File.join(File.dirname(target), File.basename(target, ".*"))
+          headers_file = "#{base_path}.headers.xml"
+          headers = read_header(headers_file)
 
-          nil
+          {
+            command: result,
+            files: [target, headers_file] + headers[:attachments].map { |a| File.join("#{base_path}-attachments", a) },
+            headers: headers
+          }
         end
-      end
 
+        private
+
+        def read_header(headers_file)
+          headers = {}
+          return headers unless File.exist?(headers_file)
+          doc = REXML::Document.new(File.new(headers_file))
+          root = doc.root
+          root.elements("/emailheader/*").each do |element|
+            case element.name
+            when "attachments"
+              headers[:attachments] = element.elements.map { |e| e.text }
+            else
+              headers[element.name.to_sym] = element.text
+            end
+          end
+          headers
+        end
+      end
     end
   end
 end

data/lib/libis/format/tool/{ffmpeg.rb → ff_mpeg.rb}

@@ -13,6 +13,11 @@ module Libis
         include Singleton
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:ffmpeg_cmd], "-h")
+          result[:status] == 0
+        end
+
         def self.run(source, target, options = {})
           self.instance.run source, target, options
         end
@@ -38,7 +43,11 @@ module Libis
 
           warn "FFMpeg warnings: #{(result[:err] + result[:out]).join("\n")}" unless result[:err].empty?
 
-          result[:out]
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
 
       end

data/lib/libis/format/tool/fop_pdf.rb

@@ -13,6 +13,12 @@ module Libis
       class FopPdf
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:fop_jar])
+        end
+
         def self.run(xml, target, options = [])
           self.new.run xml, target, options
         end
@@ -37,6 +43,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/office_to_pdf.rb

@@ -13,6 +13,11 @@ module Libis
       class OfficeToPdf
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:soffice_cmd], "--version")
+          result[:status] == 0
+        end
+
         def self.run(source, target, options = {})
           self.new.run source, target, options
         end
@@ -50,7 +55,11 @@ module Libis
         ensure
           FileUtils.rmtree workdir rescue nil
 
-          result[:out]
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_copy.rb

@@ -13,6 +13,12 @@ module Libis
       class PdfCopy
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:pdf_tool])
+        end
+
         def self.run(source, target, options = [])
           self.new.run source, target, options
         end
@@ -38,7 +44,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_merge.rb

@@ -13,6 +13,12 @@ module Libis
       class PdfMerge
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:pdf_tool])
+        end
+
         def self.run(source, target, options = [])
           self.new.run source, target, options
         end
@@ -39,7 +45,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_optimizer.rb

@@ -13,6 +13,11 @@ module Libis
       class PdfOptimizer
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:ghostscript_cmd], "--version")
+          result[:status] == 0
+        end
+
         def self.run(source, target, quality)
           self.new.run source, target, quality
         end
@@ -21,7 +26,7 @@ module Libis
 
           timeout = Libis::Format::Config[:timeouts][:pdf_optimizer]
           result = Libis::Tools::Command.run(
-              'gs',
+              Libis::Format::Config[:ghostscript_cmd],
               '-sDEVICE=pdfwrite',
               '-dCompatibilityLevel=1.4',
               "-dPDFSETTINGS=/#{quality}",
@@ -36,7 +41,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_split.rb

@@ -13,6 +13,12 @@ module Libis
       class PdfSplit
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:pdf_tool])
+        end
+
         def self.run(source, target, options = [])
           self.new.run source, target, options
         end
@@ -38,7 +44,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
-          result
+          {
+            command: result,
+            files: [ target ] # TODO: collect the files
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_to_pdfa.rb

@@ -16,6 +16,11 @@ module Libis
       class PdfToPdfa
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:ghostscript_cmd])
+          result == 0
+        end
+
         def self.run(source, target = nil, options = {})
           self.new.run source, target, options
         end
@@ -61,7 +66,11 @@ module Libis
             result[:err] << 'Failed to validate generated PDF/A file.'
           end
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
 
 

data/lib/libis/format/tool/spreadsheet_to_ods.rb

@@ -13,6 +13,11 @@ module Libis
       class SpreadsheetToOds
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:soffice_cmd], "--version")
+          result == 0
+        end
+
         def self.run(source, target, options = {})
           self.new.run source, target, options
         end
@@ -51,7 +56,11 @@ module Libis
         ensure
           FileUtils.rmtree workdir rescue nil
 
-          result[:out]
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool.rb

@@ -10,7 +10,7 @@ module Libis
       autoload :FileTool, 'libis/format/tool/file_tool'
 
       autoload :OfficeToPdf, 'libis/format/tool/office_to_pdf'
-      autoload :FFMpeg, 'libis/format/tool/ffmpeg'
+      autoload :FFMpeg, 'libis/format/tool/ff_mpeg'
       autoload :FopPdf, 'libis/format/tool/fop_pdf'
       autoload :PdfCopy, 'libis/format/tool/pdf_copy'
       autoload :PdfMerge, 'libis/format/tool/pdf_merge'

data/lib/libis/format/version.rb

@@ -1,5 +1,5 @@
 module Libis
   module Format
-    VERSION = '1.0.8'
+    VERSION = '1.1.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libis-format
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.1.0
 platform: ruby
 authors:
 - Kris Dekeyser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-18 00:00:00.000000000 Z
+date: 2023-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -228,7 +228,7 @@ files:
 - lib/libis/format/tool/droid.rb
 - lib/libis/format/tool/email_to_pdf.rb
 - lib/libis/format/tool/extension_identification.rb
-- lib/libis/format/tool/ffmpeg.rb
+- lib/libis/format/tool/ff_mpeg.rb
 - lib/libis/format/tool/fido.rb
 - lib/libis/format/tool/file_tool.rb
 - lib/libis/format/tool/fop_pdf.rb
@@ -248,6 +248,7 @@ files:
 - tools/PdfTool.jar
 - tools/bcpkix-jdk15on-1.49.jar
 - tools/bcprov-jdk15on-1.49.jar
+- tools/emailconverter.jar
 - tools/fop/build/fop.jar
 - tools/fop/conf/fop.xconf
 - tools/fop/fop