libis-format 1.0.7 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09143d3be9aca7d01b5c6ba3512e4b106d4ee2f0e370fad9c565bf0724c61846'
-  data.tar.gz: 5c9f5fb67527e828a2b293ffe6e4bf8616d7a08f67780b963e5f292bb50ae385
+  metadata.gz: 40c91211db466867a86d905e707302783d77f8ce96bce970229294fc3506f4dd
+  data.tar.gz: 3254048984ba29c9c6a1d556f25e7dc346ba1c0fc2082b80562c3ae64f41dc8f
 SHA512:
-  metadata.gz: cbf99230e3566a2aacdfd02e97f2622cd83d32601cb772fec50a94c04054754e367ea1a09276e7cdee6d9bc564f8642a63967159a2d3290769c368945fe3c233
-  data.tar.gz: c0f33d0d7d0208ea7ed8678da52dea0dd30ee54438c321a4be0a1b5ccc01f2451591251615775074639b03e4548d9a97cf4218e4ae0b8ce0aa2819e70f12a964
+  metadata.gz: caa9a7a5cbc6e99c7d1afb8af83274b2db4de847fde394d3d9cdc1129019044353bdbe6527fb4aabbcb8817be175e99f6bac1eeef0b79eb3e7ab94b7594db4ce
+  data.tar.gz: f3ade6093b38aab4b8c529c0d80e5a822aa2042264efc8d076077f74ee0cd20ebd91dbe5b2d6d030394607fcde196b28c8dabc2659788308e926981cca880905

data/.travis.yml

@@ -2,36 +2,51 @@ language: ruby
 sudo: false
 dist: trusty
 cache: bundler
+
 rvm:
-  - 2.1.10
-  - 2.2.8
-  - 2.3.5
-  - 2.4.2
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
   - ruby-head
-  - jruby-9.0.5.0
-  - jruby-9.1.9.0
+  - jruby-9.1
   - jruby-head
 jdk:
   - openjdk8
-  - oraclejdk8
+  - openjdk10
+  - openjdk11
 matrix:
   exclude:
-    - rvm: 2.1.10
-      jdk: oraclejdk8
-    - rvm: 2.2.8
-      jdk: oraclejdk8
-    - rvm: 2.3.5
-      jdk: oraclejdk8
-    - rvm: 2.4.2
-      jdk: oraclejdk8
+    - rvm: 2.3
+      jdk: openjdk10
+    - rvm: 2.3
+      jdk: openjdk11
+    - rvm: 2.4
+      jdk: openjdk10
+    - rvm: 2.4
+      jdk: openjdk11
+    - rvm: 2.5
+      jdk: openjdk10
+    - rvm: 2.5
+      jdk: openjdk11
+    - rvm: 2.6
+      jdk: openjdk10
+    - rvm: 2.6
+      jdk: openjdk11
     - rvm: ruby-head
-      jdk: oraclejdk8
+      jdk: openjdk10
+    - rvm: ruby-head
+      jdk: openjdk11
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
 branches:
   only:
       - master
+
+env:
+  - LC_ALL=en_IE.UTF-8
+
 before_install:
   - sudo add-apt-repository ppa:mc3man/trusty-media -y
   - sudo apt-get update -y -q
@@ -45,9 +60,6 @@ before_install:
   - sudo locale-gen "en_US.UTF-8"
   - sudo dpkg-reconfigure locales
   - echo "8\n9\n" | sudo dpkg-reconfigure tzdata
-env:
-  - LC_ALL=en_US.UTF-8
-  - LANG=en_US.UTF-8
 addons:
   apt:
     packages:

data/Gemfile

@@ -3,3 +3,5 @@ source 'https://rubygems.org'
 gem 'coveralls', group: :test, require: false
 
 gemspec name: 'libis-format', development_group: :test
+
+gem 'standard'

data/lib/libis/format/config.rb

@@ -21,6 +21,7 @@ module Libis
     Config[:fido_formats] = [(File.join(Libis::Format::DATA_DIR, 'lias_formats.xml'))]
     Config[:pdf_tool] = File.join(Libis::Format::TOOL_DIR, 'PdfTool.jar')
     Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
+    Config[:emai2pdf_jar] = File.join(Libis::Format::TOOL_DIR, 'emailconverter.jar')
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
     Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
@@ -34,6 +35,7 @@ module Libis
         fop: 5 * 60,
         identification_tool: 5 * 60,
         office_to_pdf: 5 * 60,
+        email2pdf: 5 * 60,
         pdf_copy: 5 * 60,
         pdf_merge: 5 * 60,
         pdf_optimizer: 5 * 60,

data/lib/libis/format/converter/audio_converter.rb

@@ -97,19 +97,18 @@ module Libis
 
           end
 
-          target
-
         end
 
         def assemble_and_convert(sources, target)
+          result = {}
           Tempfile.create(%w(list .txt)) do |f|
             sources.each {|src| f.puts src}
             opts[:global] ||= []
             opts[:global] += %w(-f concat)
             f.close
-            target = convert_file(f.to_path, target)
+            result = convert_file(f.to_path, target)
           end
-          target
+          result
         end
 
         def self.sounds_like(file1, file2, threshold, rate, channels)
@@ -155,6 +154,7 @@ module Libis
 
         def convert_file(source, target)
           opts = {global: [], input: [], filter: [], output: []}
+
           opts[:global] << '-hide_banner'
           opts[:global] << '-loglevel' << (@options[:quiet] ? 'fatal' : 'warning')
           opts[:output] << '-vn' # disable input video stream in case it exists
@@ -168,10 +168,8 @@ module Libis
           opts[:output] << '-ar' << @options[:sampling_freq] if @options[:sampling_freq]
           opts[:output] << '-ac' << @options[:channels] if @options[:channels]
           opts[:output] << '-f' << @options[:format] if @options[:format]
-          result = Libis::Format::Tool::FFMpeg.run(source, target, opts)
-          info "FFMpeg output: #{result}"
-          result
-          target
+
+          Libis::Format::Tool::FFMpeg.run(source, target, opts)
         end
 
       end

data/lib/libis/format/converter/chain.rb

@@ -68,14 +68,18 @@ module Libis
           end
 
           temp_files = []
+          xtra_files = []
+
+          result = { commands: [] }
 
           # noinspection RubyParenthesesAroundConditionInspection
-          result = @converter_chain.each_with_index do |node, i|
+          conversion_success = converter_chain.each_with_index do |node, i|
 
             target_type = node[:output]
             converter_class = node[:converter]
             converter = converter_class.new
 
+
             node[:operations].each do |operation|
               converter.send operation[:method], operation[:argument]
             end if node[:operations]
@@ -90,17 +94,25 @@ module Libis
 
             FileUtils.mkdir_p File.dirname(target)
 
-            src_file = converter.convert(src_file, target, target_type)
+            r = converter.convert(src_file, target, target_type)
+
+            src_file = r[:files].first
+            xtra_files += r[:files][1..]
+            break :failed unless src_file1
 
-            break :failed unless src_file
+            result[:commands] << r.merge(converter: converter_class.name)
+
+            :success
 
           end
 
+          result[:files] = [src_file] + xtra_files
+
           temp_files.each do |f|
             FileUtils.rm(f, force: true)
           end
 
-          result == :failed ? nil : target_file
+          conversion_success == :failed ? nil : result
 
         end
 

data/lib/libis/format/converter/email_converter.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+require_relative 'base'
+
+require 'libis/format/tool/email_to_pdf'
+require 'libis/format/type_database'
+require 'rexml/document'
+
+module Libis
+  module Format
+    module Converter
+
+      class EmailConverter < Libis::Format::Converter::Base
+
+        def self.input_types
+          [ :MSG ]
+        end
+
+        def self.output_types(format = nil)
+          return [] unless input_types.include?(format)
+          [ :PDF ]
+        end
+
+        def email_convert(_)
+          #force usage of this converter
+        end
+
+        def convert(source, target, format, opts = {})
+          super
+
+          Format::Tool::EmailToPdf.run(source, target)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter/fop_pdf_converter.rb

@@ -31,8 +31,6 @@ module Libis
 
           Libis::Format::Tool::FopPdf.run(source, target)
 
-          target
-
         end
 
       end

data/lib/libis/format/converter/image_converter.rb

@@ -180,9 +180,11 @@ module Libis
               convert_image(source, target, format)
             end
           end
-
-          target
-
+          
+          { 
+            files: [target],
+            converter: self.class.name
+          }
         end
 
         def assemble_and_convert(sources, target, format)

data/lib/libis/format/converter/jp2_converter.rb

@@ -90,16 +90,16 @@ module Libis
           end
 
 
-          Libis::Tools::Command.run(
+          result = Libis::Tools::Command.run(
               Libis::Format::Config[:j2k_cmd],
               '--input-file-name', source,
               '--set-output-type', 'JP2',
               *options,
               '--output-file-name', target,
-
           )
 
-          target
+          result.merge(files: [target], converter: self.class.name)
+
         end
       end
     end

data/lib/libis/format/converter/office_converter.rb

@@ -43,9 +43,7 @@ module Libis
         def convert(source, target, format, opts = {})
           super
 
-          return nil unless Format::Tool::OfficeToPdf.run(source, target)
-
-          target
+          Format::Tool::OfficeToPdf.run(source, target)
 
         end
 

data/lib/libis/format/converter/pdf_converter.rb

@@ -119,7 +119,10 @@ module Libis
             result = pdf_to_pdfa(source, target)
           end
 
-          result
+          { 
+            files: [result]
+            converter: self.class.name
+          }
 
         end
 

data/lib/libis/format/converter/spreadsheet_converter.rb

@@ -31,9 +31,7 @@ module Libis
         def convert(source, target, format, opts = {})
           super
 
-          return nil unless Format::Tool::SpreadsheetToOds.run(source, target)
-
-          target
+          Format::Tool::SpreadsheetToOds.run(source, target)
 
         end
 

data/lib/libis/format/converter/video_converter.rb

@@ -1,5 +1,5 @@
 require_relative 'base'
-require 'libis/format/tool/ffmpeg'
+require 'libis/format/tool/ff_mpeg'
 
 require 'fileutils'
 
@@ -179,7 +179,10 @@ module Libis
 
           end
 
-          target
+          {
+            files: [target],
+            converter: self.class.name
+          }
 
         end
 

data/lib/libis/format/tool/email_to_pdf.rb

@@ -0,0 +1,75 @@
+require "fileutils"
+
+require "libis/tools/extend/string"
+require "libis/tools/logger"
+require "libis/tools/command"
+
+require "libis/format/config"
+require "rexml/document"
+
+module Libis
+  module Format
+    module Tool
+      class EmailToPdf
+        include ::Libis::Tools::Logger
+
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:email2pdf_cmd], "-v")
+          result[:status] == 0
+        end
+
+        def self.run(source, target, options = {})
+          new.run source, target, options
+        end
+
+        def run(source, target, _ = {})
+          timeout = Libis::Format::Config[:timeouts][:email2pdf] || 120
+          result = Libis::Tools::Command.run(
+            Libis::Format::Config[:java_cmd],
+            "-Duser.timezone=Europe/Brussels -Duser.language=nl -Duser.country=BE",
+            "jar", Libis::Format::Config[:email2pdf_cmd],
+            "-e", "-hd", "-a",
+            "-o", target,
+            source,
+            timeout: timeout,
+            kill_after: timeout * 2
+          )
+
+          warn "EmailToPdf conversion messages: \n\t#{result[:out].join("\n\t")}" unless result[:out].empty?
+
+          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise "#{self.class} failed to generate target file #{target}" unless File.exist?(target)
+          raise "#{self.class} command failed with status code #{result[:status]}" unless result[:status] == 0
+
+          base_path = File.join(File.dirname(target), File.basename(target, ".*"))
+          headers_file = "#{base_path}.headers.xml"
+          headers = read_header(headers_file)
+
+          {
+            command: result,
+            files: [target, headers_file] + headers[:attachments].map { |a| File.join("#{base_path}-attachments", a) },
+            headers: headers
+          }
+        end
+
+        private
+
+        def read_header(headers_file)
+          headers = {}
+          return headers unless File.exist?(headers_file)
+          doc = REXML::Document.new(File.new(headers_file))
+          root = doc.root
+          root.elements("/emailheader/*").each do |element|
+            case element.name
+            when "attachments"
+              headers[:attachments] = element.elements.map { |e| e.text }
+            else
+              headers[element.name.to_sym] = element.text
+            end
+          end
+          headers
+        end
+      end
+    end
+  end
+end

data/lib/libis/format/tool/{ffmpeg.rb → ff_mpeg.rb}

@@ -13,6 +13,11 @@ module Libis
         include Singleton
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:ffmpeg_cmd], "-h")
+          result[:status] == 0
+        end
+
         def self.run(source, target, options = {})
           self.instance.run source, target, options
         end
@@ -38,7 +43,11 @@ module Libis
 
           warn "FFMpeg warnings: #{(result[:err] + result[:out]).join("\n")}" unless result[:err].empty?
 
-          result[:out]
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
 
       end

data/lib/libis/format/tool/fop_pdf.rb

@@ -13,6 +13,12 @@ module Libis
       class FopPdf
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:fop_jar])
+        end
+
         def self.run(xml, target, options = [])
           self.new.run xml, target, options
         end
@@ -37,6 +43,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/office_to_pdf.rb

@@ -13,6 +13,11 @@ module Libis
       class OfficeToPdf
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:soffice_cmd], "--version")
+          result[:status] == 0
+        end
+
         def self.run(source, target, options = {})
           self.new.run source, target, options
         end
@@ -50,7 +55,11 @@ module Libis
         ensure
           FileUtils.rmtree workdir rescue nil
 
-          result[:out]
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_copy.rb

@@ -13,6 +13,12 @@ module Libis
       class PdfCopy
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:pdf_tool])
+        end
+
         def self.run(source, target, options = [])
           self.new.run source, target, options
         end
@@ -38,7 +44,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_merge.rb

@@ -13,6 +13,12 @@ module Libis
       class PdfMerge
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:pdf_tool])
+        end
+
         def self.run(source, target, options = [])
           self.new.run source, target, options
         end
@@ -39,7 +45,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_optimizer.rb

@@ -13,6 +13,11 @@ module Libis
       class PdfOptimizer
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:ghostscript_cmd], "--version")
+          result[:status] == 0
+        end
+
         def self.run(source, target, quality)
           self.new.run source, target, quality
         end
@@ -21,7 +26,7 @@ module Libis
 
           timeout = Libis::Format::Config[:timeouts][:pdf_optimizer]
           result = Libis::Tools::Command.run(
-              'gs',
+              Libis::Format::Config[:ghostscript_cmd],
               '-sDEVICE=pdfwrite',
               '-dCompatibilityLevel=1.4',
               "-dPDFSETTINGS=/#{quality}",
@@ -36,7 +41,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_split.rb

@@ -13,6 +13,12 @@ module Libis
       class PdfSplit
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:java_cmd], "-version")
+          return false unless result[:status] == 0
+          File.exist?(Libis::Format::Config[:pdf_tool])
+        end
+
         def self.run(source, target, options = [])
           self.new.run source, target, options
         end
@@ -38,7 +44,11 @@ module Libis
           raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
-          result
+          {
+            command: result,
+            files: [ target ] # TODO: collect the files
+          }
+
         end
       end
 

data/lib/libis/format/tool/pdf_to_pdfa.rb

@@ -16,6 +16,11 @@ module Libis
       class PdfToPdfa
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:ghostscript_cmd])
+          result == 0
+        end
+
         def self.run(source, target = nil, options = {})
           self.new.run source, target, options
         end
@@ -61,7 +66,11 @@ module Libis
             result[:err] << 'Failed to validate generated PDF/A file.'
           end
 
-          result
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
 
 

data/lib/libis/format/tool/spreadsheet_to_ods.rb

@@ -13,6 +13,11 @@ module Libis
       class SpreadsheetToOds
         include ::Libis::Tools::Logger
 
+        def self.installed?
+          result = Libis::Tools::Command.run(Libis::Format::Config[:soffice_cmd], "--version")
+          result == 0
+        end
+
         def self.run(source, target, options = {})
           self.new.run source, target, options
         end
@@ -51,7 +56,11 @@ module Libis
         ensure
           FileUtils.rmtree workdir rescue nil
 
-          result[:out]
+          {
+            command: result,
+            files: [ target ]
+          }
+
         end
       end
 

data/lib/libis/format/tool.rb

@@ -10,7 +10,7 @@ module Libis
       autoload :FileTool, 'libis/format/tool/file_tool'
 
       autoload :OfficeToPdf, 'libis/format/tool/office_to_pdf'
-      autoload :FFMpeg, 'libis/format/tool/ffmpeg'
+      autoload :FFMpeg, 'libis/format/tool/ff_mpeg'
       autoload :FopPdf, 'libis/format/tool/fop_pdf'
       autoload :PdfCopy, 'libis/format/tool/pdf_copy'
       autoload :PdfMerge, 'libis/format/tool/pdf_merge'

data/lib/libis/format/version.rb

@@ -1,5 +1,5 @@
 module Libis
   module Format
-    VERSION = '1.0.7'
+    VERSION = '1.1.0'
   end
 end

data/libis-format.gemspec

@@ -4,6 +4,7 @@ lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'libis/format/version'
+require 'bundler'
 
 Gem::Specification.new do |spec|
   spec.name          = 'libis-format'
@@ -26,7 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.1'
   spec.add_development_dependency 'awesome_print'
   spec.add_development_dependency 'equivalent-xml', '~> 0.5'
-  if spec.platform == Gem::Platform::JAVA
+  if Gem::Platform::JAVA && spec.platform == Gem::Platform::JAVA
     spec.add_development_dependency 'saxon-xslt'
   else
     spec.add_development_dependency 'nokogiri'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libis-format
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.1.0
 platform: ruby
 authors:
 - Kris Dekeyser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-13 00:00:00.000000000 Z
+date: 2023-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -213,6 +213,7 @@ files:
 - lib/libis/format/converter/audio_converter.rb
 - lib/libis/format/converter/base.rb
 - lib/libis/format/converter/chain.rb
+- lib/libis/format/converter/email_converter.rb
 - lib/libis/format/converter/fop_pdf_converter.rb
 - lib/libis/format/converter/image_converter.rb
 - lib/libis/format/converter/jp2_converter.rb
@@ -225,8 +226,9 @@ files:
 - lib/libis/format/identifier.rb
 - lib/libis/format/tool.rb
 - lib/libis/format/tool/droid.rb
+- lib/libis/format/tool/email_to_pdf.rb
 - lib/libis/format/tool/extension_identification.rb
-- lib/libis/format/tool/ffmpeg.rb
+- lib/libis/format/tool/ff_mpeg.rb
 - lib/libis/format/tool/fido.rb
 - lib/libis/format/tool/file_tool.rb
 - lib/libis/format/tool/fop_pdf.rb
@@ -246,6 +248,7 @@ files:
 - tools/PdfTool.jar
 - tools/bcpkix-jdk15on-1.49.jar
 - tools/bcprov-jdk15on-1.49.jar
+- tools/emailconverter.jar
 - tools/fop/build/fop.jar
 - tools/fop/conf/fop.xconf
 - tools/fop/fop
@@ -317,8 +320,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: LIBIS File format format services.