libis-format 1.2.9 → 2.0.0

metadata

@@ -1,101 +1,87 @@
 --- !ruby/object:Gem::Specification
 name: libis-format
 version: !ruby/object:Gem::Version
-  version: 1.2.9
+  version: 2.0.0
 platform: ruby
 authors:
 - Kris Dekeyser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-11-27 00:00:00.000000000 Z
+date: 2020-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: chromaprint
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.2
-- !ruby/object:Gem::Dependency
-  name: deep_dive
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
-  type: :runtime
+        version: '10.3'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '10.3'
 - !ruby/object:Gem::Dependency
-  name: libis-mapi
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
-  type: :runtime
+        version: '3.1'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
-  name: libis-tools
+  name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: mini_magick
+  name: equivalent-xml
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.12'
-  type: :runtime
+        version: '0.5'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.12'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: naturally
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.2'
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: new_rfc_2047
+  name: libis-tools
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -112,142 +98,109 @@ dependencies:
   name: os
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 0.9.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 0.9.6
 - !ruby/object:Gem::Dependency
-  name: pdfinfo
+  name: mini_magick
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '4.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '4.3'
 - !ruby/object:Gem::Dependency
-  name: pdfkit
+  name: deep_dive
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
-- !ruby/object:Gem::Dependency
-  name: awesome_print
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: equivalent-xml
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: chromaprint
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 0.0.2
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.2
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: naturally
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '2.1'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: Collection of tools and classes that help to identify file formats and
-  create derivative copies.
+        version: '2.1'
+description: Collection of tools and classes that help to identify formats of binary
+  files and create derivative copies (e.g. PDF from Word).
 email:
 - kris.dekeyser@libis.be
 executables:
-- droid_tool
-- fido_tool
+- droid
+- fido
 - formatinfo
 - libis_format
 - pdf_copy
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
-- bin/droid_tool
-- bin/fido_tool
+- Rakefile
+- base/Dockerfile
+- base/Dockerfile.alpine
+- base/Dockerfile.rvm
+- base/rework_path
+- bin/droid
+- bin/fido
 - bin/formatinfo
 - bin/libis_format
 - bin/pdf_copy
-- data/AdobeRGB1998.icc
 - data/ISOcoated_v2_eci.icc
 - data/PDFA_def.ps
 - data/ead.xsd
+- data/eciRGB_v2.icc
 - data/lias_formats.xml
 - data/types.yml
 - data/xlink.xsd
+- docker_cfg.yml
 - lib/libis-format.rb
 - lib/libis/format.rb
 - lib/libis/format/cli/convert.rb
@@ -260,26 +213,33 @@ files:
 - lib/libis/format/converter/audio_converter.rb
 - lib/libis/format/converter/base.rb
 - lib/libis/format/converter/chain.rb
-- lib/libis/format/converter/email_converter.rb
 - lib/libis/format/converter/fop_pdf_converter.rb
+- lib/libis/format/converter/image_assembler.rb
 - lib/libis/format/converter/image_converter.rb
+- lib/libis/format/converter/image_splitter.rb
+- lib/libis/format/converter/image_watermarker.rb
 - lib/libis/format/converter/jp2_converter.rb
 - lib/libis/format/converter/office_converter.rb
+- lib/libis/format/converter/pdf_assembler.rb
 - lib/libis/format/converter/pdf_converter.rb
+- lib/libis/format/converter/pdf_optimizer.rb
+- lib/libis/format/converter/pdf_splitter.rb
+- lib/libis/format/converter/pdf_watermarker.rb
 - lib/libis/format/converter/repository.rb
 - lib/libis/format/converter/spreadsheet_converter.rb
 - lib/libis/format/converter/video_converter.rb
 - lib/libis/format/converter/xslt_converter.rb
 - lib/libis/format/identifier.rb
+- lib/libis/format/info.rb
+- lib/libis/format/library.rb
 - lib/libis/format/tool.rb
 - lib/libis/format/tool/droid.rb
 - lib/libis/format/tool/extension_identification.rb
-- lib/libis/format/tool/ff_mpeg.rb
+- lib/libis/format/tool/ffmpeg.rb
 - lib/libis/format/tool/fido.rb
 - lib/libis/format/tool/file_tool.rb
 - lib/libis/format/tool/fop_pdf.rb
 - lib/libis/format/tool/identification_tool.rb
-- lib/libis/format/tool/msg_to_pdf.rb
 - lib/libis/format/tool/office_to_pdf.rb
 - lib/libis/format/tool/pdf_copy.rb
 - lib/libis/format/tool/pdf_merge.rb
@@ -288,9 +248,8 @@ files:
 - lib/libis/format/tool/pdf_to_pdfa.rb
 - lib/libis/format/tool/pdfa_validator.rb
 - lib/libis/format/tool/spreadsheet_to_ods.rb
-- lib/libis/format/type_database.rb
-- lib/libis/format/type_database_impl.rb
 - lib/libis/format/version.rb
+- lib/libis/format/yaml_loader.rb
 - libis-format.gemspec
 - tools/PdfTool.jar
 - tools/bcpkix-jdk15on-1.49.jar
@@ -345,7 +304,6 @@ files:
 - tools/fop/lib/xmlgraphics-commons-2.3.jar
 - tools/fop/lib/xmlgraphics-commons.LICENSE.txt
 - tools/fop/lib/xmlgraphics-commons.NOTICE.txt
-- tools/pdf2pdfa
 - tools/pdfbox/pdfbox-app-2.0.13.jar
 - tools/pdfbox/preflight-app-2.0.13.jar
 homepage: ''
@@ -360,14 +318,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: LIBIS File format format services.

data/lib/libis/format/converter/email_converter.rb

@@ -1,38 +0,0 @@
-# encoding: utf-8
-
-require_relative 'base'
-
-require 'libis/format/tool/msg_to_pdf'
-require 'libis/format/type_database'
-require 'rexml/document'
-
-module Libis
-  module Format
-    module Converter
-
-      class EmailConverter < Libis::Format::Converter::Base
-
-        def self.input_types
-          [ :MSG ]
-        end
-
-        def self.output_types(format = nil)
-          return [] unless input_types.include?(format)
-          [ :PDF ]
-        end
-
-        def email_convert(_)
-          #force usage of this converter
-        end
-
-        def convert(source, target, format, opts = {})
-          super
-
-          Format::Tool::MsgToPdf.run(source, target)
-        end
-
-      end
-
-    end
-  end
-end

data/lib/libis/format/tool/msg_to_pdf.rb

@@ -1,270 +0,0 @@
-# frozen_string_literal: true
-
-require 'mapi/msg'
-require 'rfc_2047'
-require 'cgi'
-require 'pdfkit'
-require 'time'
-require 'fileutils'
-require 'pathname'
-require 'libis/format/config'
-
-module Libis
-  module Format
-    module Tool
-      class MsgToPdf
-        include ::Libis::Tools::Logger
-
-        HEADER_STYLE = '<style>.header-table {margin: 0 0 20 0;padding: 0;font-family: Arial, Helvetica, sans-serif;}.header-name {padding-right: 5px;color: #9E9E9E;text-align: right;vertical-align: top;font-size: 12px;}.header-value {font-size: 12px;}#header_fields {#background: white;#margin: 0;#border: 1px solid #DDD;#border-radius: 3px;#padding: 8px;#width: 100%%;#box-sizing: border-box;#}</style><script type="text/javascript">function timer() {try {parent.postMessage(Math.max(document.body.offsetHeight, document.body.scrollHeight), \'*\');} catch (r) {}setTimeout(timer, 10);};timer();</script>' # rubocop:disable Layout/LineLength
-        HEADER_TABLE_TEMPLATE = '<div class="header-table"><table id="header_fields"><tbody>%s</tbody></table></div>'
-        HEADER_FIELD_TEMPLATE = '<tr><td class="header-name">%s</td><td class="header-value">%s</td></tr>'
-        HTML_WRAPPER_TEMPLATE = '<!DOCTYPE html><html><head><style>body {font-size: 0.5cm;}</style><title>title</title></head><body>%s</body></html>' # rubocop:disable Layout/LineLength
-
-        IMG_CID_PLAIN_REGEX = /\[cid:(.*?)\]/m
-        IMG_CID_HTML_REGEX = /cid:([^"]*)/m
-
-        def self.installed?
-          File.exist?(Libis::Format::Config[:wkhtmltopdf])
-        end
-
-        def self.run(source, target, **options)
-          new.run source, target, **options
-        end
-
-        def run(source, target, **options)
-          # Preliminary checks
-          # ------------------
-
-          @warnings = []
-
-          # Check if source file exists
-          raise "File #{source} does not exist" unless File.exist?(source)
-
-          # Retrieving the message
-          # ----------------------
-
-          # Open the message
-          msg = Mapi::Msg.open(source)
-
-          target_format = options.delete(:to_html) ? :HTML : :PDF
-          result = msg_to_pdf(msg, target, target_format, options)
-          msg.close
-          result
-        end
-
-        def msg_to_pdf(msg, target, target_format, pdf_options, root_msg: true)
-          # Make sure the target directory exists
-          outdir = File.dirname(target)
-          FileUtils.mkdir_p(outdir)
-
-          # Get the body of the message in HTML
-          body = msg.properties.body_html
-
-          # Embed plain body in HTML as a fallback
-          body ||= HTML_WRAPPER_TEMPLATE % msg.properties.body
-
-          # Check and fix the character encoding
-          begin
-            # Try to encode into UTF-8
-            body.encode!('UTF-8', universal_newline: true)
-          rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
-            begin
-              # If it fails, the text may be in Windows' Latin1 (ISO-8859-1)
-              body.force_encoding('ISO-8859-1').encode!('UTF-8', universal_newline: true)
-            rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError => e
-              # If that fails too, log a warning and replace the invalid/unknown with a ? character.
-              @warnings << "#{e.class}: #{e.message}"
-              body.encode!('UTF-8', universal_newline: true, invalid: :replace, undef: :replace)
-            end
-          end
-
-          # Process headers
-          # ---------------
-          headers = {}
-          hdr_html = ''
-
-          %w[From To Cc Subject Date].each do |key|
-            value = find_hdr(msg.headers, key)
-            if value
-              headers[key.downcase.to_sym] = value
-              hdr_html += hdr_html(key, value)
-            end
-          end
-
-          [:date].each do |key|
-            next unless headers[key]
-
-            headers[key] = DateTime.parse(headers[key]).to_time.localtime.iso8601
-          end
-
-          # Add header section to the HTML body
-          unless hdr_html.empty?
-            # Insert header block styles
-            if body =~ %r{</head>}
-              # if head exists, append the style block
-              body.gsub!(%r{</head>}, "#{HEADER_STYLE}</head>")
-            elsif body =~ %r{<head/>}
-              # empty head, replace with the style block
-              body.gsub!(%r{<head/>}, "<head>#{HEADER_STYLE}</head>")
-            else
-              # otherwise insert a head section before the body tag
-              body.gsub!(/<body/, "<head>#{HEADER_STYLE}</head><body")
-            end
-            # Add the headers html table as first element in the body section
-            body.gsub!(/<body[^>]*>/) { |m| "#{m}#{HEADER_TABLE_TEMPLATE % hdr_html}" }
-          end
-
-          # Embed inline images
-          # -------------------
-          attachments = msg.attachments
-          used_files = []
-
-          # First process plaintext cid entries
-          body.gsub!(IMG_CID_PLAIN_REGEX) do |_match|
-            data = get_attachment_data(attachments, ::Regexp.last_match(1))
-            if data
-              used_files << ::Regexp.last_match(1)
-              "<img src=\"data:#{data[:mime_type]};base64,#{data[:base64]}\"/>"
-            else
-              '<img src=""/>'
-            end
-          end
-
-          # Then process HTML img tags with CID entries
-          body.gsub!(IMG_CID_HTML_REGEX) do |_match|
-            data = get_attachment_data(attachments, ::Regexp.last_match(1))
-            if data
-              used_files << ::Regexp.last_match(1)
-              "data:#{data[:mime_type]};base64,#{data[:base64]}"
-            else
-              ''
-            end
-          end
-
-          # Create PDF
-          # ----------
-          files = []
-
-          if target_format == :PDF
-            # PDF creation options
-            pdf_options = {
-              page_size: 'A4',
-              margin_top: '10mm',
-              margin_bottom: '10mm',
-              margin_left: '10mm',
-              margin_right: '10mm',
-              # image_quality: 100,
-              # viewport_size: '2480x3508',
-              dpi: 300
-            }.merge pdf_options
-
-            subject = find_hdr(msg.headers, 'Subject')
-            kit = PDFKit.new(body, title: (subject || 'message'), **pdf_options)
-            pdf = kit.to_pdf
-            File.open(target, 'wb') { |f| f.write(pdf) }
-          else
-            File.open(target, 'wb') { |f| f.write(body) }
-          end
-          files << target if File.exist?(target)
-
-          # Save attachments
-          # ----------------
-          outdir = File.join(outdir, "#{File.basename(target)}.attachments")
-          digits = ((attachments.count + 1) / 10) + 1
-          i = 1
-          attachments.delete_if { |a| a.properties.attachment_hidden }.each do |a|
-            prefix = "#{format('%0*d', digits, i)}-"
-            if (sub_msg = a.instance_variable_get(:@embedded_msg))
-              subject = a.properties[:display_name] || sub_msg.subject || ''
-              file = File.join(outdir, "#{prefix}#{subject}.msg.#{target_format.to_s.downcase}")
-              result = msg_to_pdf(sub_msg, file, target_format, pdf_options, root_msg: false)
-              if (e = result[:error])
-                raise e
-              end
-
-              files += result[:files]
-            elsif a.filename
-              next if used_files.include?(a.filename)
-
-              file = File.join(outdir, "#{prefix}#{a.filename}")
-              FileUtils.mkdir_p(File.dirname(file))
-              File.open(file, 'wb') { |f| a.save(f) }
-              files << file
-            else
-              @warnings << "Attachment #{a.properties[:display_name]} cannot be extracted"
-              next
-            end
-            i += 1
-          end
-
-          if root_msg
-            p = Pathname(File.dirname(files.first))
-            files[1..].each do |f|
-              (headers[:attachments] ||= []) << Pathname.new(f).relative_path_from(p).to_s
-            end
-          end
-
-          {
-            command: { status: 0 },
-            files:,
-            headers:,
-            warnings: @warnings
-          }
-        rescue Exception => e
-          raise unless root_msg
-
-          msg.close
-          {
-            command: { status: -1 },
-            files: [],
-            headers: {},
-            errors: [
-              {
-                error: e.message,
-                error_class: e.class.name,
-                error_trace: e.backtrace
-              }
-            ],
-            warnings: @warnings
-          }
-        end
-
-        protected
-
-        def eml_to_html; end
-
-        private
-
-        def find_hdr(list, key)
-          keys = list.keys
-          if (k = keys.find { |x| x.to_s =~ /^#{key}$/i })
-            v = list[k]
-            v = v.first if v.is_a? Array
-            v = Rfc2047.decode(v).strip if v.is_a? String
-            return v
-          end
-          nil
-        end
-
-        def hdr_html(key, value)
-          return format(HEADER_FIELD_TEMPLATE, key, CGI.escapeHTML(value)) if key.is_a?(String) && value.is_a?(String) && !value.empty?
-
-          ''
-        end
-
-        def get_attachment_data(attachments, cid)
-          attachments.each do |attachment|
-            next unless attachment.properties.attach_content_id == cid
-
-            attachment.data.rewind
-            return {
-              mime_type: attachment.properties.attach_mime_tag,
-              base64: Base64.encode64(attachment.data.read).gsub(/[\r\n]/, '')
-            }
-          end
-          nil
-        end
-      end
-    end
-  end
-end