libis-format 1.2.3 → 1.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bb468afbfdd7031a466736c6d355919395bf295356032f6b3d3488908bb61b5
-  data.tar.gz: b2ddc4588f0f3a1ac4ddde120ce726af21ab9013c98bae303674dca2033fec58
+  metadata.gz: 19055f943473fa0d2a9a1fdebf499c87652fcab845239a4b1f21341229ad08ad
+  data.tar.gz: f6f64bcaf5472f0e11aceb2cbe97b3f367b0ed4ccc4dfc389e5857f191c59897
 SHA512:
-  metadata.gz: 176685c4eb8fac9734666928c7d22b18cc870158b4a45151b35f5cc81922d4f6f0521af026719efb04d9b3c3920ae887c271739b91ade104e5d1f2e6651e8f3e
-  data.tar.gz: f3ca3c913aef08d75ff1dadef3790e7ad6838429bc91dbcff3173391f8d052d3f23ef9bcf3309c4c062650064947821cc046431cfe1196644964c04236cff6a7
+  metadata.gz: 7b0ee24177c3f541a8cd12733e04a04649d03b2785c68c8eaa44ca3fdebd7c657346afc9a824d59e872666bfa3a657483b00b6126970cc739ef8433889621847
+  data.tar.gz: 0bdbc5a928d6e556a816f65d05f2151e6b6ac331de37ce2d12030ace291f2b4cae452420236d9d5371f3566a7346f60854976dc33181e17549663b8ada5302f6

data/.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "rdbg",
+            "name": "Debug current file with rdbg",
+            "request": "launch",
+            "script": "${file}",
+            "args": [],
+            "askParameters": true
+        },
+        {
+            "type": "rdbg",
+            "name": "Attach with rdbg",
+            "request": "attach"
+        }
+    ]
+}

data/Rakefile

@@ -1,6 +1,10 @@
+require 'rake'
+
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 
+require_relative 'spec/spec_helper'
+
 RSpec::Core::RakeTask.new
 
 task :default => :spec

data/lib/libis/format/config.rb

@@ -1,10 +1,10 @@
 # encoding: utf-8
 require 'os'
+require 'libis-tools'
 
 module Libis
   module Format
 
-    # noinspection RubyConstantNamingConvention
     Config = ::Libis::Tools::Config
 
     Config[:converter_chain_max_level] = 8
@@ -21,8 +21,7 @@ module Libis
     Config[:fido_formats] = [(File.join(Libis::Format::DATA_DIR, 'lias_formats.xml'))]
     Config[:pdf_tool] = File.join(Libis::Format::TOOL_DIR, 'PdfTool.jar')
     Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
-    Config[:email2pdf_jar] = File.join(Libis::Format::TOOL_DIR, 'emailconverter.jar')
-    # noinspection RubyStringKeysInHashInspection
+    Config[:wkhtmltopdf] = 'wkhtmltopdf'
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
     Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
     Config[:raw_audio_convert_cmd] = 'sox -V1 %s -e signed -b 16 -t wav %s rate %d channels %d'

data/lib/libis/format/converter/email_converter.rb

@@ -2,7 +2,7 @@
 
 require_relative 'base'
 
-require 'libis/format/tool/email_to_pdf'
+require 'libis/format/tool/msg_to_pdf'
 require 'libis/format/type_database'
 require 'rexml/document'
 
@@ -28,7 +28,7 @@ module Libis
         def convert(source, target, format, opts = {})
           super
 
-          Format::Tool::EmailToPdf.run(source, target)
+          Format::Tool::MsgToPdf.run(source, target)
         end
 
       end

data/lib/libis/format/tool/msg_to_pdf.rb

@@ -0,0 +1,298 @@
+# encoding: utf-8
+
+require 'mapi/msg'
+require 'rfc_2047'
+require 'cgi'
+require 'pdfkit'
+
+require 'fileutils'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+    module Tool
+      class MsgToPdf
+        include ::Libis::Tools::Logger
+
+        HEADER_STYLE = '<style>.header-table {margin: 0 0 20 0;padding: 0;font-family: Arial, Helvetica, sans-serif;}.header-name {padding-right: 5px;color: #9E9E9E;text-align: right;vertical-align: top;font-size: 12px;}.header-value {font-size: 12px;}#header_fields {background: white;margin: 0;border: 1px solid #DDD;border-radius: 3px;padding: 8px;width: 100%%;box-sizing: border-box;}</style><script type="text/javascript">function timer() {try {parent.postMessage(Math.max(document.body.offsetHeight, document.body.scrollHeight), \'*\');} catch (r) {}setTimeout(timer, 10);};timer();</script>'
+        HEADER_TABLE_TEMPLATE = '<div class="header-table"><table id="header_fields"><tbody>%s</tbody></table></div>'
+        HEADER_FIELD_TEMPLATE = '<tr><td class="header-name">%s</td><td class="header-value">%s</td></tr>'
+        HTML_WRAPPER_TEMPLATE = '<!DOCTYPE html><html><head><style>body {font-size: 0.5cm;}</style><title>title</title></head><body>%s</body></html>'
+
+        IMG_CID_PLAIN_REGEX = %r/\[cid:(.*?)\]/m
+        IMG_CID_HTML_REGEX = %r/cid:([^"]*)/m
+
+        def self.installed?
+          File.exist?(Libis::Format::Config[:wkhtmltopdf])
+        end
+
+        def self.run(source, target, options = {})
+          new.run source, target, options
+        end
+
+        def run(source, target, options = {})
+          # Preliminary checks
+          # ------------------
+
+          @warnings = []
+
+          # Check if source file exists
+          raise "File #{source} does not exist" unless File.exist?(source)
+
+          # Retrieving the message
+          # ----------------------
+
+          # Open the message
+          msg = Mapi::Msg.open(source)
+
+          target_format = options.delete(:to_html) ? :HTML : :PDF
+          result = msg_to_pdf(msg, target, target_format, options)
+          msg.close
+          return result
+        end
+
+        def msg_to_pdf(msg, target, target_format, pdf_options, reraise: false)
+
+          # Make sure the target directory exists
+          outdir = File.dirname(target)
+          FileUtils.mkdir_p(outdir)
+
+# puts "Headers:"
+# puts '--------'
+# pp msg.headers
+
+# puts "Recipients:"
+# puts '-----------'
+# pp msg.recipients
+
+# puts "Body:"
+# puts '-----'
+# puts msg.properties.body
+# puts '-----'
+# puts msg.properties.body_rtf
+# puts '-----'
+# puts msg.properties.body_html
+
+# puts "Attachments:"
+# puts '------------'
+# msg.attachments.each {|a| p "#{a.filename} - #{a.properties.attach_content_id}"}
+
+# puts "Converting:"
+# puts '-----------'
+
+          # Get the body of the message in HTML
+          body = msg.properties.body_html
+          body ||= begin
+            # Embed plain body in HTML as a fallback
+            HTML_WRAPPER_TEMPLATE % msg.properties.body
+          end
+
+          # Check and fix the character encoding
+          begin
+            # Try to encode into UTF-8
+            body.encode!('UTF-8', universal_newline: true)
+          rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
+            begin
+            # If it fails, the text may be in Windows' Latin1 (ISO-8859-1)
+            body.force_encoding('ISO-8859-1').encode!('UTF-8', universal_newline: true)
+            rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError => e
+              # If that fails too, log a warning and replace the invalid/unknown with a ? character.
+              @warnings << "#{e.class}: #{e.message}"
+              body.encode!('UTF-8', universal_newline: true, invalid: :replace, undef: :replace)
+            end
+          end
+
+          # Process headers
+          # ---------------
+          headers = {}
+          hdr_html = ''
+
+          %w"From To Cc Subject Date".each do |key|
+            value = find_hdr(msg.headers, key)
+            if value
+              headers[key.downcase.to_sym] = value
+              hdr_html += hdr_html(key, value)
+            end
+          end
+
+          # Add header section to the HTML body
+          unless hdr_html.empty?
+            # Insert header block styles
+            if body =~ /<\/head>/
+              # if head exists, append the style block
+              body.gsub!(/<\/head>/, HEADER_STYLE + '</head>')
+            else
+              # otherwise insert a head section before the body tag
+              body.gsub!(/<body/, '<head>' + HEADER_STYLE + '</head><body')
+            end
+            # Add the headers html table as first element in the body section
+            body.gsub!(/<body[^>]*>/) {|m| "#{m}#{HEADER_TABLE_TEMPLATE % hdr_html}"}
+          end
+
+          # Embed inline images
+          # -------------------
+          attachments = msg.attachments
+          used_files = []
+
+          # First process plaintext cid entries
+          body.gsub!(IMG_CID_PLAIN_REGEX) do |match|
+# puts "CID found: #{match}, looking for #{$1}"
+            data = getAttachmentData(attachments, $1)
+            unless data
+# puts "cid #{$1} not found"
+              return '<img src=""/>'
+            end
+# puts "cid #{$1} data: #{data.inspect}"
+            used_files << $1
+            "<img src=\"data:#{data[:mime_type]};base64,#{data[:base64]}\"/>"
+          end
+          
+          # Then process HTML img tags with CID entries
+          body.gsub!(IMG_CID_HTML_REGEX) do |match|
+# puts "CID found: #{match}, looking for #{$1}"
+            data = getAttachmentData(attachments, $1)
+            unless data
+# puts "cid #{$1} not found"
+              return ''
+            end
+# puts "cid #{$1} data: #{data.inspect}"
+            used_files << $1
+            "data:#{data[:mime_type]};base64,#{data[:base64]}"
+          end
+
+          # Create PDF
+          # ----------
+          files = []
+
+          if target_format == :PDF
+          # PDF creation options
+            pdf_options = {
+              page_size: 'A4',
+              margin_top: '10mm',
+              margin_bottom: '10mm',
+              margin_left: '10mm',
+              margin_right: '10mm',
+              dpi: 300,
+              # image_quality: 100,
+              # viewport_size: '2480x3508',
+            }.merge pdf_options
+
+# pp pdf_options
+# puts "Final HTML body:"
+# pp body
+            subject = find_hdr(msg.headers, 'Subject')
+            kit = PDFKit.new(body, title: (subject || 'message'), **pdf_options)
+            pdf = kit.to_pdf
+            File.open(target, 'wb') {|f| f.write(pdf)}
+# puts "message #{subject} converted to PDF file '#{target}'"
+          else
+            File.open(target, 'wb') {|f| f.write(body)}
+# puts "message #{subject} converted to HTML file '#{target}'"
+          end
+          files << target if File.exist?(target)
+
+          # Save attachments
+          # ----------------
+          outdir = File.join(outdir, "#{File.basename(target)}.attachments")
+          digits = ((attachments.count + 1)/ 10) + 1
+          i = 0
+          attachments.delete_if {|a| a.properties.attachment_hidden}.each do |a|
+            prefix = "#{"%0*d" % [digits, i]}-"
+            if sub_msg = a.instance_variable_get(:@embedded_msg)
+# puts "Embedded email message ..."
+              subject = a.properties[:display_name] || sub_msg.subject || ""
+              file = File.join(outdir, "#{prefix}#{subject}.#{target_format.to_s.downcase}")
+
+              result = msg_to_pdf(sub_msg, file, target_format, pdf_options, reraise: true)
+              if e = result[:error]
+                raise 
+              end
+              files += result[:files]
+            elsif a.filename
+              next if used_files.include?(a.filename)
+              file = File.join(outdir, "#{prefix}#{a.filename}")
+
+              FileUtils.mkdir_p(File.dirname(file))
+              File.open(file, 'wb') {|f| a.save(f)}
+              files << file
+# puts "Attachment file '#{file}' created"
+            else
+              @warnings << "Attachment #{a.properties[:display_name]} cannot be saved"
+              next
+            end
+            i += 1
+          end
+          
+          {
+            command: {status: 0},
+            files: files,
+            headers: headers,
+            warnings: @warnings
+          }
+          
+        rescue Exception => e
+# puts "ERROR: Exception #{e.class} raised: #{e.message}"
+# e.backtrace.each {|t| puts " - #{t}"}
+          raise if reraise
+          msg.close
+          return {
+            command: {status: -1},
+            files: [],
+            headers: {},
+            errors: [
+              {
+                error: e.message,
+                error_class: e.class.name,
+                error_trace: e.backtrace,
+              }
+            ],
+            warnings: @warnings
+          }
+        end
+
+        protected
+
+        def eml_to_html
+
+        end
+
+        private
+
+        def find_hdr(list, key)
+          keys = list.keys
+          if k = keys.find {|x| x.to_s =~ /^#{key}$/i}
+            v = list[k]
+            v = v.first if v.is_a? Array
+            v = Rfc2047.decode(v).strip if v.is_a? String
+            return v
+          end
+          nil
+        end
+
+        def hdr_html(key, value)
+          return HEADER_FIELD_TEMPLATE % [key, CGI::escapeHTML(value)] if key.is_a?(String) && value.is_a?(String) && !value.empty?
+          ''
+        end
+
+        def getAttachmentData(attachments, cid)
+          attachments.each do |attachment|
+            if attachment.properties.attach_content_id == cid
+              attachment.data.rewind
+              return {
+                mime_type: attachment.properties.attach_mime_tag,
+                base64: Base64.encode64(attachment.data.read).gsub(/[\r\n]/, '')
+              }
+            end
+          end
+          return nil
+        end
+
+        def read_header(headers_file)
+          headers = YAML.load_file(headers_file)
+          headers.symbolize_keys
+        end
+      end
+    end
+  end
+end

data/lib/libis/format/tool.rb

@@ -18,6 +18,7 @@ module Libis
       autoload :PdfSplit, 'libis/format/tool/pdf_split'
       autoload :PdfToPdfa, 'libis/format/tool/pdf_to_pdfa'
       autoload :PdfaValidator, 'libis/format/tool/pdfa_validator'
+      autoload :MsgToPdf, 'libis/format/tool/msg_to_pdf'
 
     end
   end

data/lib/libis/format/version.rb

@@ -1,5 +1,5 @@
 module Libis
   module Format
-    VERSION = '1.2.3'
+    VERSION = '1.2.5'
   end
 end

data/libis-format.gemspec

@@ -40,5 +40,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'chromaprint', '~> 0.0.2'
   spec.add_runtime_dependency 'naturally', '~> 2.1'
   spec.add_runtime_dependency 'pdfinfo', '~> 1.4'
+  spec.add_runtime_dependency 'ruby-msg-nx'
+  spec.add_runtime_dependency 'new_rfc_2047'
+  spec.add_runtime_dependency 'pdfkit'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libis-format
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.2.5
 platform: ruby
 authors:
 - Kris Dekeyser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-19 00:00:00.000000000 Z
+date: 2023-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -178,6 +178,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: ruby-msg-nx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: new_rfc_2047
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pdfkit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Collection of tools and classes that help to identify formats of binary
   files and create derivative copies (e.g. PDF from Word).
 email:
@@ -194,6 +236,7 @@ files:
 - ".coveralls.yml"
 - ".gitignore"
 - ".travis.yml"
+- ".vscode/launch.json"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -240,13 +283,13 @@ files:
 - lib/libis/format/identifier.rb
 - lib/libis/format/tool.rb
 - lib/libis/format/tool/droid.rb
-- lib/libis/format/tool/email_to_pdf.rb
 - lib/libis/format/tool/extension_identification.rb
 - lib/libis/format/tool/ff_mpeg.rb
 - lib/libis/format/tool/fido.rb
 - lib/libis/format/tool/file_tool.rb
 - lib/libis/format/tool/fop_pdf.rb
 - lib/libis/format/tool/identification_tool.rb
+- lib/libis/format/tool/msg_to_pdf.rb
 - lib/libis/format/tool/office_to_pdf.rb
 - lib/libis/format/tool/pdf_copy.rb
 - lib/libis/format/tool/pdf_merge.rb
@@ -262,7 +305,6 @@ files:
 - tools/PdfTool.jar
 - tools/bcpkix-jdk15on-1.49.jar
 - tools/bcprov-jdk15on-1.49.jar
-- tools/emailconverter.jar
 - tools/fop/build/fop.jar
 - tools/fop/conf/fop.xconf
 - tools/fop/fop
@@ -335,7 +377,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: LIBIS File format format services.

data/lib/libis/format/tool/email_to_pdf.rb

@@ -1,75 +0,0 @@
-require "fileutils"
-
-require "libis/tools/extend/string"
-require "libis/tools/logger"
-require "libis/tools/command"
-
-require "libis/format/config"
-require "rexml/document"
-
-module Libis
-  module Format
-    module Tool
-      class EmailToPdf
-        include ::Libis::Tools::Logger
-
-        def self.installed?
-          result = Libis::Tools::Command.run(Libis::Format::Config[:email2pdf_cmd], "-v")
-          result[:status] == 0
-        end
-
-        def self.run(source, target, options = {})
-          new.run source, target, options
-        end
-
-        def run(source, target, _ = {})
-          timeout = Libis::Format::Config[:timeouts][:email2pdf] || 120
-          result = Libis::Tools::Command.run(
-            Libis::Format::Config[:java_cmd],
-            "-Duser.timezone=Europe/Brussels", "-Duser.language=nl", "-Duser.country=BE",
-            "-jar", Libis::Format::Config[:email2pdf_jar],
-            "-e", "-hd", "-a",
-            "-o", target,
-            source,
-            timeout: timeout,
-            kill_after: timeout * 2
-          )
-
-          warn "EmailToPdf conversion messages: \n\t#{result[:err].join("\n\t")}" unless result[:err].empty?
-
-          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-          raise "#{self.class} failed to generate target file #{target}" unless File.exist?(target)
-          raise "#{self.class} command failed with status code #{result[:status]}" unless result[:status] == 0
-
-          base_path = File.join(File.dirname(target), File.basename(target, ".*"))
-          headers_file = "#{base_path}.headers.xml"
-          headers = read_header(headers_file)
-
-          {
-            command: result,
-            files: [target, headers_file] + headers[:attachments].map { |a| File.join("#{base_path}-attachments", a) },
-            headers: headers
-          }
-        end
-
-        private
-
-        def read_header(headers_file)
-          headers = {}
-          return headers unless File.exist?(headers_file)
-          doc = REXML::Document.new(File.new(headers_file))
-          root = doc.root
-          root.children.each do |element|
-            case element.name
-            when "attachments"
-              headers[:attachments] = element.elements.map { |e| e.text }
-            else
-              headers[element.name.to_sym] = element.text
-            end
-          end
-          headers
-        end
-      end
-    end
-  end
-end