libis-format 1.2.6 → 1.2.8

data/lib/libis/format/tool/identification_tool.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'csv'
 require 'tmpdir'
 
@@ -12,42 +14,41 @@ require 'libis/format/type_database'
 module Libis
   module Format
     module Tool
-
-        class IdentificationTool
+      class IdentificationTool
         include Singleton
         include ::Libis::Tools::Logger
 
         def self.bad_mimetype(mimetype)
-          self.instance.bad_mimetype(mimetype)
+          instance.bad_mimetype(mimetype)
         end
 
-        def self.run(file, recursive = false, options = {})
-          options ||= {}
+        def self.run(file, recursive = false, **options)
           if file.is_a?(Array)
-            return run_list file, options
-          elsif file.is_a?(String) && File.exists?(file) && File.readable?(file)
+            return run_list file, **options
+          elsif file.is_a?(String) && File.exist?(file) && File.readable?(file)
             if File.directory?(file)
-              return run_dir(file, recursive, options)
+              return run_dir(file, recursive, **options)
             elsif File.file?(file)
-              return self.instance.run(file, options)
+              return instance.run(file, **options)
             end
           end
+
           raise ArgumentError,
                 'IdentificationTool: file argument should be a path to an existing file or directory or a list of those'
         end
 
-        def self.run_dir(file, recursive = true, options = {})
-          self.instance.run_dir file, recursive, options
+        def self.run_dir(file, recursive = true, **options)
+          instance.run_dir file, recursive, **options
         end
 
-        def self.run_list(filelist , options = {})
-          self.instance.run_list filelist, options
+        def self.run_list(filelist, **options)
+          instance.run_list filelist, **options
         end
 
         protected
 
         def create_list_file(filelist)
-          list_file = Tempfile.new(%w'file .list')
+          list_file = Tempfile.new(%w[file .list])
           filelist.each do |fname|
             list_file.write "#{fname}\n"
           end
@@ -83,20 +84,19 @@ module Libis
         #   { mimetype: <mimetype>, puid: <puid>, matchtype: <matchtype>, score: <score>, ...}
         #
         def process_output(output)
-          output.reduce({}) do |results, x|
+          output.each_with_object({}) do |x, results|
             filepath = File.absolute_path(x.delete(:filepath)).freeze
             results[filepath] ||= []
             results[filepath] << annotate(x)
-            results
           end
         end
 
         # Enhance the output with mimetype and score
         def annotate(result)
           # Enhance result with mimetype if needed
-          if bad_mimetypes.include?(result[:mimetype]) && !bad_puids.include?(result[:puid])
+          bad_mimetypes.include?(result[:mimetype]) &&
+            !bad_puids.include?(result[:puid]) &&
             result[:mimetype] = get_mimetype(result[:puid])
-          end
 
           # Normalize the mimetype
           Libis::Format::TypeDatabase.normalize(result, PUID: :puid, MIME: :mimetype)
@@ -108,55 +108,54 @@ module Libis
           result[:score] = 1 if bad_mimetypes.include? result[:mimetype]
 
           # freeze all strings
-          result.each {|_, v| v.freeze if v.is_a?(String)}
+          result.each { |_, v| v.freeze if v.is_a?(String) }
 
           # Adapt score based on matchtype
           result[:matchtype] = result[:matchtype].to_s.downcase
           case result[:matchtype]
 
             # Signature match increases score with 2
-            when 'signature'
-              result[:score] += 2
+          when 'signature'
+            result[:score] += 2
             # typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(result[:puid])
             # ext = File.extname(result[:filename])
             # result[:score] += 1 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
 
             # Container match increases score with 4
-            when 'container'
-              result[:score] += 4
+          when 'container'
+            result[:score] += 4
             # typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(result[:puid])
             # ext = File.extname(result[:filename])
             # result[:score] += 1 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
 
             # Extension match is the weakest identification; score is lowered by 2 points
-            when 'extension'
-              result[:score] -= 2
+          when 'extension'
+            result[:score] -= 2
 
             # Magic code (file tool) is to be trused even less
-            when 'magic'
-              result[:score] -= 3
+          when 'magic'
+            result[:score] -= 3
 
-            # Or no change otherwise
-            else
-              # do nothing
           end
 
           # Detecting a zip file should decrease the score as it may hide one of the many zip-based formats (e.g. epub,
           # Office OpenXML, OpenDocument, jar, maff, svx)
-          if result[:mimetype] == 'application/zip'
-            result[:score] -= 2
-          end
+          result[:score] -= 2 if result[:mimetype] == 'application/zip'
 
           # Return result enhanced with mimetype and score fields
           result
         end
 
         def get_mimetype(puid)
-          ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first rescue nil
+          ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first
+        rescue StandardError
+          nil
         end
 
         def get_puid(mimetype)
-          ::Libis::Format::TypeDatabase.mime_infos(mimetype).first[:PUID].first rescue nil
+          ::Libis::Format::TypeDatabase.mime_infos(mimetype).first[:PUID].first
+        rescue StandardError
+          nil
         end
 
         attr_accessor :bad_mimetypes, :bad_puids
@@ -170,7 +169,6 @@ module Libis
           @bad_mimetypes << mimetype
         end
       end
-
     end
   end
 end

data/lib/libis/format/tool/msg_to_pdf.rb

@@ -1,12 +1,12 @@
-# encoding: utf-8
+# frozen_string_literal: true
 
 require 'mapi/msg'
 require 'rfc_2047'
 require 'cgi'
 require 'pdfkit'
-
+require 'time'
 require 'fileutils'
-
+require 'pathname'
 require 'libis/format/config'
 
 module Libis
@@ -15,23 +15,23 @@ module Libis
       class MsgToPdf
         include ::Libis::Tools::Logger
 
-        HEADER_STYLE = '<style>.header-table {margin: 0 0 20 0;padding: 0;font-family: Arial, Helvetica, sans-serif;}.header-name {padding-right: 5px;color: #9E9E9E;text-align: right;vertical-align: top;font-size: 12px;}.header-value {font-size: 12px;}#header_fields {background: white;margin: 0;border: 1px solid #DDD;border-radius: 3px;padding: 8px;width: 100%%;box-sizing: border-box;}</style><script type="text/javascript">function timer() {try {parent.postMessage(Math.max(document.body.offsetHeight, document.body.scrollHeight), \'*\');} catch (r) {}setTimeout(timer, 10);};timer();</script>'
+        HEADER_STYLE = '<style>.header-table {margin: 0 0 20 0;padding: 0;font-family: Arial, Helvetica, sans-serif;}.header-name {padding-right: 5px;color: #9E9E9E;text-align: right;vertical-align: top;font-size: 12px;}.header-value {font-size: 12px;}#header_fields {#background: white;#margin: 0;#border: 1px solid #DDD;#border-radius: 3px;#padding: 8px;#width: 100%%;#box-sizing: border-box;#}</style><script type="text/javascript">function timer() {try {parent.postMessage(Math.max(document.body.offsetHeight, document.body.scrollHeight), \'*\');} catch (r) {}setTimeout(timer, 10);};timer();</script>' # rubocop:disable Layout/LineLength
         HEADER_TABLE_TEMPLATE = '<div class="header-table"><table id="header_fields"><tbody>%s</tbody></table></div>'
         HEADER_FIELD_TEMPLATE = '<tr><td class="header-name">%s</td><td class="header-value">%s</td></tr>'
-        HTML_WRAPPER_TEMPLATE = '<!DOCTYPE html><html><head><style>body {font-size: 0.5cm;}</style><title>title</title></head><body>%s</body></html>'
+        HTML_WRAPPER_TEMPLATE = '<!DOCTYPE html><html><head><style>body {font-size: 0.5cm;}</style><title>title</title></head><body>%s</body></html>' # rubocop:disable Layout/LineLength
 
-        IMG_CID_PLAIN_REGEX = %r/\[cid:(.*?)\]/m
-        IMG_CID_HTML_REGEX = %r/cid:([^"]*)/m
+        IMG_CID_PLAIN_REGEX = /\[cid:(.*?)\]/m
+        IMG_CID_HTML_REGEX = /cid:([^"]*)/m
 
         def self.installed?
           File.exist?(Libis::Format::Config[:wkhtmltopdf])
         end
 
-        def self.run(source, target, options = {})
-          new.run source, target, options
+        def self.run(source, target, **options)
+          new.run source, target, **options
         end
 
-        def run(source, target, options = {})
+        def run(source, target, **options)
           # Preliminary checks
           # ------------------
 
@@ -52,41 +52,16 @@ module Libis
           result
         end
 
-        def msg_to_pdf(msg, target, target_format, pdf_options, reraise: false)
-
+        def msg_to_pdf(msg, target, target_format, pdf_options, root_msg: true)
           # Make sure the target directory exists
           outdir = File.dirname(target)
           FileUtils.mkdir_p(outdir)
 
-# puts "Headers:"
-# puts '--------'
-# pp msg.headers
-
-# puts "Recipients:"
-# puts '-----------'
-# pp msg.recipients
-
-# puts "Body:"
-# puts '-----'
-# puts msg.properties.body
-# puts '-----'
-# puts msg.properties.body_rtf
-# puts '-----'
-# puts msg.properties.body_html
-
-# puts "Attachments:"
-# puts '------------'
-# msg.attachments.each {|a| p "#{a.filename} - #{a.properties.attach_content_id}"}
-
-# puts "Converting:"
-# puts '-----------'
-
           # Get the body of the message in HTML
           body = msg.properties.body_html
-          body ||= begin
-            # Embed plain body in HTML as a fallback
-            HTML_WRAPPER_TEMPLATE % msg.properties.body
-          end
+
+          # Embed plain body in HTML as a fallback
+          body ||= HTML_WRAPPER_TEMPLATE % msg.properties.body
 
           # Check and fix the character encoding
           begin
@@ -94,8 +69,8 @@ module Libis
             body.encode!('UTF-8', universal_newline: true)
           rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
             begin
-            # If it fails, the text may be in Windows' Latin1 (ISO-8859-1)
-            body.force_encoding('ISO-8859-1').encode!('UTF-8', universal_newline: true)
+              # If it fails, the text may be in Windows' Latin1 (ISO-8859-1)
+              body.force_encoding('ISO-8859-1').encode!('UTF-8', universal_newline: true)
             rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError => e
               # If that fails too, log a warning and replace the invalid/unknown with a ? character.
               @warnings << "#{e.class}: #{e.message}"
@@ -108,7 +83,7 @@ module Libis
           headers = {}
           hdr_html = ''
 
-          %w"From To Cc Subject Date".each do |key|
+          %w[From To Cc Subject Date].each do |key|
             value = find_hdr(msg.headers, key)
             if value
               headers[key.downcase.to_sym] = value
@@ -116,18 +91,27 @@ module Libis
             end
           end
 
+          [:date].each do |key|
+            next unless headers[key]
+
+            headers[key] = DateTime.parse(headers[key]).to_time.localtime.iso8601
+          end
+
           # Add header section to the HTML body
           unless hdr_html.empty?
             # Insert header block styles
-            if body =~ /<\/head>/
+            if body =~ %r{</head>}
               # if head exists, append the style block
-              body.gsub!(/<\/head>/, HEADER_STYLE + '</head>')
+              body.gsub!(%r{</head>}, "#{HEADER_STYLE}</head>")
+            elsif body =~ %r{<head/>}
+              # empty head, replace with the style block
+              body.gsub!(%r{<head/>}, "<head>#{HEADER_STYLE}</head>")
             else
               # otherwise insert a head section before the body tag
-              body.gsub!(/<body/, '<head>' + HEADER_STYLE + '</head><body')
+              body.gsub!(/<body/, "<head>#{HEADER_STYLE}</head><body")
             end
             # Add the headers html table as first element in the body section
-            body.gsub!(/<body[^>]*>/) {|m| "#{m}#{HEADER_TABLE_TEMPLATE % hdr_html}"}
+            body.gsub!(/<body[^>]*>/) { |m| "#{m}#{HEADER_TABLE_TEMPLATE % hdr_html}" }
           end
 
           # Embed inline images
@@ -136,29 +120,23 @@ module Libis
           used_files = []
 
           # First process plaintext cid entries
-          body.gsub!(IMG_CID_PLAIN_REGEX) do |match|
-# puts "CID found: #{match}, looking for #{$1}"
-            data = getAttachmentData(attachments, $1)
+          body.gsub!(IMG_CID_PLAIN_REGEX) do |_match|
+            data = get_attachment_data(attachments, ::Regexp.last_match(1))
             if data
-# puts "cid #{$1} data: #{data.inspect}"
-              used_files << $1
+              used_files << ::Regexp.last_match(1)
               "<img src=\"data:#{data[:mime_type]};base64,#{data[:base64]}\"/>"
             else
-  # puts "cid #{$1} not found"
               '<img src=""/>'
             end
           end
-          
+
           # Then process HTML img tags with CID entries
-          body.gsub!(IMG_CID_HTML_REGEX) do |match|
-# puts "CID found: #{match}, looking for #{$1}"
-            data = getAttachmentData(attachments, $1)
-            if  data
-# puts "cid #{$1} data: #{data.inspect}"
-            used_files << $1
-            "data:#{data[:mime_type]};base64,#{data[:base64]}"
+          body.gsub!(IMG_CID_HTML_REGEX) do |_match|
+            data = get_attachment_data(attachments, ::Regexp.last_match(1))
+            if data
+              used_files << ::Regexp.last_match(1)
+              "data:#{data[:mime_type]};base64,#{data[:base64]}"
             else
-# puts "cid #{$1} not found"
               ''
             end
           end
@@ -168,85 +146,83 @@ module Libis
           files = []
 
           if target_format == :PDF
-          # PDF creation options
+            # PDF creation options
             pdf_options = {
               page_size: 'A4',
               margin_top: '10mm',
               margin_bottom: '10mm',
               margin_left: '10mm',
               margin_right: '10mm',
-              dpi: 300,
               # image_quality: 100,
               # viewport_size: '2480x3508',
+              dpi: 300
             }.merge pdf_options
 
-# pp pdf_options
-# puts "Final HTML body:"
-# pp body
             subject = find_hdr(msg.headers, 'Subject')
             kit = PDFKit.new(body, title: (subject || 'message'), **pdf_options)
             pdf = kit.to_pdf
-            File.open(target, 'wb') {|f| f.write(pdf)}
-# puts "message #{subject} converted to PDF file '#{target}'"
+            File.open(target, 'wb') { |f| f.write(pdf) }
           else
-            File.open(target, 'wb') {|f| f.write(body)}
-# puts "message #{subject} converted to HTML file '#{target}'"
+            File.open(target, 'wb') { |f| f.write(body) }
           end
           files << target if File.exist?(target)
 
           # Save attachments
           # ----------------
           outdir = File.join(outdir, "#{File.basename(target)}.attachments")
-          digits = ((attachments.count + 1)/ 10) + 1
+          digits = ((attachments.count + 1) / 10) + 1
           i = 1
-          attachments.delete_if {|a| a.properties.attachment_hidden}.each do |a|
-            prefix = "#{"%0*d" % [digits, i]}-"
-            if sub_msg = a.instance_variable_get(:@embedded_msg)
-# puts "Embedded email message ..."
-              subject = a.properties[:display_name] || sub_msg.subject || ""
-              file = File.join(outdir, "#{prefix}#{subject}.#{target_format.to_s.downcase}")
-
-              result = msg_to_pdf(sub_msg, file, target_format, pdf_options, reraise: true)
-              if e = result[:error]
-                raise 
+          attachments.delete_if { |a| a.properties.attachment_hidden }.each do |a|
+            prefix = "#{format('%0*d', digits, i)}-"
+            if (sub_msg = a.instance_variable_get(:@embedded_msg))
+              subject = a.properties[:display_name] || sub_msg.subject || ''
+              file = File.join(outdir, "#{prefix}#{subject}.msg.#{target_format.to_s.downcase}")
+              result = msg_to_pdf(sub_msg, file, target_format, pdf_options, root_msg: false)
+              if (e = result[:error])
+                raise e
               end
+
               files += result[:files]
             elsif a.filename
               next if used_files.include?(a.filename)
-              file = File.join(outdir, "#{prefix}#{a.filename}")
 
+              file = File.join(outdir, "#{prefix}#{a.filename}")
               FileUtils.mkdir_p(File.dirname(file))
-              File.open(file, 'wb') {|f| a.save(f)}
+              File.open(file, 'wb') { |f| a.save(f) }
               files << file
-# puts "Attachment file '#{file}' created"
             else
-              @warnings << "Attachment #{a.properties[:display_name]} cannot be saved"
+              @warnings << "Attachment #{a.properties[:display_name]} cannot be extracted"
               next
             end
             i += 1
           end
-          
+
+          if root_msg
+            p = Pathname(File.dirname(files.first))
+            files[1..].each do |f|
+              (headers[:attachments] ||= []) << Pathname.new(f).relative_path_from(p).to_s
+            end
+          end
+
           {
-            command: {status: 0},
-            files: files,
-            headers: headers,
+            command: { status: 0 },
+            files:,
+            headers:,
             warnings: @warnings
           }
-          
         rescue Exception => e
-# puts "ERROR: Exception #{e.class} raised: #{e.message}"
-# e.backtrace.each {|t| puts " - #{t}"}
-          raise if reraise
+          raise unless root_msg
+
           msg.close
-          return {
-            command: {status: -1},
+          {
+            command: { status: -1 },
             files: [],
             headers: {},
             errors: [
               {
                 error: e.message,
                 error_class: e.class.name,
-                error_trace: e.backtrace,
+                error_trace: e.backtrace
               }
             ],
             warnings: @warnings
@@ -255,15 +231,13 @@ module Libis
 
         protected
 
-        def eml_to_html
-
-        end
+        def eml_to_html; end
 
         private
 
         def find_hdr(list, key)
           keys = list.keys
-          if k = keys.find {|x| x.to_s =~ /^#{key}$/i}
+          if (k = keys.find { |x| x.to_s =~ /^#{key}$/i })
             v = list[k]
             v = v.first if v.is_a? Array
             v = Rfc2047.decode(v).strip if v.is_a? String
@@ -273,27 +247,23 @@ module Libis
         end
 
         def hdr_html(key, value)
-          return HEADER_FIELD_TEMPLATE % [key, CGI::escapeHTML(value)] if key.is_a?(String) && value.is_a?(String) && !value.empty?
+          return format(HEADER_FIELD_TEMPLATE, key, CGI.escapeHTML(value)) if key.is_a?(String) && value.is_a?(String) && !value.empty?
+
           ''
         end
 
-        def getAttachmentData(attachments, cid)
+        def get_attachment_data(attachments, cid)
           attachments.each do |attachment|
-            if attachment.properties.attach_content_id == cid
-              attachment.data.rewind
-              return {
-                mime_type: attachment.properties.attach_mime_tag,
-                base64: Base64.encode64(attachment.data.read).gsub(/[\r\n]/, '')
-              }
-            end
+            next unless attachment.properties.attach_content_id == cid
+
+            attachment.data.rewind
+            return {
+              mime_type: attachment.properties.attach_mime_tag,
+              base64: Base64.encode64(attachment.data.read).gsub(/[\r\n]/, '')
+            }
           end
           nil
         end
-
-        def read_header(headers_file)
-          headers = YAML.load_file(headers_file)
-          headers.symbolize_keys
-        end
       end
     end
   end

data/lib/libis/format/tool/office_to_pdf.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'fileutils'
 
 require 'libis/tools/extend/string'
@@ -9,60 +11,61 @@ require 'libis/format/config'
 module Libis
   module Format
     module Tool
-
       class OfficeToPdf
         include ::Libis::Tools::Logger
 
         def self.installed?
-          result = Libis::Tools::Command.run(Libis::Format::Config[:soffice_cmd], "--version")
-          result[:status] == 0
+          result = Libis::Tools::Command.run(Libis::Format::Config[:soffice_cmd], '--version')
+          (result[:status]).zero?
         end
 
-        def self.run(source, target, options = {})
-          self.new.run source, target, options
+        def self.run(source, target, **options)
+          new.run source, target, **options
         end
 
-        def run(source, target, options = {})
+        def run(source, target, **options)
           workdir = '/...'
           workdir = Dir.tmpdir unless Dir.exist? workdir
 
-          workdir = File.join(workdir, rand(1000000).to_s)
+          workdir = File.join(workdir, rand(1_000_000).to_s)
           FileUtils.mkpath(workdir)
 
           src_file = File.join(workdir, File.basename(source))
           FileUtils.symlink source, src_file
 
-          tgt_file = File.join(workdir, File.basename(source, '.*') + '.pdf')
+          tgt_file = File.join(workdir, "#{File.basename(source, '.*')}.pdf")
 
           export_filter = options[:export_filter] || 'pdf'
 
           timeout = Libis::Format::Config[:timeouts][:office_to_pdf]
           result = Libis::Tools::Command.run(
-              Libis::Format::Config[:soffice_cmd], '--headless',
-              "-env:UserInstallation=file://#{workdir}",
-              '--convert-to', export_filter,
-              '--outdir', workdir, src_file,
-              timeout: timeout,
-              kill_after: timeout * 2
+            Libis::Format::Config[:soffice_cmd], '--headless',
+            "-env:UserInstallation=file://#{workdir}",
+            '--convert-to', export_filter,
+            '--outdir', workdir, src_file,
+            timeout:,
+            kill_after: timeout * 2
           )
 
-          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+
           warn "OfficeToPdf conversion messages: \n\t#{result[:err].join("\n\t")}" unless result[:err].empty?
-          raise RuntimeError, "#{self.class} failed to generate target file #{tgt_file}" unless File.exist?(tgt_file)
+          raise "#{self.class} failed to generate target file #{tgt_file}" unless File.exist?(tgt_file)
 
           FileUtils.copy tgt_file, target, preserve: true
 
           {
             command: result,
-            files: [ target ]
+            files: [target]
           }
-
         ensure
-          FileUtils.rmtree workdir rescue nil
-
+          begin
+            FileUtils.rmtree workdir
+          rescue StandardError
+            nil
+          end
         end
       end
-
     end
   end
 end