libis-format 1.3.2 → 2.0.0

data/lib/libis/format/tool/pdf_optimizer.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'os'
 
 require 'libis/tools/extend/string'
@@ -11,38 +9,37 @@ require 'libis/format/config'
 module Libis
   module Format
     module Tool
+
       class PdfOptimizer
         include ::Libis::Tools::Logger
 
-        def self.installed?
-          result = Libis::Tools::Command.run(Libis::Format::Config[:ghostscript_cmd], '--version')
-          (result[:status]).zero?
-        end
-
         def self.run(source, target, quality)
-          new.run source, target, quality
+          self.new.run source, target, quality
         end
 
         def run(source, target, quality)
-          timeout = Libis::Format::Config[:timeouts][:pdf_optimizer]
-          args = [
-            Libis::Format::Config[:ghostscript_cmd],
-            '-sDEVICE=pdfwrite',
-            '-dCompatibilityLevel=1.4',
-            "-dPDFSETTINGS=/#{quality}",
-            '-dNOPAUSE',
-            '-dBATCH',
-            "-sOutputFile=#{target}",
-            source.to_s
-          ]
-
-          result = Libis::Tools::Command.run(*args, timeout:, kill_after: timeout * 2)
 
-          result[:err] << "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          timeout = Libis::Format::Config[:timeouts][:pdf_optimizer]
+          result = Libis::Tools::Command.run(
+              'gs',
+              '-sDEVICE=pdfwrite',
+              '-dCompatibilityLevel=1.4',
+              "-dPDFSETTINGS=/#{quality}",
+              '-dNOPAUSE',
+              '-dBATCH',
+              "-sOutputFile=#{target}",
+              "#{source}",
+              timeout: timeout,
+              kill_after: timeout * 2
+          )
+
+          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0
 
           result
         end
       end
+
     end
   end
 end

data/lib/libis/format/tool/pdf_split.rb

@@ -0,0 +1,47 @@
+require 'os'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+    module Tool
+
+      class PdfSplit
+        include ::Libis::Tools::Logger
+
+        def self.run(source, target, options = [])
+          self.new.run source, target, options
+        end
+
+        def run(source, target, options = [])
+
+          if OS.java?
+            # TODO: import library and execute in current VM. For now do exactly as in MRI.
+          end
+
+          timeout = Libis::Format::Config[:timeouts][:pdf_split]
+          result = Libis::Tools::Command.run(
+              Libis::Format::Config[:java_cmd],
+              '-cp', Libis::Format::Config[:pdf_tool],
+              'SplitPdf',
+              '--file_input', source,
+              '--file_output', target,
+              *options,
+              timeout: timeout,
+              kill_after: timeout * 2
+          )
+
+          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
+
+          result
+        end
+      end
+
+    end
+  end
+end

data/lib/libis/format/tool/pdf_to_pdfa.rb

@@ -1,9 +1,6 @@
-# frozen_string_literal: true
-
 require 'tempfile'
 require 'csv'
 require 'fileutils'
-require 'pdfinfo'
 
 require 'libis/tools/extend/string'
 require 'libis/tools/logger'
@@ -15,24 +12,19 @@ require 'libis/format'
 module Libis
   module Format
     module Tool
+
       class PdfToPdfa
         include ::Libis::Tools::Logger
 
-        def self.installed?
-          result = Libis::Tools::Command.run(Libis::Format::Config[:ghostscript_cmd])
-          result.zero?
+        def self.run(source, target = nil, options = {})
+          self.new.run source, target, options
         end
 
-        def self.run(source, target = nil, **options)
-          new.run source, target, **options
-        end
+        def run(source, target = nil, options = nil)
 
-        def run(source, target = nil, **options)
           tmp_target = Tools::TempFile.name(File.basename(source, '.*'), '.pdf')
           target ||= tmp_target
 
-          metadata = get_metadata(source)
-
           icc_info = icc_options(options[:colorspace])
 
           icc_file = Tools::TempFile.name(icc_info[:icc_name], '.icc')
@@ -40,58 +32,52 @@ module Libis
 
           def_filename = Tools::TempFile.name('PDFA_def', '.ps')
           File.open(def_filename, 'w') do |f|
-            f.puts File.read(File.join(Libis::Format::DATA_DIR, 'PDFA_def.ps'))
-                       .gsub('[**ICC profile**]', icc_file)
-                       .gsub('[**ICC reference**]', icc_info[:icc_ref])
-                       .gsub('[**METADATA**]', metadata)
+            f.puts File.read(File.join(Libis::Format::DATA_DIR, 'PDFA_def.ps')).
+                gsub('[** Fill in ICC profile location **]', icc_file).
+                gsub('[** Fill in ICC reference name **]', icc_info[:icc_ref])
           end
 
           timeout = Libis::Format::Config[:timeouts][:pdf_to_pdfa]
           result = Libis::Tools::Command.run(
-            Libis::Format::Config[:ghostscript_cmd],
-            '-q',
-            '-dBATCH', '-dNOPAUSE', '-dNOOUTERSAVE', '-dNOSAFER',
-            # "-dNOPLATFONTS", "-dUseCIEColor=true",
-            # "-sColorConversionStrategy=/UseDeviceIndependentColor",
-            "-sProcessColorModel=#{icc_info[:device]}",
-            "-sOutputICCProfile=#{icc_file}",
-            '-dCompatibilityLevel=1.4',
-            '-sDEVICE=pdfwrite', '-dPDFA=1', '-dPDFACompatibilityPolicy=1',
-            '-o', File.absolute_path(target),
-            def_filename,
-            source,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:ghostscript_cmd],
+              '-dBATCH', '-dNOPAUSE', '-dNOOUTERSAVE',
+              '-sColorConversionStrategy=/UseDeviceIndependentColor',
+              "-sProcessColorModel=#{icc_info[:device]}",
+              '-sDEVICE=pdfwrite', '-dPDFA', '-dPDFACompatibilityPolicy=1',
+              "-sOutputICCProfile=#{icc_file}",
+              '-o', File.absolute_path(target),
+              def_filename,
+              source,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
 
-          result[:err] << "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
 
           FileUtils.rm [icc_file, def_filename].compact, force: true
 
+          unless Format::Tool::PdfaValidator.run(target)
+            result[:status] = -999
+            result[:err] << 'Failed to validate generated PDF/A file.'
+          end
+
           result
         end
 
-        private
 
-        def get_metadata(source)
-          info = Pdfinfo.new(source)
-          metadata = "/Title (#{info.title})"
-          metadata += "\n  /Author (#{info.author})" if info.author
-          metadata += "\n  /Subject (#{info.subject})" if info.subject
-          metadata += "\n  /Keywords (#{info.keywords})" if info.keywords
-          metadata += "\n  /Creator (#{info.creator})" if info.creator
-          metadata
-        end
+        private
 
         def icc_options(colorspace)
           case colorspace.to_s.downcase
-          when 'cmyk'
-            { icc_name: 'ISOcoated_v2_eci', icc_ref: 'FOGRA39L', device: 'DeviceCMYK' }
-          else
-            { icc_name: 'AdobeRGB1998', icc_ref: 'sRGB', device: 'DeviceRGB' }
+            when 'cmyk'
+              {icc_name: 'ISOcoated_v2_eci', icc_ref: 'FOGRA39L', device: 'DeviceCMYK'}
+            else
+              {icc_name: 'eciRGB_v2', icc_ref: 'sRGB', device: 'DeviceRGB'}
           end
         end
+
       end
+
     end
   end
 end

data/lib/libis/format/tool/pdfa_validator.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require 'fileutils'
 
 require 'libis/tools/extend/string'
@@ -11,57 +9,65 @@ require 'libis/format/config'
 module Libis
   module Format
     module Tool
+
       class PdfaValidator
         include ::Libis::Tools::Logger
 
         def self.run(source)
-          new.run source
+          self.new.run source
         end
 
         def run(source)
+
           src_file = File.absolute_path(source)
 
           timeout = Libis::Format::Config[:timeouts][:pdfa_validator]
-          result = nil
           if (pdfa = Libis::Format::Config[:pdfa_cmd])
             # Keep it clean: tool generates fontconfig/ cache dir in current working dir
             previous_wd = Dir.getwd
             Dir.chdir(Dir.tmpdir)
 
             result = Libis::Tools::Command.run(
-              pdfa,
-              '--noxml',
-              '--level', 'B',
-              '--verb', '0',
-              src_file,
-              timeout:,
-              kill_after: timeout * 2
+                pdfa,
+                '--noxml',
+                '--level', 'B',
+                '--verb', '0',
+                src_file,
+                timeout: timeout,
+                kill_after: timeout * 2
             )
 
-            result[:err] << "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+            raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
+            raise RuntimeError, "#{self.class} errors: #{result[:err].join("\n")}" unless result[:status] == 0 && result[:err].empty?
 
             Dir.chdir(previous_wd)
 
-            out, err = result[:out].partition { |line| line =~ /^VLD-\[PASS\]/ }
-            result[:out] = out
-            result[:err] += err
-
+            unless result[:out].any? {|line| line =~ /^VLD-\[PASS\]/}
+              warn "Validator failed to validate the PDF file '%s' against PDF/A-1B constraints:\n%s", source,
+                   result[:out].join("\n")
+              return false
+            end
           else
             jar = Libis::Format::Config[:preflight_jar]
             result = Libis::Tools::Command.run(
-              Libis::Format::Config[:java_cmd],
-              '-jar', jar,
-              src_file,
-              timeout:,
-              kill_after: timeout * 2
+                Libis::Format::Config[:java_cmd],
+                '-jar', jar,
+                src_file,
+                timeout: timeout,
+                kill_after: timeout * 2
             )
+            raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
 
-            result[:err] << "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-
+            unless result[:status] == 0
+              warn "Validator failed to validate the PDF file '%s' against PDF/A-1B constraints:\n%s", source,
+                   result[:out].join("\n")
+              return false
+            end
           end
-          result
+          true
         end
       end
+
     end
   end
 end

data/lib/libis/format/tool/spreadsheet_to_ods.rb

@@ -1,5 +1,4 @@
-# frozen_string_literal: true
-
+# noinspection RubyResolve
 require 'fileutils'
 
 require 'libis/tools/extend/string'
@@ -11,62 +10,52 @@ require 'libis/format/config'
 module Libis
   module Format
     module Tool
+
       class SpreadsheetToOds
         include ::Libis::Tools::Logger
 
-        def self.installed?
-          result = Libis::Tools::Command.run(Libis::Format::Config[:soffice_cmd], '--version')
-          result.zero?
-        end
-
         def self.run(source, target, options = {})
-          new.run source, target, options
+          self.new.run source, target, options
         end
 
         def run(source, target, options = {})
           workdir = '/...'
           workdir = Dir.tmpdir unless Dir.exist? workdir
 
-          workdir = File.join(workdir, rand(1_000_000).to_s)
+          workdir = File.join(workdir, rand(1000000).to_s)
           FileUtils.mkpath(workdir)
 
           src_file = File.join(workdir, File.basename(source))
           FileUtils.symlink source, src_file
 
-          tgt_file = File.join(workdir, "#{File.basename(source, '.*')}.ods")
+          tgt_file = File.join(workdir, File.basename(source, '.*') + '.ods')
 
           export_filter = options[:export_filter] || 'ods'
 
           timeout = Libis::Format::Config[:timeouts][:spreadsheet_to_ods] ||
-                    Libis::Format::Config[:timeouts][:office_to_pdf]
+              Libis::Format::Config[:timeouts][:office_to_pdf]
           result = Libis::Tools::Command.run(
-            Libis::Format::Config[:soffice_cmd], '--headless',
-            "-env:UserInstallation=file://#{workdir}",
-            '--convert-to', export_filter,
-            '--outdir', workdir, src_file,
-            timeout:,
-            kill_after: timeout * 2
+              Libis::Format::Config[:soffice_cmd], '--headless',
+              "-env:UserInstallation=file://#{workdir}",
+              '--convert-to', export_filter,
+              '--outdir', workdir, src_file,
+              timeout: timeout,
+              kill_after: timeout * 2
           )
 
-          raise "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
-
+          raise RuntimeError, "#{self.class} took too long (> #{timeout} seconds) to complete" if result[:timeout]
           warn "OfficeToPdf conversion messages: \n\t#{result[:err].join("\n\t")}" unless result[:err].empty?
-          raise "#{self.class} failed to generate target file #{tgt_file}" unless File.exist?(tgt_file)
+          raise RuntimeError, "#{self.class} failed to generate target file #{tgt_file}" unless File.exist?(tgt_file)
 
           FileUtils.copy tgt_file, target, preserve: true
 
-          {
-            command: result,
-            files: [target]
-          }
         ensure
-          begin
-            FileUtils.rmtree workdir
-          rescue StandardError
-            nil
-          end
+          FileUtils.rmtree workdir rescue nil
+
+          result[:out]
         end
       end
+
     end
   end
 end

data/lib/libis/format/tool.rb

@@ -1,17 +1,16 @@
-# frozen_string_literal: true
-
 # code utf-8
 
 module Libis
   module Format
     module Tool
+
       autoload :Droid, 'libis/format/tool/droid'
       autoload :ExtensionIdentification, 'libis/format/tool/extension_identification'
       autoload :Fido, 'libis/format/tool/fido'
       autoload :FileTool, 'libis/format/tool/file_tool'
 
       autoload :OfficeToPdf, 'libis/format/tool/office_to_pdf'
-      autoload :FFMpeg, 'libis/format/tool/ff_mpeg'
+      autoload :FFMpeg, 'libis/format/tool/ffmpeg'
       autoload :FopPdf, 'libis/format/tool/fop_pdf'
       autoload :PdfCopy, 'libis/format/tool/pdf_copy'
       autoload :PdfMerge, 'libis/format/tool/pdf_merge'
@@ -19,7 +18,7 @@ module Libis
       autoload :PdfSplit, 'libis/format/tool/pdf_split'
       autoload :PdfToPdfa, 'libis/format/tool/pdf_to_pdfa'
       autoload :PdfaValidator, 'libis/format/tool/pdfa_validator'
-      autoload :MsgToPdf, 'libis/format/tool/msg_to_pdf'
+
     end
   end
 end

data/lib/libis/format/version.rb

@@ -1,7 +1,5 @@
-# frozen_string_literal: true
-
 module Libis
   module Format
-    VERSION = '1.3.2'
+    VERSION = '2.0.0'
   end
 end

data/lib/libis/format/yaml_loader.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+require 'yaml'
+require 'singleton'
+
+module Libis
+  module Format
+    class YamlLoader
+      # noinspection RubyResolve
+      include Singleton
+
+      def query(key, value)
+        case key.to_s.downcase.to_sym
+        when :name
+          return [database[value.to_s.upcase.to_sym]]
+        when :category
+          database.find_all { |_, info| info.category == value.to_s.upcase.to_sym }
+        when :puid
+          database.find_all { |_, info| info.puids.include?(value) }
+        when :mimetype
+          database.find_all { |_, info| info.mimetypes.include?(value) }
+        when :extension
+          database.find_all { |_, info| info.extensions.include?(value) }
+        else
+          return []
+        end.map(&:last)
+      end
+
+      def load_formats(file_or_hash)
+        hash = file_or_hash.is_a?(Hash) ? file_or_hash : YAML::load_file(file_or_hash)
+        hash.each do |category, format_list|
+          format_list.each do |format_name, format_info|
+            format_info.symbolize_keys!
+            format_name = format_name.to_sym
+            new_info = Libis::Format::Info.new(
+                name: format_name,
+                category: category.to_sym,
+                description: format_info[:NAME],
+                puids: format_info[:PUID]&.strip&.split(/[\s,]+/)&.map { |v| v.strip } || [],
+                mimetypes: format_info[:MIME]&.strip&.split(/[\s,]+/)&.map(&:strip) || [],
+                extensions: format_info[:EXTENSIONS]&.strip&.split(/[\s,]+/)&.map { |v| v.strip } || []
+            )
+            if (old_info = database[format_name])
+              new_info = Libis::Format::Info.new(
+                  name: format_name,
+                  category: category.to_sym,
+                  description: new_info.description.blank? ? old_info.description : new_info.description,
+                  puids: (old_info.puids + new_info.puids).uniq,
+                  mimetypes: (old_info.mimetypes + new_info.mimetypes).uniq,
+                  extensions: (old_info.extensions + new_info.extensions).uniq
+              )
+            end
+            database[format_name] = new_info
+          end
+        end
+
+      end
+
+      private
+
+      attr_reader :database
+
+      def initialize
+        @database = {}
+        format_database = Libis::Format::Config[:format_library_database]
+        load_formats(format_database)
+      end
+
+    end
+
+  end
+end

data/lib/libis/format.rb

@@ -1,12 +1,13 @@
-# frozen_string_literal: true
-
 require 'libis/format/version'
 
 module Libis
   module Format
     autoload :Config, 'libis/format/config'
-    autoload :TypeDatabase, 'libis/format/type_database'
-    autoload :TypeDatabaseImpl, 'libis/format/type_database_impl'
+
+    autoload :Info, 'libis/format/info'
+    autoload :Library, 'libis/format/library'
+    autoload :YamlLoader, 'libis/format/yaml_loader'
+
     autoload :Identifier, 'libis/format/identifier'
 
     autoload :Tool, 'libis/format/tool'
@@ -15,5 +16,6 @@ module Libis
     ROOT_DIR = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..'))
     DATA_DIR = File.join(ROOT_DIR, 'data')
     TOOL_DIR = File.join(ROOT_DIR, 'tools')
+
   end
-end
+end

data/lib/libis-format.rb

@@ -1,3 +1 @@
-# frozen_string_literal: true
-
 require 'libis/format'

data/libis-format.gemspec

@@ -1,48 +1,42 @@
-# frozen_string_literal: true
+# coding: utf-8
 
-lib = File.expand_path('lib', __dir__)
+lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
-require 'libis/format/version'
 require 'bundler'
+require 'libis/format/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'libis-format'
   spec.version       = Libis::Format::VERSION
   spec.authors       = ['Kris Dekeyser']
   spec.email         = ['kris.dekeyser@libis.be']
-  spec.summary       = 'LIBIS File format format services.'
-  spec.description   = 'Collection of tools and classes that help to identify file formats and create derivative copies.'
+  spec.summary       = %q{LIBIS File format format services.}
+  spec.description   = %q{Collection of tools and classes that help to identify formats of binary files and create derivative copies (e.g. PDF from Word).}
   spec.homepage      = ''
   spec.license       = 'MIT'
 
   spec.platform      = Gem::Platform::JAVA if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
-  spec.required_ruby_version = '>= 3.2'
 
-  spec.files = `git ls-files -z`.split("\x0").select do |f|
-    f.match(%r{^(bin/|lib/|data/|tools/|Gemfile|libis-format.gemspec|LICENSE\.txt|README\.md)})
-  end
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.executables   = spec.files.grep(%r{^bin/[^/]+$}) { |f| File.basename(f) }
+  # spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency 'chromaprint', '~> 0.0.2'
-  spec.add_runtime_dependency 'deep_dive', '~> 0.3'
-  spec.add_runtime_dependency 'libis-mapi', '~> 0.3'
-  spec.add_runtime_dependency 'libis-tools', '~> 1.1'
-  spec.add_runtime_dependency 'mini_magick', '~> 4.12'
-  spec.add_runtime_dependency 'naturally', '~> 2.2'
-  spec.add_runtime_dependency 'new_rfc_2047', '~> 1.0'
-  spec.add_runtime_dependency 'os', '~> 1.1'
-  spec.add_runtime_dependency 'pdfinfo', '~> 1.4'
-  spec.add_runtime_dependency 'pdfkit', '~> 0.8'
-
+  spec.add_development_dependency 'rake', '~> 10.3'
+  spec.add_development_dependency 'rspec', '~> 3.1'
   spec.add_development_dependency 'awesome_print'
-  spec.add_development_dependency 'equivalent-xml'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec'
-  if Gem::Platform::JAVA && spec.platform == Gem::Platform::JAVA
+  spec.add_development_dependency 'equivalent-xml', '~> 0.5'
+  if spec.platform == Gem::Platform::JAVA
     spec.add_development_dependency 'saxon-xslt'
   else
     spec.add_development_dependency 'nokogiri'
   end
+
+  spec.add_runtime_dependency 'libis-tools', '~> 1.0'
+  spec.add_runtime_dependency 'os', '= 0.9.6'
+  spec.add_runtime_dependency 'mini_magick', '~> 4.3'
+  spec.add_runtime_dependency 'deep_dive', '~> 0.3'
+  spec.add_runtime_dependency 'chromaprint', '~> 0.0.2'
+  spec.add_runtime_dependency 'naturally', '~> 2.1'
 end