libis-format 0.9.41 → 0.9.44

data/lib/libis/format/fido.rb

@@ -1,89 +0,0 @@
-require 'libis/tools/extend/string'
-require 'libis/tools/command'
-
-require 'csv'
-require 'libis/format/config'
-
-require_relative 'identification_tool'
-
-module Libis
-  module Format
-
-    class Fido < Libis::Format::IdentificationTool
-
-      def self.add_formats(formats_file)
-        self.instance.formats << formats_file unless self.instance.formats.include?(formats_file)
-      end
-
-      def self.del_formats(formats_file)
-        self.instance.formats.delete(formats_file)
-      end
-
-      attr_reader :formats
-
-      def run_list(filelist)
-        create_list_file(filelist) do |list_file|
-          output = runner(nil, '-input', list_file.escape_for_string)
-          process_output(output)
-        end
-      end
-
-      def run_dir(dir, recursive = true)
-        args = []
-        args << '-recurse' if recursive
-        output = runner(dir, *args)
-        process_output(output)
-      end
-
-      def run(file)
-        output = runner(file)
-        process_output(output)
-      end
-
-      protected
-
-      def initialize
-        super
-        @formats = Libis::Format::Config[:fido_formats].dup
-        bad_mimetype('application/vnd.oasis.opendocument.text')
-        bad_mimetype('application/vnd.oasis.opendocument.spreadsheet')
-      end
-
-      attr_writer :formats
-
-      def runner(filename, *args)
-        # Load custome format definitions if present
-        args << '-loadformats' << "#{formats.join(',')}" unless formats.empty?
-
-        # Workaround for Fido performance bug
-        args << '-bufsize' << '1000'
-
-        # Add filename to argument list (optional)
-        args << "#{filename.escape_for_string}" if filename
-
-        # No header output
-        args << '-q'
-
-        # Run command and capture results
-        fido = ::Libis::Tools::Command.run(Libis::Format::Config[:fido_path], *args)
-
-        # Log warning if needed
-        raise RuntimeError, "Fido errors: #{fido[:err].join("\n")}" unless fido[:err].empty?
-
-        # Parse output (CSV) text into array and return result
-        keys = [:status, :time, :puid, :format_name, :format_version, :filesize, :filepath, :mimetype, :matchtype]
-        result = CSV.parse(fido[:out].join("\n"))
-                     .map {|a| Hash[keys.zip(a)]}
-                     .select {|a| a[:status] == 'OK'}
-        result.each do |r|
-          r.delete(:time)
-          r.delete(:status)
-          r.delete(:filesize)
-          r[:source] = :fido
-        end
-      end
-
-    end
-
-  end
-end

data/lib/libis/format/file_tool.rb

@@ -1,76 +0,0 @@
-require_relative 'identification_tool'
-
-module Libis
-  module Format
-
-    class FileTool < Libis::Format::IdentificationTool
-
-      def run_list(filelist)
-
-        create_list_file(filelist) do |list_file|
-
-          output = runner(nil, '--files-from', list_file)
-
-          process_output(output)
-
-        end
-
-      end
-
-      def run_dir(dir, recursive = true)
-
-        filelist = find_files(dir, recursive)
-
-        create_list_file(filelist) do |list_file|
-
-          output = runner(nil, '--files-from', list_file)
-
-          process_output(output)
-
-        end
-
-      end
-
-      def run(file)
-
-        output = runner(file)
-
-        process_output(output)
-
-      end
-
-      protected
-
-      def runner(filename, *args)
-
-        # Create new argument list
-        opts = []
-
-        # Add fixed options
-        # -L : follow symlinks
-        # --mime-type : only print MIME type
-        opts << '-L' << '--mime-type'
-
-        # Append passed arguments
-        opts += args
-
-        # Finally add the filename to process
-        opts << filename.escape_for_string if filename
-
-        # Run the UNIX file command and capture the results
-        file_tool = ::Libis::Tools::Command.run('file', *opts)
-
-        raise RuntimeError, "File command errors: #{file_tool[:err].join("\n")}" unless file_tool[:err].empty?
-
-
-        # Parse output text into array and return result
-        file_tool[:out].map do |line|
-          r = line.split(/:\s+/)
-          {filepath: r[0], mimetype: r[1], matchtype: 'magic', source: :file}
-        end
-      end
-
-    end
-
-  end
-end

data/lib/libis/format/identification_tool.rb

@@ -1,174 +0,0 @@
-require 'csv'
-require 'tmpdir'
-
-require 'singleton'
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
-
-require 'libis/format/config'
-require 'libis/format/type_database'
-
-module Libis
-  module Format
-
-    class IdentificationTool
-      include Singleton
-      include ::Libis::Tools::Logger
-
-      def self.bad_mimetype(mimetype)
-        self.instance.bad_mimetype(mimetype)
-      end
-
-      def self.run(file, recursive = false)
-        if file.is_a?(Array)
-          return run_list file
-        elsif file.is_a?(String) && File.exists?(file) && File.readable?(file)
-          if File.directory?(file)
-            return run_dir(file, recursive)
-          elsif File.file?(file)
-            return self.instance.run(file)
-          end
-        end
-        raise ArgumentError,
-              'IdentificationTool: file argument should be a path to an existing file or directory or a list of those'
-      end
-
-      def self.run_dir(file, recursive = true)
-        self.instance.run_dir file, recursive
-      end
-
-      def self.run_list(filelist)
-        self.instance.run_list filelist
-      end
-
-      protected
-
-      def create_list_file(filelist)
-        list_file = Dir::Tmpname.make_tmpname(%w'file .list', nil)
-        File.open(list_file, 'w') do |f|
-          filelist.each do |fname|
-            f.write "#{fname}\n"
-          end
-        end
-        yield(list_file)
-      ensure
-        File.delete(list_file)
-      end
-
-      def find_files(dir, recurse = true)
-        args = []
-        args << '-L'
-        args << dir.escape_for_string
-        args << '-maxdepth' << '1' unless recurse
-        args << '-type' << 'f'
-        args << '-print'
-        output = ::Libis::Tools::Command.run('find', *args)
-        warn "Find command errors: #{output[:err].join("\n")}" unless output[:err].empty?
-        output[:out]
-      end
-
-      # Reformat output to make it easier to post-process and decide on the preferred format
-      #
-      # input format:
-      # [
-      #   { filepath: <filename>, mimetype: <mimetype>, matchtype: <matchtype>, ... }
-      # ]
-      #
-      # output format:
-      #   { <filename> => [<result>, ...], ... }
-      #
-      # <result> is the enchanced Hash output of the identification tool:
-      #   { mimetype: <mimetype>, puid: <puid>, matchtype: <matchtype>, score: <score>, ...}
-      #
-      def process_output(output)
-        output.reduce({}) do |results, x|
-          filepath = x.delete(:filepath)
-          results[filepath] ||= []
-          results[filepath.freeze] << annotate(x)
-          results
-        end
-      end
-
-      # Enhance the output with mimetype and score
-      def annotate(result)
-        # Enhance result with mimetype if needed
-        if bad_mimetypes.include?(result[:mimetype]) && !bad_puids.include?(result[:puid])
-          result[:mimetype] = get_mimetype(result[:puid])
-        end
-
-        # Normalize the mimetype
-        Libis::Format::TypeDatabase.normalize(result, PUID: :puid, MIME: :mimetype)
-
-        # Default score is 5
-        result[:score] = 5
-
-        # Weak detection score is 1
-        result[:score] = 1 if bad_mimetypes.include? result[:mimetype]
-
-        # freeze all strings
-        result.each {|_, v| v.freeze if v.is_a?(String)}
-
-        # Adapt score based on matchtype
-        result[:matchtype] = result[:matchtype].to_s.downcase
-        case result[:matchtype]
-
-          # Signature match increases score with 2
-          when 'signature'
-            result[:score] += 2
-          # typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(result[:puid])
-          # ext = File.extname(result[:filename])
-          # result[:score] += 1 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
-
-          # Container match increases score with 4
-          when 'container'
-            result[:score] += 4
-          # typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(result[:puid])
-          # ext = File.extname(result[:filename])
-          # result[:score] += 1 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
-
-          # Extension match is the weakest identification; score is lowered by 2 points
-          when 'extension'
-            result[:score] -= 2
-
-          # Magic code (file tool) is to be trused even less
-          when 'magic'
-            result[:score] -= 3
-
-          # Or no change otherwise
-          else
-            # do nothing
-        end
-
-        # Detecting a zip file should decrease the score as it may hide one of the many zip-based formats (e.g. epub,
-        # Office OpenXML, OpenDocument, jar, maff, svx)
-        if result[:mimetype] == 'application/zip'
-          result[:score] -= 2
-        end
-
-        # Return result enhanced with mimetype and score fields
-        result
-      end
-
-      def get_mimetype(puid)
-        ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first rescue nil
-      end
-
-      def get_puid(mimetype)
-        ::Libis::Format::TypeDatabase.mime_infos(mimetype).first[:PUID].first rescue nil
-      end
-
-      attr_accessor :bad_mimetypes, :bad_puids
-
-      def initialize
-        @bad_mimetypes = [nil, '', 'None', 'application/octet-stream']
-        @bad_puids = [nil, 'fmt/unknown']
-      end
-
-      def bad_mimetype(mimetype)
-        @bad_mimetypes << mimetype
-      end
-    end
-
-  end
-end

data/lib/libis/format/office_to_pdf.rb

@@ -1,52 +0,0 @@
-require 'fileutils'
-
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
-
-require 'libis/format/config'
-
-module Libis
-  module Format
-
-    class OfficeToPdf
-      include ::Libis::Tools::Logger
-
-      def self.run(source, target, options = {})
-        self.new.run source, target, options
-      end
-
-      def run(source, target, options = {})
-        workdir = '/...'
-        workdir = Dir.tmpdir unless Dir.exist? workdir
-
-        workdir = File.join(workdir, rand(1000000).to_s)
-        FileUtils.mkpath(workdir)
-
-        src_file = File.join(workdir, File.basename(source))
-        FileUtils.symlink source, src_file
-
-        tgt_file = File.join(workdir, File.basename(source, '.*') + '.pdf')
-
-        export_filter = options[:export_filter] || 'pdf'
-
-        result = Libis::Tools::Command.run(
-            Libis::Format::Config[:soffice_path], '--headless',
-            '--convert-to', export_filter,
-            '--outdir', workdir, src_file
-        )
-
-        unless result[:status] == 0
-          warn "PdfConvert errors: #{(result[:err] + result[:out]).join("\n")}"
-          return false
-        end
-
-        FileUtils.copy tgt_file, target, preserve: true
-        FileUtils.rmtree workdir
-
-        result[:out]
-      end
-    end
-
-  end
-end

data/lib/libis/format/pdf_copy.rb

@@ -1,40 +0,0 @@
-require 'os'
-
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
-
-require 'libis/format/config'
-
-module Libis
-  module Format
-
-    class PdfCopy
-      include ::Libis::Tools::Logger
-
-      def self.run(source, target, options = [])
-        self.new.run source, target, options
-      end
-
-      def run(source, target, options = [])
-        tool_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'tools'))
-        jar_file = File.join(tool_dir, 'PdfTool.jar')
-
-        if OS.java?
-          # TODO: import library and execute in current VM. For now do exactly as in MRI.
-        end
-
-        Libis::Tools::Command.run(
-            Libis::Format::Config[:java_path],
-            '-cp', jar_file,
-            'CopyPdf',
-            '--file_input', source,
-            '--file_output', target,
-            *options
-        )
-
-      end
-    end
-
-  end
-end

data/lib/libis/format/pdf_merge.rb

@@ -1,41 +0,0 @@
-require 'os'
-
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
-
-require 'libis/format/config'
-
-module Libis
-  module Format
-
-    class PdfMerge
-      include ::Libis::Tools::Logger
-
-      def self.run(source, target, options = [])
-        self.new.run source, target, options
-      end
-
-      def run(source, target, options = [])
-        tool_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'tools'))
-        jar_file = File.join(tool_dir, 'PdfTool.jar')
-        source = [source] unless source.is_a?(Array)
-
-        if OS.java?
-          # TODO: import library and execute in current VM. For now do exactly as in MRI.
-        end
-
-        Libis::Tools::Command.run(
-            Libis::Format::Config[:java_path],
-            '-cp', jar_file,
-            'MergePdf',
-            '--file_output', target,
-            *options,
-            *source,
-        )
-
-      end
-    end
-
-  end
-end

data/lib/libis/format/pdf_optimizer.rb

@@ -1,36 +0,0 @@
-require 'os'
-
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
-
-require 'libis/format/config'
-
-module Libis
-  module Format
-
-    class PdfOptimizer
-      include ::Libis::Tools::Logger
-
-      def self.run(source, target, quality)
-        self.new.run source, target, quality
-      end
-
-      def run(source, target, quality)
-
-        Libis::Tools::Command.run(
-            'gs',
-            '-sDEVICE=pdfwrite',
-            '-dCompatibilityLevel=1.4',
-            "-dPDFSETTINGS=/#{quality}",
-            '-dNOPAUSE',
-            '-dBATCH',
-            "-sOutputFile=#{target}",
-            "#{source}"
-        )
-
-      end
-    end
-
-  end
-end

data/lib/libis/format/pdf_split.rb

@@ -1,39 +0,0 @@
-require 'os'
-
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
-
-require 'libis/format/config'
-
-module Libis
-  module Format
-
-    class PdfSplit
-      include ::Libis::Tools::Logger
-
-      def self.run(source, target, options = [])
-        self.new.run source, target, options
-      end
-
-      def run(source, target, options = [])
-        tool_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'tools'))
-        jar_file = File.join(tool_dir, 'PdfTool.jar')
-        if OS.java?
-          # TODO: import library and execute in current VM. For now do exactly as in MRI.
-        end
-
-        Libis::Tools::Command.run(
-            Libis::Format::Config[:java_path],
-            '-cp', jar_file,
-            'SplitPdf',
-            '--file_input', source,
-            '--file_output', target,
-            *options
-        )
-
-      end
-    end
-
-  end
-end

data/lib/libis/format/pdf_to_pdfa.rb

@@ -1,74 +0,0 @@
-require 'tempfile'
-require 'csv'
-require 'fileutils'
-
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
-
-require 'libis/format'
-
-module Libis
-  module Format
-
-    class PdfToPdfa
-      include ::Libis::Tools::Logger
-
-      def self.run(source, target = nil, options = {})
-        self.new.run source, target, options
-      end
-
-      def run(source, target = nil, options = nil)
-
-        target ||= File.join(Dir.tmpdir, Dir::Tmpname.make_tmpname([File.basename(source, '.*'), '.pdf']))
-
-        icc_info = icc_options(options[:colorspace])
-
-        icc_file = File.join(Dir.tmpdir, "#{icc_info[:icc_name]}#{Random.new.bytes(12).unpack('H*').first}.icc")
-        FileUtils.cp(File.join(Libis::Format::DATA_DIR, "#{icc_info[:icc_name]}.icc"), icc_file)
-
-        def_filename = File.join(Dir.tmpdir, "PDFA_def_#{Random.new.bytes(12).unpack('H*').first}.ps")
-        File.open(def_filename, 'w') do |f|
-          f.puts File.read(File.join(Libis::Format::DATA_DIR, 'PDFA_def.ps')).
-                     gsub('[** Fill in ICC profile location **]', icc_file).
-                     gsub('[** Fill in ICC reference name **]', icc_info[:icc_ref])
-        end
-
-        result = Libis::Tools::Command.run(
-            Libis::Format::Config[:ghostscript_path],
-            '-dBATCH', '-dNOPAUSE', '-dNOOUTERSAVE',
-            '-sColorConversionStrategy=/UseDeviceIndependentColor',
-            "-sProcessColorModel=#{icc_info[:device]}",
-            '-sDEVICE=pdfwrite', '-dPDFA', '-dPDFACompatibilityPolicy=1',
-            "-sOutputICCProfile=#{icc_file}",
-            '-o', File.absolute_path(target),
-            def_filename,
-            source
-        )
-
-        FileUtils.rm [icc_file, def_filename].compact, force: true
-
-        unless PdfaValidator.run(target)
-          result[:status] = -999
-          result[:err] << 'Failed to validate generated PDF/A file.'
-        end
-
-        result
-      end
-
-
-      private
-
-      def icc_options(colorspace)
-        case colorspace.to_s.downcase
-          when 'cmyk'
-            {icc_name: 'ISOcoated_v2_eci', icc_ref: 'FOGRA39L', device: 'DeviceCMYK'}
-          else
-            {icc_name: 'eciRGB_v2', icc_ref: 'sRGB', device: 'DeviceRGB'}
-        end
-      end
-
-    end
-
-  end
-end

data/lib/libis/format/pdfa_validator.rb

@@ -1,61 +0,0 @@
-require 'fileutils'
-
-require 'libis/tools/extend/string'
-require 'libis/tools/logger'
-require 'libis/tools/command'
-
-require 'libis/format/config'
-
-module Libis
-  module Format
-
-    class PdfaValidator
-      include ::Libis::Tools::Logger
-
-      def self.run(source)
-        self.new.run source
-      end
-
-      def run(source)
-
-        src_file = File.absolute_path(source)
-
-        if (pdfa = Libis::Format::Config[:pdfa_path])
-          # Keep it clean: tool generates fontconfig/ cache dir in current working dir
-          previous_wd = Dir.getwd
-          Dir.chdir(Dir.tmpdir)
-
-          result = Libis::Tools::Command.run(
-              pdfa,
-              '--noxml',
-              '--level', 'B',
-              '--verb', '0',
-              src_file
-          )
-
-          Dir.chdir(previous_wd)
-
-          unless result[:out].any? { |line| line =~ /^VLD-\[PASS\]/ }
-            warn "Validator failed to validate the PDF file '%s' against PDF/A-1B constraints:\n%s", source,
-                 result[:out].join("\n")
-            return false
-          end
-        else
-          jar = File.join(Libis::Format::ROOT_DIR, 'tools', 'pdfbox', 'preflight-app-1.8.10.jar')
-          result = Libis::Tools::Command.run(
-              Libis::Format::Config[:java_path],
-              '-jar', jar,
-              src_file
-          )
-          unless result[:status] == 0
-            warn "Validator failed to validate the PDF file '%s' against PDF/A-1B constraints:\n%s", source,
-                 result[:out].join("\n")
-            return false
-          end
-        end
-        true
-      end
-    end
-
-  end
-end