libis-format 0.9.5-java

data/lib/libis/format/converter/image_converter.rb

@@ -0,0 +1,214 @@
+# encoding: utf-8
+
+require_relative 'base'
+require 'libis/format/identifier'
+
+require 'mini_magick'
+require 'fileutils'
+
+MiniMagick.configure do |config|
+  config.debug = false
+end
+
+module Libis
+  module Format
+    module Converter
+
+      class ImageConverter < Libis::Format::Converter::Base
+
+        def self.input_types(format = nil)
+          [:TIFF, :JPG, :PNG, :BMP, :GIF, :JP2, :PDF]
+        end
+
+        def self.output_types(_ = nil)
+          [:TIFF, :JPG, :PNG, :BMP, :GIF, :JP2, :PDF]
+        end
+
+        def initialize
+          @wm_image = nil
+          super
+        end
+
+        def image_convert(_)
+          #force usage of this converter
+        end
+
+        def scale(percent)
+          @options[:scale] = percent
+        end
+
+        def resize(geometry)
+          @options[:resize] = geometry
+        end
+
+        def quality(value)
+          @options[:quality] = value
+        end
+
+        def dpi(value)
+          @options[:density] = value
+        end
+
+        def resample(value)
+          @options[:density] = value
+        end
+
+        def flatten
+          @flags[:flatten] = true
+        end
+
+        def colorspace(value)
+          @options[:colorspace] = value
+        end
+
+        # Create or use a watermark image.
+        #
+        # The watermark options are:
+        #     - file: watermark image to use
+        #     - text: text to create a watermark from
+        #     - rotation: rotation of the watermark text (counter clockwise in degrees; integer number) - default 30
+        #     - size: size of the watermark (integer > 0, 1/n of image size) - default 4
+        #     - opacity: opacity of the watermark (fraction 0.0 - 1.0) - default 0.3
+        #     - gap: size of the gap between watermark instances. Fractions as percentage of widht/height. - default 0.2
+        # If both options are given, the file will be used as-is if it exists and is a valid image file. Otherwise the
+        # file will be created or overwritten with a newly created watermark image.
+        #
+        # The created watermark file will be a PNG image with transparent background containing the supplied text
+        # slanted by 30 degrees counter-clockwise.
+        #
+        # @param [Hash] options Hash of options for watermark creation.
+        def watermark(options = {})
+          text = options[:text] || '© LIBIS'
+          image = options[:file] || (Dir::Tmpname.create(%w(wm_image .png)) { |_|})
+          @wm_size = (options[:size] || '4').to_i
+          @wm_opacity = ((options[:opacity] || 0.1).to_f * 100).to_i
+          @wm_composition = options[:composition] || 'modulate'
+          gap = ((options[:gap] || 0.2).to_f * 100).to_i
+          rotation = 360 - (options[:rotation] || 30).to_i
+          @wm_image = MiniMagick::Image.new(image)
+          unless @wm_image.valid?
+            # noinspection RubyResolve
+            MiniMagick::Tool::Convert.new do |convert|
+              # noinspection RubyLiteralArrayInspection
+              convert.background 'transparent'
+              convert.size('2000x2000')
+              convert.gravity 'Center'
+              convert.font('Helvetica').fill('black').pointsize(72) #.stroke('black').strokewidth(1)
+              convert << "label:#{text}"
+              convert.rotate rotation
+              convert.trim.repage.+
+              convert.bordercolor('transparent').border("#{gap}%")
+              convert << image
+            end
+            if options[:file]
+              @wm_image = MiniMagick::Image.new(image)
+            else
+              @wm_image = MiniMagick::Image.open(image)
+              File.delete(image)
+            end
+            # noinspection RubyResolve
+            unless @wm_image.valid?
+              error "Problem creating watermark image '#{image}'."
+              @wm_image = nil
+            end
+          end
+        end
+
+        def convert(source, target, format, opts = {})
+          super
+
+          FileUtils.mkpath(File.dirname(target))
+
+          if source.is_a? Array
+            sources = source
+
+            unless [:PDF, :TIFF, :GIF, :PBM, :PGM, :PPM].include? format
+              error 'Can ony assemble multiple images into multi-page/layer format'
+              return nil
+            end
+
+            assemble_and_convert(sources, target, format)
+
+          elsif File.directory?(source)
+            sources = Dir[File.join(source, '**', '*')].reject { |p| File.directory? p }
+
+            unless [:TIFF, :PDF].include? format
+              error 'Can ony assemble multiple images into multi-page/layer format'
+              return nil
+            end
+
+            assemble_and_convert(sources, target, format)
+
+          else
+
+            image = MiniMagick::Image.new(source)
+
+            if image.pages.size > 1
+              assemble_and_convert(image.pages.map {|page| page.path}, target, format)
+            else
+              convert_image(source, target, format)
+            end
+          end
+
+          target
+
+        end
+
+        def assemble_and_convert(sources, target, format)
+          converted_pages = sources.inject([]) do |list, path|
+            converted = Tempfile.new(['page-', ".#{Libis::Format::TypeDatabase.type_extentions(format).first}"])
+            convert_image(path, converted.path, format)
+            list << converted
+          end
+          MiniMagick::Tool::Convert.new do |b|
+            converted_pages.each { |page| b << page.path }
+            b << target
+          end
+          converted_pages.each do |temp_file|
+            temp_file.close
+            temp_file.unlink
+          end
+        end
+
+        protected
+
+        def convert_image(source, target, format)
+
+          image = MiniMagick::Image.new(source)
+
+          MiniMagick::Tool::Convert.new do |convert|
+            if @wm_image
+              convert << @wm_image.path
+              convert.filter('Lagrange')
+              convert.resize("#{image.width / @wm_size}x#{image.height / @wm_size}").write('mpr:watermark').delete.+
+            end
+
+            convert << source
+            convert.flatten if format == :JPG
+            if @wm_image
+              # noinspection RubyResolve
+              convert.stack do |stack|
+                stack.size("#{image.width}x#{image.height}")
+                stack << 'xc:transparent'
+                stack.tile('mpr:watermark')
+                stack.draw "rectangle 0,0,#{image.width},#{image.height}"
+              end
+              convert.compose(@wm_composition).define("compose:args=#{@wm_opacity}").composite
+            end
+
+            @options.each { |o, v| convert.send(o, v) }
+            @flags.each { |f, v| v.is_a?(FalseClass) ? convert.send(f).+ : convert.send(f) }
+
+            convert.format(format)
+            convert << target
+          end
+
+          target
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter/office_converter.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+
+require_relative 'base'
+
+require 'libis/format/office_to_pdf'
+require 'libis/format/type_database'
+
+module Libis
+  module Format
+    module Converter
+
+      class OfficeConverter < Libis::Format::Converter::Base
+
+        def self.input_types(_ = nil)
+          [
+              :TXT,
+              :RTF,
+              :HTML,
+              :MSDOC,
+              :MSXLS,
+              :MSPPT,
+              :MSDOCX,
+              :MSXLSX,
+              :MSPPTX,
+              :WORDPERFECT,
+          ]
+        end
+
+        def self.output_types(_ = nil)
+          [:PDF]
+        end
+
+        def office_convert(_)
+          #force usage of this converter
+        end
+
+        def convert(source, target, format, opts = {})
+          super
+
+          return nil unless OfficeToPdf.run(source, target)
+
+          target
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter/pdf_converter.rb

@@ -0,0 +1,139 @@
+# encoding: utf-8
+
+require_relative 'base'
+
+require 'libis/tools/extend/hash'
+require 'libis/format/pdf_copy'
+require 'libis/format/pdf_to_pdfa'
+
+module Libis
+  module Format
+    module Converter
+
+      class PdfConverter < Libis::Format::Converter::Base
+
+        def self.input_types(_ = nil)
+          [:PDF]
+        end
+
+        def self.output_types(_ = nil)
+          [:PDF, :PDFA]
+        end
+
+        def pdf_convert(_)
+          #force usage of this converter
+        end
+
+        # Set metadata for Pdf file
+        #
+        # valid metadata keys are):
+        #       - title
+        #       - author
+        #       - creator
+        #       - keywords
+        #       - subject
+        #
+        # @param [Hash] values list of metadata values to set
+        def metadata(values = {})
+          values.key_strings_to_symbols!
+          values.each do |k, v|
+            next unless [:title, :author, :creator, :keywords, :subject].include?(k)
+            @options["md_#{k}"] = v
+          end
+        end
+
+        # Select a partial list of pages
+        # @param [String] selection as described in com.itextpdf.text.pdf.SequenceList: [!][o][odd][e][even]start-end
+        def range(selection)
+          @options[:ranges] = selection
+        end
+
+        # Create or use a watermark image.
+        #
+        # The watermark options are (use symbols):
+        #     - text: text to create a watermark from
+        #     - file: watermark image to use
+        #     - rotation: rotation of the watermark text (in degrees; integer number)
+        #     - size: font size of the watermark text
+        #     - opacity: opacity of the watermark (fraction 0.0 - 1.0)
+        #     - gap: size of the gap between watermark instances. Integer value is absolute size in points (1/72 inch). Fractions are percentage of widht/height.
+        # If both options are given, the file will be used as-is if it exists and is a valid image file. Otherwise the
+        # file will be created or overwritten with a newly created watermark image.
+        #
+        # The created watermark file will be a PNG image with transparent background containing the supplied text
+        # slanted by 30 degrees counter-clockwise.
+        #
+        # @param [Hash] options Hash of options for watermark creation.
+        def watermark(options = {})
+          options.key_strings_to_symbols!
+          if options[:file] && File.exist?(options[:file])
+            @options['wm_image'] = options[:file]
+          else
+            @options['wm_text'] = (options[:text] || '© LIBIS').split('\n')
+            @options['wm_text_rotation'] = options[:rotation] if options[:rotation]
+            @options['wm_font_size'] = options[:size] if options[:size]
+          end
+          @options['wm_opacity'] = options[:opacity] || '0.3'
+          @options['wm_gap_ratio'] = options[:gap] if options[:gap].to_s =~ /^\s*(0+\.\d+|1\.0+)\s*$/
+          @options['wm_gap_size'] = options[:gap] if options[:gap].to_s =~ /^\s*\d+\s*$/
+        end
+
+        def convert(source, target, format, opts = {})
+          super
+
+          result = nil
+
+          unless @options.empty?
+            result = convert_pdf(source, target)
+            return nil unless result
+            source = result
+          end
+
+          if format == :PDFA and source
+            result = pdf_to_pdfa(source, target)
+          end
+
+          result
+
+        end
+
+
+        def convert_pdf(source, target)
+
+          using_temp(target) do |tmpname|
+            result = Libis::Format::PdfCopy.run(
+                source, tmpname,
+                @options.map { |k, v|
+                  if v.nil?
+                    nil
+                  else
+                    ["--#{k}", (v.is_a?(Array) ? v : v.to_s)]
+                  end }.flatten
+            )
+            result[:err].empty? ? target : begin
+              error("Pdf conversion encountered errors:\n%s", result[:err].join('\n'))
+              nil
+            end
+            tmpname
+          end
+
+        end
+
+        def pdf_to_pdfa(source, target)
+
+          using_temp(target) do |tmpname|
+            result = Libis::Format::PdfToPdfa.run source, tmpname
+            result[:status] == 0 ? target : begin
+              error("Pdf/A conversion encountered errors:\n%s", result[:err].join('\n'))
+              nil
+            end
+            tmpname
+          end
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter/repository.rb

@@ -0,0 +1,98 @@
+# coding: utf-8
+
+require 'set'
+require 'singleton'
+
+require 'libis/tools/logger'
+require 'libis/format/config'
+
+require_relative 'chain'
+
+module Libis
+  module Format
+    module Converter
+
+      class Repository
+        include Singleton
+        include ::Libis::Tools::Logger
+
+        attr_reader :converters
+        attr_accessor :converters_glob
+
+        def initialize
+          @converters = Set.new
+          @converters_glob = File.join(File.dirname(__FILE__), '*_converter.rb')
+        end
+
+        def Repository.register(converter_class)
+          instance.converters.add? converter_class
+        end
+
+        def Repository.get_converters
+          instance.get_converters
+        end
+
+        def get_converters
+          if converters.empty?
+            Dir.glob(converters_glob).each do |filename|
+              # noinspection RubyResolve
+              require File.expand_path(filename)
+            end
+          end
+          converters
+        end
+
+        def Repository.get_converter_chain(src_type, tgt_type, operations = {})
+          instance.get_converter_chain src_type, tgt_type, operations
+        end
+
+        def get_converter_chain(src_type, tgt_type, operations = {})
+          msg = "conversion from #{src_type.to_s} to #{tgt_type.to_s}"
+          chain_list = find_chains src_type, tgt_type, operations
+          # if chain_list.length > 1
+          #   warn "Found more than one conversion chain for #{msg}. Picking the first one."
+          # end
+          if chain_list.empty?
+            error "No conversion chain found for #{msg}"
+            return nil
+          end
+          # chain_list.each do |chain|
+          #   debug "Matched chain: #{chain}"
+          # end
+          chain_list[0]
+        end
+
+        private
+
+        def find_chains(src_type, tgt_type, operations)
+          chain = Libis::Format::Converter::Chain.new(src_type, tgt_type, operations)
+          build_chains(chain)
+        end
+
+        def build_chains(chain)
+
+          found = []
+          chains = [chain]
+
+          # Avoid chains that are too long
+          Libis::Format::Config[:converter_chain_max_level].times do
+            new_chains = []
+            get_converters.each do |converter|
+              new_chains += chains.map { |c| c.append(converter) }.flatten
+            end
+
+            found = new_chains.select { |c| c.valid?}
+            return found unless found.empty?
+
+            chains = new_chains
+          end
+
+          found
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter.rb

@@ -0,0 +1,11 @@
+# code utf-8
+
+module Libis
+  module Format
+    module Converter
+      autoload :Base, 'libis/format/converter/base'
+      autoload :Chain, 'libis/format/converter/chain'
+      autoload :Repository, 'libis/format/converter/repository'
+    end
+  end
+end

data/lib/libis/format/droid.rb

@@ -0,0 +1,45 @@
+require 'tempfile'
+require 'csv'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+
+module Libis
+  module Format
+
+    class Droid
+      include ::Libis::Tools::Logger
+
+      def self.run(file)
+        self.new.run file
+      end
+
+      def run(file)
+        profile = File.join Dir.tmpdir, Dir::Tmpname.make_tmpname(%w'droid .profile', nil)
+        report = File.join Dir.tmpdir, Dir::Tmpname.make_tmpname(%w'droid .csv', nil)
+        result = Libis::Tools::Command.run(
+            Libis::Format::Config[:droid_path],
+            '-a', file.escape_for_string,
+            '-p', profile,
+            '-q',
+        )
+        warn "DROID profile errors: #{result[:err].join("\n")}" unless result[:status] == 0
+        result = Libis::Tools::Command.run(
+            Libis::Format::Config[:droid_path],
+            '-e', report,
+            '-p', profile,
+            '-q'
+        )
+        warn "DROID report errors: #{result[:err].join("\n")}" unless result[:status] == 0
+        File.delete profile
+        result = CSV.read(report , headers: true, header_converters: [:downcase, :symbol])
+        File.delete report
+        result.map{|r|r.to_hash}
+      end
+    end
+
+  end
+end

data/lib/libis/format/fido.rb

@@ -0,0 +1,102 @@
+require 'csv'
+
+require 'libis/tools/extend/string'
+require 'libis/tools/logger'
+require 'libis/tools/command'
+
+require 'libis/format/config'
+require 'libis/format/type_database'
+
+module Libis
+  module Format
+
+    class Fido
+      include ::Libis::Tools::Logger
+
+      BAD_MIMETYPES = [nil, '', 'None', 'application/octet-stream']
+
+      def self.run(file, formats = nil)
+        self.new.run file, formats
+      end
+
+      def run(file, xtra_formats = nil)
+
+        fido_results = []
+
+        fmt_list = formats.dup
+        case xtra_formats
+          when Array
+            fmt_list += xtra_formats
+          when String
+            fmt_list << xtra_formats
+          else
+            # do nothing
+        end
+
+        args = []
+        args << '-loadformats' << "#{fmt_list.join(',')}" unless fmt_list.empty?
+        args << "#{file.escape_for_string}"
+        fido = ::Libis::Tools::Command.run(Libis::Format::Config[:fido_path], *args)
+        warn "Fido errors: #{fido[:err].join("\n")}" unless fido[:err].empty?
+
+        keys = [:status, :time, :puid, :format_name, :signature_name, :filesize, :filename, :mimetype, :matchtype]
+        fido_output = CSV.parse(fido[:out].join("\n")).map { |a| Hash[keys.zip(a)] }
+
+        fido_output.each do |x|
+          if x[:status] == 'OK'
+            x[:mimetype] = get_mimetype(x[:puid]) if x[:mimetype] == 'None'
+            next if BAD_MIMETYPES.include? x[:mimetype]
+            x[:score] = 5
+            case x[:matchtype]
+              when 'signature'
+                x[:score] += 5
+              when 'container'
+                typeinfo = ::Libis::Format::TypeDatabase.puid_typeinfo(x[:puid])
+                ext = File.extname(file)
+                x[:score] += 2 if typeinfo and typeinfo[:EXTENSIONS].include?(ext)
+              else
+                # do nothing
+            end
+            fido_results << x
+          end
+        end
+
+        fido_results = fido_results.inject({}) do |result, value|
+          result[value[:score]] ||= []
+          result[value[:score]] << value
+          result
+        end
+
+        max_score = fido_results.keys.max
+
+        # Only if we find a single hit of type 'signature' or 'container', we are confident enough to return a result
+        return {} unless max_score and max_score >= 5 && fido_results[max_score].size == 1
+
+        fido_results[max_score].first
+      end
+
+      def self.add_format(f)
+        instance.formats << f
+      end
+
+      def self.formats
+        instance.formats
+      end
+
+      protected
+
+      attr_reader :formats
+
+      def initialize
+        data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
+        @formats = [(File.join(data_dir, 'lias_formats.xml'))]
+      end
+
+      def get_mimetype(puid)
+        ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first rescue nil
+      end
+
+    end
+
+  end
+end