libis-format 1.0.2 → 2.0.1

data/lib/libis/format/cli/format.rb

@@ -0,0 +1,77 @@
+require 'awesome_print'
+require 'libis-format'
+require 'libis-tools'
+
+require_relative 'sub_command'
+
+module Libis
+  module Format
+    module Cli
+      class Format < SubCommand
+
+        no_commands do
+          def self.description(field)
+            "#{STRING_CONFIG[field]}." + (DEFAULT_CONFIG[field].nil? ? '' : " default: #{DEFAULT_CONFIG[field]}")
+          end
+        end
+
+        DEFAULT_CONFIG = {
+            droid: true,
+            fido: true,
+            file: true,
+            xml_validation: true
+        }
+
+        STRING_CONFIG = {
+            droid: 'Use Droid to identify the format',
+            fido: 'Use Fido to identify the format',
+            file: 'Use File to identify the format',
+            xml_validation: 'When XML file found, validate the file against known XML schemas',
+        }
+
+        desc 'identify FILE [options]', 'Identify the FILE using a combination of the tools.'
+        long_desc <<-DESC
+  
+        'identify FILE [options]' will idnetify a file using a combination of tools.
+
+        The file will be identified by each of the selected tools in turn and the format with best score will be
+        selected to give the final result. The score is determined by the identification method and will be lowered if
+        a known weak format (e.g. zip file) is detected.
+
+        The tool will display as much information about the format as possible, including the format candidates that
+        were not selected.
+
+        DESC
+
+        method_option :droid, default: DEFAULT_CONFIG[:droid], type: :boolean, desc: STRING_CONFIG[:droid]
+        method_option :fido, default: DEFAULT_CONFIG[:fido], type: :boolean, desc: STRING_CONFIG[:fido]
+        method_option :file, default: DEFAULT_CONFIG[:file], type: :boolean, desc: STRING_CONFIG[:file]
+        method_option :xml_validation, default: DEFAULT_CONFIG[:xml_validation], type: :boolean, desc: STRING_CONFIG[:xml_validation]
+
+        def identify(source_file)
+          ::Libis::Tools::Config.logger.level = :WARN
+          opts = options.inject({}) { |h, x| h[x.first.to_sym] = x.last; h}
+          opts[:keep_output] = true
+          result = ::Libis::Format::Identifier.get source_file, opts
+          puts '--- messages ---'
+          result[:messages].each do |message|
+            puts "#{message[0]} : #{message[1]}"
+          end
+
+          puts '--- formats ---'
+          result[:formats].each do |file, info|
+            puts "#{file}:"
+            ap info
+          end
+
+          puts '--- tool results ---'
+          result[:output].each do |file, info|
+            puts "#{file}:"
+            ap info
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/format/cli/prompt_helper.rb

@@ -0,0 +1,87 @@
+require 'tty-prompt'
+require 'pastel'
+
+module Libis
+  module Format
+    module Cli
+      module PromptHelper
+
+        attr_reader :prompt, :pastel
+
+        def initialize(*args)
+          @prompt = TTY::Prompt.new
+          @pastel = Pastel.new
+          super
+        end
+
+        protected
+
+        private
+
+        def index_of(list, value)
+          i = list.index(value)
+          i += 1 if i
+          i || 1
+        end
+
+        def ask(question, bool: false, enum: nil, default: nil, mask: false)
+          cmd, args, opts = :ask, [question], {}
+          if enum
+            cmd = :select
+            args << enum
+            # Change default to its index in the enum
+            default = index_of(enum, default)
+          end
+          cmd = :mask if mask
+          opts[:default] = default if default
+          cmd = (opts[:default] ? :yes? : :no?) if bool
+          prompt.send(cmd, *args, opts)
+        end
+
+        def tree_select(path, question: nil, file: false, page_size: 22, filter: true, cycle: false, create: false,
+                        default_choices: nil)
+          path = Pathname.new(path) unless path.is_a? Pathname
+
+          return path unless path.exist?
+          path = path.realpath
+
+          dirs = path.children.select(&:directory?).sort
+          files = file ? path.children.select(&:file?).sort : []
+
+          choices = []
+          choices << {name: "Folder: #{path}", value: path, disabled: file ? '' : false}
+          choices += default_choices if default_choices
+          choices << {name: '-- new directory --', value: -> do
+            new_name = prompt.ask('new directory name:', modify: :trim, required: true)
+            new_path = path + new_name
+            FileUtils.mkdir(new_path.to_path)
+            new_path
+          end
+          } if create
+
+          choices << {name: "-- new file --", value: -> do
+            new_name = prompt.ask('new file name:', modify: :trim, required: true)
+            path + new_name
+          end
+          } if file && create
+
+          choices << {name: '[..]', value: path.parent}
+
+          dirs.each {|d| choices << {name: "[#{d.basename}]", value: d}}
+          files.each {|f| choices << {name: f.basename.to_path, value: f}}
+
+          question ||= "Select #{'file or ' if files}directory"
+          selection = prompt.select question, choices,
+                                    per_page: page_size, filter: filter, cycle: cycle, default: file ? 2 : 1
+
+          return selection unless selection.is_a? Pathname
+          return selection.to_path if selection == path || selection.file?
+
+          tree_select selection, question: question, file: file, page_size: page_size, filter: filter,
+                      cycle: cycle, create: create, default_choices: default_choices
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/format/cli/sub_command.rb

@@ -0,0 +1,22 @@
+require 'thor'
+require_relative 'prompt_helper'
+
+module Libis
+  module Format
+    module Cli
+      class SubCommand < Thor
+
+        include PromptHelper
+
+        def self.banner(command, namespace = nil, subcommand = false)
+          "#{basename} #{subcommand_prefix} #{command.usage}"
+        end
+
+        def self.subcommand_prefix
+          self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+        end
+
+      end
+    end
+  end
+end

data/lib/libis/format/command_line.rb

@@ -0,0 +1,25 @@
+require 'thor'
+require 'tty-prompt'
+require 'tty-config'
+
+require 'libis/format/cli/convert'
+require 'libis/format/cli/format'
+
+module Libis
+  module Format
+
+    class CommandLine < Thor
+
+      def self.exit_on_failure?
+        true
+      end
+
+      desc 'convert', 'perform format conversion on a given file'
+      subcommand 'convert', Cli::Convert
+
+      desc 'format', 'perform format identification on a given file or directory'
+      subcommand 'format', Cli::Format
+    end
+
+  end
+end

data/lib/libis/format/config.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'os'
+require 'libis-tools'
 
 module Libis
   module Format
@@ -9,20 +10,23 @@ module Libis
 
     Config[:converter_chain_max_level] = 8
 
-    Config[:java_path] = 'java'
-    Config[:j2kdriver] = 'j2kdriver'
-    Config[:soffice_path] = 'soffice'
-    Config[:ghostscript_path] = 'gs'
-    Config[:droid_path] = '/opt/droid/droid.sh'
-    Config[:fido_path] = '/usr/local/bin/fido'
-    Config[:fop_jar] = '/opt/fop/current/fop/build/fop.jar'
-    Config[:ffmpeg_path] = 'ffmpeg'
+    Config[:java_cmd] = 'java'
+    Config[:j2k_cmd] = 'j2kdriver'
+    Config[:soffice_cmd] = 'soffice'
+    Config[:ghostscript_cmd] = 'gs'
+    Config[:droid_cmd] = '/opt/droid/droid.sh'
+    Config[:droid_temp_path] = '/tmp'
+    Config[:fido_cmd] = '/usr/local/bin/fido'
+    Config[:fop_jar] = File.join(Libis::Format::TOOL_DIR, 'fop', 'build', 'fop.jar')
+    Config[:ffmpeg_cmd] = 'ffmpeg'
     Config[:fido_formats] = [(File.join(Libis::Format::DATA_DIR, 'lias_formats.xml'))]
     Config[:pdf_tool] = File.join(Libis::Format::TOOL_DIR, 'PdfTool.jar')
+    Config[:preflight_jar] = File.join(Libis::Format::TOOL_DIR, 'pdfbox', 'preflight-app-2.0.13.jar')
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(Libis::Format::DATA_DIR, 'ead.xsd')]]
-    Config[:type_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
-    Config[:raw_audio_convert_cmd] = 'sox %s -e signed -b 16 -t wav %s rate %d channels %d'
+    Config[:format_library_implementation] = 'Libis::Format::YamlLoader.instance'
+    Config[:format_library_database] = File.join(Libis::Format::DATA_DIR, 'types.yml')
+    Config[:raw_audio_convert_cmd] = 'sox -V1 %s -e signed -b 16 -t wav %s rate %d channels %d'
     Config[:watermark_font] = '/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf'
     Config[:timeouts] = {
         droid: 20 * 60,
@@ -32,6 +36,7 @@ module Libis
         fop: 5 * 60,
         identification_tool: 5 * 60,
         office_to_pdf: 5 * 60,
+        pdf_select: 5 * 60,
         pdf_copy: 5 * 60,
         pdf_merge: 5 * 60,
         pdf_optimizer: 5 * 60,

data/lib/libis/format/converter/audio_converter.rb

@@ -15,7 +15,7 @@ module Libis
         end
 
         def self.output_types(format = nil)
-          return [] unless input_types.include?(format)
+          return [] unless format.nil? || input_types.include?(format)
           [:MP3, :FLAC, :AC3, :AAC, :WMA, :ALAC, :WAV, :AIFF, :AMR, :AU, :M4A]
         end
 
@@ -69,50 +69,16 @@ module Libis
           (@options[:preset] ||= {})[stream] = name
         end
 
-        # def encoder(value)
-        #   @options[:encoder] = value
-        # end
-        #
-        # def encoder_options(value)
-        #   @options[:encoder_options] = value
-        # end
-
         def convert(source, target, _format, opts = {})
           super
 
           FileUtils.mkpath(File.dirname(target))
 
-          if source.is_a? Array
-
-            assemble_and_convert(source, target)
-
-          elsif File.directory?(source)
-
-            sources = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
-            assemble_and_convert(sources, target)
+          convert_file(source, target)
 
-          else
-
-            convert_file(source, target)
-
-          end
-
-          target
-
-        end
-
-        def assemble_and_convert(sources, target)
-          Tempfile.create(%w(list .txt)) do |f|
-            sources.each {|src| f.puts src}
-            opts[:global] ||= []
-            opts[:global] += %w(-f concat)
-            f.close
-            target = convert_file(f.to_path, target)
-          end
-          target
         end
 
-        def sounds_like(file1, file2, threshold, rate, channels)
+        def self.sounds_like(file1, file2, threshold, rate, channels)
           rate ||= 96000
           channels ||= 2
           threshold ||= 0.85

data/lib/libis/format/converter/base.rb

@@ -4,7 +4,8 @@
 require 'tmpdir'
 require 'libis/tools/logger'
 require 'libis/tools/temp_file'
-require 'libis/format/type_database'
+require 'libis/tools/extend/string'
+require 'libis/format/library'
 
 require_relative 'repository'
 
@@ -22,10 +23,23 @@ module Libis
           @flags = {}
         end
 
+        def self.category
+          :converter
+        end
+
+        def check_file_exist(file)
+          unless File.exist? file
+            error "Cannot find file '#{file}'."
+            return false
+          end
+          true
+        end
+
         def convert(source, target, format, opts = {})
-          unless File.exist? source
-            error "Cannot find file '#{source}'."
-            return nil
+          if source.is_a?(Array)
+            return nil unless source.map { |f| check_file_exist(f) }.reduce(:&)
+          else
+            return nil unless check_file_exist(source)
           end
           @options.merge!(opts[:options]) if opts[:options]
           @flags.merge!(opts[:flags]) if opts[:flags]
@@ -51,7 +65,7 @@ module Libis
           target
         end
 
-        def Base.inherited( klass )
+        def Base.inherited(klass)
 
           Repository.register klass
 
@@ -73,12 +87,12 @@ module Libis
             end
 
             def input_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               input_type? type_id
             end
 
             def output_mimetype?(mimetype)
-              type_id = TypeDatabase.instance.mime_types(mimetype).first
+              type_id = Libis::Format::Library.get_field_by(:mimetype, mimetype, :format)
               output_type? type_id
             end
 
@@ -91,7 +105,7 @@ module Libis
             end
 
             def extension?(extension)
-              !TypeDatabase.ext_types(extension).first.nil?
+              !Libis::Format::Library.get_field_by(:extension, extension, :format).nil?
             end
 
           end

data/lib/libis/format/converter/chain.rb

@@ -5,7 +5,7 @@ require 'deep_dive'
 
 require 'libis/tools/logger'
 require 'libis/tools/extend/hash'
-require 'libis/format/type_database'
+require 'libis/format/library'
 
 module Libis
   module Format
@@ -83,8 +83,8 @@ module Libis
             target = target_file
 
             if i < size - 1
-              target += ".temp.#{TypeDatabase.type_extentions(target_type).first}"
-              target += ".#{TypeDatabase.type_extentions(target_type).first}" while File.exist? target
+              target += ".temp.#{Libis::Format::Library.get_field(target_type, :extensions).first}"
+              target += ".#{Libis::Format::Library.get_field(target_type, :extensions).first}" while File.exist? target
               temp_files << target
             end
 

data/lib/libis/format/converter/image_assembler.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+
+require_relative 'base'
+require 'libis/format/identifier'
+
+require 'mini_magick'
+
+MiniMagick.logger.level = ::Logger::UNKNOWN
+
+MiniMagick.configure do |config|
+  # config.cli = :graphicsmagick
+  config.validate_on_create = false
+  config.validate_on_write = false
+  config.whiny = false
+end
+
+module Libis
+  module Format
+    module Converter
+
+      # noinspection RubyTooManyInstanceVariablesInspection,DuplicatedCode
+      class ImageAssembler < Libis::Format::Converter::Base
+
+        def self.input_types
+          [:TIFF, :JPG, :PNG, :BMP, :GIF, :PDF, :JP2]
+        end
+
+        def self.output_types(format = nil)
+          return [] unless input_types.include?(format) if format
+          [:PDF, :TIFF, :GIF, :PBM, :PGM, :PPM]
+        end
+
+        def self.category
+          :assembler
+        end
+
+        def image_assemble(_)
+          #force usage of this converter
+        end
+
+        def quiet(v)
+          @quiet = !!v
+        end
+
+        def convert(source, target, format, opts = {})
+          super
+
+          FileUtils.mkpath(File.dirname(target))
+
+          if source.is_a? Array
+            assemble(source, target, format)
+          elsif File.directory?(source)
+            source_list = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
+            assemble(source_list, target, format)
+          else
+            image = MiniMagick::Image.open(source) {|b| b.quiet}
+            if image.pages.size > 1
+              assemble(image.pages.map {|page| page.path}, target, format)
+            else
+              assemble([source], target, format)
+            end
+          end
+
+          target
+
+        end
+
+        private
+
+        def assemble(sources, target, format)
+          MiniMagick::Tool::Convert.new do |b|
+            sources.each {|source| b << source}
+            convert.format(format)
+            b << target
+          end
+        end
+
+      end
+
+    end
+  end
+end