libis-format 1.3.4 → 2.0.0

data/lib/libis/format/converter/video_converter.rb

@@ -1,26 +1,29 @@
-# frozen_string_literal: true
-
 require_relative 'base'
-require 'libis/format/tool/ff_mpeg'
+require 'libis/format/tool/ffmpeg'
 
 require 'fileutils'
 
 module Libis
   module Format
     module Converter
+
       class VideoConverter < Libis::Format::Converter::Base
+
         def self.input_types
-          %i[WEBM MP4 MPG MKV MJP2 QTFF AVI OGGV WMV DV FLV SWF]
+          [:WEBM, :MP4, :MPG, :MKV, :MJP2, :QTFF, :AVI, :OGGV, :WMV, :DV, :FLV, :SWF]
         end
 
         def self.output_types(format = nil)
-          return [] unless input_types.include?(format)
+          return [] unless input_types.include?(format) if format
+          [:GIF, :WEBM, :MP4, :MPG, :MKV, :MJP2, :QTFF, :AVI, :OGGV, :WMV, :DV, :FLV, :SWF]
+        end
 
-          %i[GIF WEBM MP4 MPG MKV MJP2 QTFF AVI OGGV WMV DV FLV SWF]
+        def initialize
+          super
         end
 
-        def quiet(value)
-          @flags[:quiet] = !!value
+        def quiet(v)
+          @flags[:quiet] = !!v
         end
 
         def format(format)
@@ -123,10 +126,10 @@ module Libis
 
         # @param [Boolean] value If set to true automatically selects optimal format for web viewing. Default: false
         def web_stream(value)
-          return unless value
-
-          @options[:video_codec] = 'h264'
-          @options[:audio_codec] = 'acc'
+          if value
+            @options[:video_codec] = 'h264'
+            @options[:audio_codec] = 'acc'
+          end
         end
 
         # @param [String] name name of a preset. See FFMpeg documentation for more info
@@ -167,7 +170,7 @@ module Libis
 
           elsif File.directory?(source)
 
-            sources = Dir[File.join(source, '**', '*')].reject { |p| File.directory? p }
+            sources = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
             assemble_and_convert(sources, target)
 
           else
@@ -176,17 +179,15 @@ module Libis
 
           end
 
-          {
-            files: [target],
-            converter: self.class.name
-          }
+          target
+
         end
 
         def assemble_and_convert(sources, target)
-          Tempfile.create(%w[list .txt]) do |f|
-            sources.each { |src| f.puts src }
+          Tempfile.create(%w(list .txt)) do |f|
+            sources.each {|src| f.puts src}
             opts[:global] ||= []
-            opts[:global] += %w[-f concat]
+            opts[:global] += %w(-f concat)
             f.close
             target = convert_file(f.to_path, target)
           end
@@ -197,15 +198,15 @@ module Libis
 
         def convert_file(source, target)
           # FLV special: only supports aac and speex audio codecs
-          format = (@options[:format] || File.extname(target)[1..]).to_s.downcase
-          @options[:audio_codec] ||= 'aac' if %w[flv].include?(format)
+          format = (@options[:format] || File.extname(target)[1..-1]).to_s.downcase
+          @options[:audio_codec] ||= 'aac' if %w'flv'.include?(format)
 
           # SWF special: only supports mp3 audio codec
-          format = (@options[:format] || File.extname(target)[1..]).to_s.downcase
-          @options[:audio_codec] ||= 'mp3' if %w[swf].include?(format)
+          format = (@options[:format] || File.extname(target)[1..-1]).to_s.downcase
+          @options[:audio_codec] ||= 'mp3' if %w'swf'.include?(format)
 
           # Set up FFMpeg command line parameters
-          opts = { global: [], input: [], filter: [], output: [] }
+          opts = {global: [], input: [], filter: [], output: []}
           opts[:global] << '-hide_banner'
           opts[:global] << '-loglevel' << (@options[:quiet] ? 'fatal' : 'warning')
 
@@ -213,21 +214,29 @@ module Libis
           @options[:watermark_opacity] ||= 0.5
           if @options[:watermark_image]
             opts[:filter] << '-i' << @options[:watermark_image] << '-filter_complex'
-            opts[:filter] << Kernel.format('[1:v]format=argb,colorchannelmixer=aa=%f[wm];[0:v][wm]overlay=%s',
-                                           @options[:watermark_opacity], watermark_position_text)
+            opts[:filter] << "[1:v]format=argb,colorchannelmixer=aa=%f[wm];[0:v][wm]overlay=%s" %
+                [@options[:watermark_opacity], watermark_position_text]
           elsif @options[:watermark_text]
             @options[:watermark_text_size] ||= 10
             @options[:watermark_text_color] ||= 'white'
             @options[:watermark_text_shadow_color] ||= 'black'
             @options[:watermark_text_shadow_offset] ||= 1
-            filter_text = Kernel.format("drawtext=text='%s':%s:fontfile=%s:fontsize=%d:fontcolor=%s@%f",
-                                        @options[:watermark_text], watermark_position_text(true), Config[:watermark_font],
-                                        @options[:watermark_text_size], @options[:watermark_text_color], @options[:watermark_opacity])
-            if (@options[:watermark_text_shadow_offset]).positive?
-              filter_text += Kernel.format(':shadowcolor=%s@%f:shadowx=%d:shadowy=%d',
-                                           @options[:watermark_text_shadow_color], @options[:watermark_opacity],
-                                           @options[:watermark_text_shadow_offset], @options[:watermark_text_shadow_offset])
-            end
+            filter_text = "drawtext=text='%s':%s:fontfile=%s:fontsize=%d:fontcolor=%s@%f" %
+                [
+                    @options[:watermark_text],
+                    watermark_position_text(true),
+                    Config[:watermark_font],
+                    @options[:watermark_text_size],
+                    @options[:watermark_text_color],
+                    @options[:watermark_opacity]
+                ]
+            filter_text += ':shadowcolor=%s@%f:shadowx=%d:shadowy=%d' %
+                [
+                    @options[:watermark_text_shadow_color],
+                    @options[:watermark_opacity],
+                    @options[:watermark_text_shadow_offset],
+                    @options[:watermark_text_shadow_offset]
+                ] if @options[:watermark_text_shadow_offset] > 0
             opts[:filter] << '-vf' << filter_text
           end
           opts[:output] << '-ac' << @options[:audio_channels] if @options[:audio_channels]
@@ -239,7 +248,7 @@ module Libis
           opts[:output] << '-map_metadata:g' << '0:g' # Copy global metadata
           opts[:output] << '-map_metadata:s:a' << '0:s:a' # Copy audio metadata
           opts[:output] << '-map_metadata:s:v' << '0:s:v' # Copy video metadata
-          opts[:input] << '-accurate_seek' << (@options[:start].to_i.negative? ? '-sseof' : '-ss') << @options[:start] if @options[:start]
+          opts[:input] << '-accurate_seek' << (@options[:start].to_i < 0 ? '-sseof' : '-ss') << @options[:start] if @options[:start]
           opts[:input] << '-t' << @options[:duration] if @options[:duration]
           opts[:output] << '-qscale' << @options[:video_quality] if @options[:video_quality]
           opts[:output] << '-q:a' << @options[:audio_quality] if @options[:audio_quality]
@@ -265,19 +274,21 @@ module Libis
           w = for_text ? 'tw' : 'w'
           h = for_text ? 'th' : 'h'
           case @options[:watermark_position]
-          when 'bottom_left'
-            "x=#{margin}:y=H-#{h}-#{margin}"
-          when 'top_left'
-            "x=#{margin}:y=#{margin}"
-          when 'bottom_right'
-            "x=W-#{w}-#{margin}:y=H-#{h}-#{margin}"
-          when 'top_right'
-            "x=W-#{w}-#{margin}:y=#{margin}"
-          else
-            "x=#{margin}:y=H-#{h}-#{margin}"
+            when 'bottom_left'
+              "x=#{margin}:y=H-#{h}-#{margin}"
+            when 'top_left'
+              "x=#{margin}:y=#{margin}"
+            when 'bottom_right'
+              "x=W-#{w}-#{margin}:y=H-#{h}-#{margin}"
+            when 'top_right'
+              "x=W-#{w}-#{margin}:y=#{margin}"
+            else
+              "x=#{margin}:y=H-#{h}-#{margin}"
           end
         end
+
       end
+
     end
   end
-end
+end

data/lib/libis/format/converter/xslt_converter.rb

@@ -1,19 +1,18 @@
-# frozen_string_literal: true
-
 require_relative 'base'
 
 module Libis
   module Format
     module Converter
+
       class XsltConverter < Libis::Format::Converter::Base
+
         def self.input_types
           [:XML]
         end
 
         def self.output_types(format = nil)
-          return [] if format && !input_types.include?(format)
-
-          %i[XML HTML TXT]
+          return [] unless input_types.include?(format) if format
+          [:XML, :HTML, :TXT]
         end
 
         def xsl_file(file_path)
@@ -35,12 +34,12 @@ module Libis
 
           FileUtils.mkpath(File.dirname(target))
 
-          if RUBY_PLATFORM == 'java'
+          if RUBY_PLATFORM == "java"
             require 'saxon-xslt'
             xsl = Saxon.XSLT(File.open(@options[:xsl_file]))
             xml = Saxon.XML(File.open(source))
             result = xsl.transform(xml)
-            File.write(target, result.to_s)
+            File.open(target, 'w') {|f| f.write(result.to_s)}
           else
             require 'nokogiri'
 
@@ -87,17 +86,16 @@ module Libis
               error "Error transforming '#{source}' with '#{file}': #{e.message} @ #{e.backtrace[0]}"
               return nil
             ensure
-              fp.close unless fp.nil? || fp.closed?
+              fp.close unless fp.nil? or fp.closed?
             end
 
-            {
-              command: { status: 0 },
-              files: [target]
-            }
-
+            target
           end
+
         end
+
       end
+
     end
   end
 end

data/lib/libis/format/converter.rb

@@ -1,4 +1,4 @@
-# frozen_string_literal: true
+# code utf-8
 
 module Libis
   module Format

data/lib/libis/format/identifier.rb

@@ -1,4 +1,4 @@
-# frozen_string_literal: true
+# encoding: utf-8
 
 require 'singleton'
 require 'pathname'
@@ -7,8 +7,9 @@ require 'libis-tools'
 require 'libis/tools/extend/hash'
 require 'libis/tools/extend/string'
 require 'libis/tools/extend/empty'
+require 'nori/core_ext/object'
 
-require 'libis/format/type_database'
+require 'libis/format/library'
 
 require_relative 'config'
 require_relative 'tool/fido'
@@ -18,10 +19,13 @@ require_relative 'tool/extension_identification'
 
 module Libis
   module Format
+
     class Identifier
       include ::Libis::Tools::Logger
       include Singleton
 
+      public
+
       def self.add_xml_validation(mimetype, xsd_file)
         instance.xml_validations[mimetype] = xsd_file
       end
@@ -37,28 +41,29 @@ module Libis
       attr_reader :xml_validations
 
       def get(file, options = {})
-        options[:droid] = true unless options.keys.include?(:droid) || (options[:tool] && (options[:tool] != :droid))
-        options[:fido] = true unless options.keys.include?(:fido) || (options[:tool] && (options[:tool] != :fido))
-        options[:file] = true unless options.keys.include?(:file) || (options[:tool] && (options[:tool] != :file))
+
+        options[:droid] = true unless options.keys.include?(:droid) or (options[:tool] and options[:tool] != :droid)
+        options[:fido] = true unless options.keys.include?(:fido) or (options[:tool] and options[:tool] != :fido)
+        options[:file] = true unless options.keys.include?(:file) or (options[:tool] and options[:tool] != :file)
         options[:xml_validation] = true if options[:xml_validation].nil?
 
-        result = { messages: [], output: {}, formats: {} }
+        result = {messages: [], output: {}, formats: {}}
 
         begin
           get_droid_identification(file, result, options) if options[:droid]
-        rescue StandardError => e
+        rescue => e
           log_msg(result, :error, "Error running Droid: #{e.message} @ #{e.backtrace.first}")
         end
 
         begin
           get_fido_identification(file, result, options) if options[:fido]
-        rescue StandardError => e
+        rescue => e
           log_msg(result, :error, "Error running Fido: #{e.message} @ #{e.backtrace.first}")
         end
 
         begin
           get_file_identification(file, result, options) if options[:file]
-        rescue StandardError => e
+        rescue => e
           log_msg(result, :error, "Error running File: #{e.message} @ #{e.backtrace.first}")
         end
 
@@ -69,13 +74,14 @@ module Libis
         # Libis::Tools::Format::Identifier.add_xml_validation('my_type', '/path/to/my_type.xsd')
         begin
           validate_against_xml_schema(result, options[:base_dir]) if options[:xml_validation]
-        rescue StandardError => e
+        rescue => e
           log_msg(result, :error, "Error validating XML files: #{e.message} @ #{e.backtrace.first}")
         end
 
         process_results(result, !options[:keep_output])
 
         result
+
       end
 
       protected
@@ -91,7 +97,7 @@ module Libis
       end
 
       def get_fido_identification(file, result, options)
-        output = ::Libis::Format::Tool::Fido.run(file, options[:recursive], **(options[:fido_options] || {}))
+        output = ::Libis::Format::Tool::Fido.run(file, options[:recursive], options[:fido_options])
         process_tool_output(output, result, options[:base_dir])
         output
       end
@@ -117,21 +123,18 @@ module Libis
       end
 
       def xml_validate(file, file_result, result, base_dir)
-        return unless file_result[:mimetype] =~ %r{^(text|application)/xml$}
-
+        return unless file_result[:mimetype] =~ /^(text|application)\/xml$/
         filepath = base_dir ? File.join(base_dir, file) : file
         doc = ::Libis::Tools::XmlDocument.open filepath
         xml_validations.each do |mime, xsd_file|
           next unless xsd_file
-
           begin
             if doc.validates_against?(xsd_file)
               log_msg result, :debug, "XML file validated against XML Schema: #{xsd_file}"
-              info = { mimetype: mime, tool_raw: file_result[:tool], tool: :xsd_validation, match_type: 'xsd_validation',
-                       format_version: '' }
-              file_result.merge! Libis::Format::TypeDatabase.enrich(info, PUID: :puid, MIME: :mimetype, NAME: :format_name)
+              info = {mimetype: mime, tool_raw: file_result[:tool], tool: :xsd_validation, match_type: 'xsd_validation', format_version: '', }
+              file_result.merge! Libis::Format::Library.enrich(info, puid: :puid, mimetype: :mimetype, name: :format_name)
             end
-          rescue StandardError => e
+          rescue => e
             # Do nothing - probably Nokogiri chrashed during validation. Could have many causes
             # (remote schema (firewall, network, link rot, ...), schema syntax error, corrupt XML,...)
             # so we log and continue.
@@ -139,7 +142,7 @@ module Libis
                     "Error during XML validation of file #{file} against #{File.basename(xsd_file)}: #{e.message}")
           end
         end
-      rescue StandardError => e
+      rescue => e
         # Not much we can do. probably Nokogiri chrashed opening the XML file. What caused this?
         # (XML not parsable, false XML identification, ???)
         # so we log and continue.
@@ -147,30 +150,30 @@ module Libis
       end
 
       def process_results(result, delete_output = true)
-        result[:output].each_key do |file|
+        result[:output].keys.each do |file|
           output = result[:output][file]
           file_result = result[:formats][file] = {}
           if output.empty?
             log_msg(result, :warn, "Could not identify format of '#{file}'.")
-            file_result.merge!(
-              mimetype: 'application/octet-stream',
-              puid: 'fmt/unknown',
-              score: 0,
-              tool: nil
-            )
+            file_result = {
+                mimetype: 'application/octet-stream',
+                puid: 'fmt/unknown',
+                score: 0,
+                tool: nil
+            }
           else
-            format_matches = output.group_by { |x| [x[:mimetype], x[:puid]] }
+            format_matches = output.group_by {|x| [x[:mimetype], x[:puid]]}
             format_matches.each do |match, group|
-              format_matches[match] = group.group_by { |x| x[:score] }.sort.reverse.to_h
+              format_matches[match] = group.group_by {|x| x[:score]}.sort.reverse.to_h
             end
             case format_matches.count
-            when 0
-              # No this really cannot happen. If there are no hits, there will be at least a format [nil,nil]
-            when 1
-              # only one match, that's easy. The first of the highest score will be used
-              file_result.merge!(get_best_result(output))
-            else
-              process_multiple_formats(file_result, format_matches, output)
+              when 0
+                # No this really cannot happen. If there are no hits, there will be at least a format [nil,nil]
+              when 1
+                # only one match, that's easy. The first of the highest score will be used
+                file_result.merge!(get_best_result(output))
+              else
+                process_multiple_formats(file_result, format_matches, output)
             end
           end
         end
@@ -181,13 +184,12 @@ module Libis
         # multiple matches. Let's select the highest score matches
         file_result.merge!(get_best_result(output))
         file_result[:alternatives] = []
-        format_matches.each_key do |mime, puid|
+        format_matches.keys.each do |mime, puid|
           next if file_result[:mimetype] == mime && puid.nil?
-
-          selection = output.select { |x| x[:mimetype] == mime && x[:puid] == puid }
+          selection = output.select {|x| x[:mimetype] == mime && x[:puid] == puid}
           file_result[:alternatives] << get_best_result(selection)
         end
-        file_result[:alternatives] = file_result[:alternatives].sort_by { |x| x[:score] }.reverse
+        file_result[:alternatives] = file_result[:alternatives].sort_by {|x| x[:score]}.reverse
         file_result.delete(:alternatives) if file_result[:alternatives].size <= 1
       end
 
@@ -206,15 +208,15 @@ module Libis
       end
 
       def get_mimetype(puid)
-        ::Libis::Format::TypeDatabase.puid_typeinfo(puid)[:MIME].first
-      rescue StandardError
-        nil
+        ::Libis::Format::Library.get_field_by(:puid, puid, :mimetypes)
       end
 
       def get_best_result(results)
-        score = results.map { |x| x[:score] }.max
-        results.select { |x| x[:score] == score }.reduce(:apply_defaults)
+        score = results.map {|x| x[:score]}.max
+        results.select {|x| x[:score] == score}.reduce(:apply_defaults)
       end
+
     end
+
   end
 end

data/lib/libis/format/info.rb

@@ -0,0 +1,27 @@
+module Libis
+  module Format
+    class Info
+      attr_reader :name, :category, :description, :puids, :mimetypes, :extensions
+
+      def initialize(name:, category:, description: '', puids: [], mimetypes: [], extensions: [])
+        @name = name
+        @category = category
+        @description = description
+        @puids = puids
+        @mimetypes = mimetypes
+        @extensions = extensions
+      end
+
+      def to_hash
+        {
+            name: name,
+            description: description.dup,
+            category: category,
+            puids: puids.dup,
+            mimetypes: mimetypes.dup,
+            extensions: extensions.dup
+        }
+      end
+    end
+  end
+end

data/lib/libis/format/library.rb

@@ -0,0 +1,147 @@
+# froze_string_litteral: true
+# coding: utf-8
+require 'singleton'
+
+module Libis
+  module Format
+
+    class Library
+      include Singleton
+
+      class << self
+        def implementation=(impl)
+          instance.implementation = impl
+        end
+
+        def get_info(format)
+          instance.get_info(format)
+        end
+
+        def get_info_by(key, value)
+          instance.get_info_by(key, value)
+        end
+
+        def get_infos_by(key, value)
+          instance.get_infos_by(key, value)
+        end
+
+        def get_field(format, field)
+          instance.get_field(format, field)
+        end
+
+        def get_field_by(key, value, field)
+          instance.get_field_by(key, value, field)
+        end
+
+        def get_fields_by(key, value, field)
+          instance.get_fields_by(key, value, field)
+        end
+
+        def known?(key, value)
+          instance.known?(key, value)
+        end
+
+        def enrich(info, map_keys = {})
+          instance.enrich(info, map_keys)
+        end
+
+        def normalize(info, map_keys = {})
+          instance.normalize(info, map_keys)
+        end
+      end
+
+      def implementation=(impl)
+        @implementation = impl
+      end
+
+      def get_field(format, field)
+        get_field_by(:name, format, field)
+      end
+
+      def get_field_by(key, value, field)
+        info = get_info_by(key, value)
+        return nil unless info
+        case field
+        when :mimetype
+          info[:mimetypes]&.first
+        when :puid
+          info[:puids]&.first
+        when :extension
+          info[:extensions]&.first
+        else
+          info[field]
+        end
+      end
+
+      def get_fields_by(key, value, field)
+        get_infos_by(key, value)&.map { |info| info[field] }.compact
+      end
+
+      def get_info(format)
+        get_info_by(:name, format)
+      end
+
+      def get_info_by(key, value)
+        get_infos_by(key, value)&.first
+      end
+
+      def get_infos_by(key, value)
+        result = @implementation.query(key, value)
+        result.map(&:to_hash)
+      end
+
+      def known?(key, value)
+        !get_info_by(key, value).nil?
+      end
+
+      def enrich(info, map_keys = {})
+        info = normalize(info, map_keys)
+        mapper = Hash.new { |hash, key| hash[key] = key }.merge(map_keys)
+        unless (format = info[mapper[:name]]).nil?
+          lib_info = get_info(format)
+          mapper.keys.each do |key|
+            case key
+            when :mimetype
+              info[mapper[key]] = lib_info[:mimetypes].first if lib_info[:mimetypes].first
+            when :puid
+              info[mapper[key]] = lib_info[:puids].first if lib_info[:puids].first
+            when :extension
+              info[mapper[key]] = lib_info[:extensions].first if lib_info[:extensions].first
+            else
+              info[mapper[key]] = lib_info[key] if lib_info[key]
+            end
+          end
+        end
+        info
+      end
+
+      # Derive name from the available info
+      def normalize(info, map_keys = {})
+        return {} unless info.is_a? Hash
+        mapper = Hash.new { |hash, key| hash[key] = key }.merge(map_keys)
+        # fill format from looking up by puid
+        unless (puid = info[mapper[:puid]]).blank?
+          info[mapper[:name]] ||= get_field_by(:puid, puid, :name)
+        end
+        # fill format from looking up by mimetype
+        unless (mime = info[mapper[:mimetype]]).blank?
+          info[mapper[:name]] ||= get_field_by(:mimetype, mime, :name)
+        end
+        # finally complete the information from looking up by format name
+        unless (format = info[mapper[:name]]).nil?
+          info[mapper[:mimetype]] = get_field(format, :mimetype)
+          info[mapper[:category]] = get_field(format, :category)
+        end
+        info
+      end
+
+      private
+
+      def initialize
+        @implementation = eval(Libis::Format::Config[:format_library_implementation])
+      end
+
+    end
+
+  end
+end