libis-format 0.9.38 → 0.9.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aee2f761cde56c377abdfff7b7d40d09c32c0c93
-  data.tar.gz: 5d6384f6484d04ec01705883e04ead5b0d02dacb
+  metadata.gz: d41852f4f00f61ffa6458f3ba0f9ce58215573d8
+  data.tar.gz: 96158848af69ff1ad5ffafc198d6f04b30a94f9c
 SHA512:
-  metadata.gz: 5a359cd06cc1f3f442b69981858e44e55f4cd085e0e0291e4b03c0a530e86d5f5e72d1083cc7b2fa306fdd4dba033ce4eb985c49373fe0d4467052f1406146a3
-  data.tar.gz: 619bdb26cfea98753544d47fd1096150cd2f7ec56e5322bffabf6a4c598797f31be68d22b3f7f267aebcc5f743ae6d60bcd498a1eb27af183a5b0a5cbae7c0a9
+  metadata.gz: 7db9b6c77b4429fa91ec099d8a10c55799050e6ca345b1af0f2273c5645841b2edb33b5be9cfe2967eedf52d0438e6bbe92e0ad499f2be3460ec8e53d7e076b1
+  data.tar.gz: 6399b4b84f80b7e48c2c85c3390e34fe39b70c8ea1c4a096dfd33981bbcc2eca2ba88f8be9e4738ff424f0e12446f5b898b1db248206ebd080fff098c585af6b

data/.travis.yml

@@ -1,5 +1,6 @@
 language: ruby
 sudo: false
+dist: trusty
 cache: bundler
 rvm:
   - 2.1.0
@@ -7,31 +8,25 @@ rvm:
   - ruby-head
   - jruby-9.0.1.0
 jdk:
-  - openjdk7
-  - oraclejdk7
+  - openjdk8
   - oraclejdk8
 matrix:
   exclude:
-    - rvm: 2.1.0
-      jdk: oraclejdk7
     - rvm: 2.1.0
       jdk: oraclejdk8
-    - rvm: 2.2.0
-      jdk: oraclejdk7
     - rvm: 2.2.0
       jdk: oraclejdk8
-    - rvm: ruby-head
-      jdk: oraclejdk7
     - rvm: ruby-head
       jdk: oraclejdk8
-  allow_failures:
-    - rvm: 2.1.0
-    - rvm: 2.2.0
-    - rvm: ruby-head
-    - rvm: jruby-9.0.1.0
 branches:
   only:
       - master
+before_install:
+  - sudo add-apt-repository ppa:mc3man/trusty-media -y
+  - sudo apt-get update -y
+  - sudo apt-get install -y ffmpeg gstreamer0.10-ffmpeg
+  - sudo pip install opf-fido --upgrade --ignore-installed six
+
 addons:
   apt:
     packages:
@@ -39,3 +34,8 @@ addons:
       - imagemagick
       - libreoffice
       - ghostscript
+      - sox
+      - libsox-dev
+      - libsox-fmt-all
+      - libchromaprint-dev
+      - libchromaprint-tools

data/data/types.yml

@@ -70,66 +70,142 @@ IMAGE:
 
 AUDIO:
 
-  WAV:
-    NAME: Waveform Audio File Format (WAVE)
-    MIME: audio/x-wav
-    EXTENSIONS: wav
-
   MP3:
     NAME: MPEG-1 or MPEG-2 Audio Layer III (MP3)
     MIME: audio/mpeg
+    PUID: fmt/134
     EXTENSIONS: mp3
 
   FLAC:
     NAME: Free Lossless Audio Codec (FLAC)
     MIME: audio/flac
+    PUID: fmt/279
     EXTENSIONS: flac
 
+  AC3:
+    NAME: Dolby Digital AC-3
+    MIME: audio/ac3
+    PUID: fmt/735
+    EXTENSIONS: ac3
+
+  AAC:
+    NAME: Advanced Audio Coding (AAC)
+    MIME: audio/aac
+    EXTENSIONS: aac
+
+  WMA:
+    NAME: Windows Media Audio (WMA)
+    MIME: audio/x-ms-wma
+    PUID: fmt/132
+    EXTENSIONS: wma
+
+  ALAC:
+    NAME: Apple Lossless Audio Codec (ALAC)
+    MIME: audio/alac
+    EXTENSIONS: alac
+
+  WAV:
+    NAME: Waveform Audio File Format (WAVE)
+    MIME: audio/x-wav
+    PUID: fmt/6, fmt/141, fmt/142, fmt/143, fmt/1, fmt/703, fmt/706, fmt/709, fmt/2, fmt/704, fmt/707, fmt/710, fmt/527, fmt/705, fmt/708, fmt/711, x-fmt/397, x-fmt/389, x-fmt/396, fmt/712, fmt/713
+    EXTENSIONS: wav
+
+  AIFF:
+    NAME: Audio Interchange File Format (AIFF)
+    MIME: audio/aiff
+    PUID: fmt/414, x-fmt/135, x-fmt/136
+    EXTENSIONS: aif aiff aifc
+
+  AMR:
+    NAME: Adaptive Multi-Rate Audio (AMR)
+    MIME: audio/amr
+    PUID: fmt/954
+    EXTENSIONS: amr
+
+  AU:
+    NAME: NeXT/Sun sound (AU)
+    MIME: audio/basic
+    PUID: x-fmt/139
+    EXTENSIONS: au
+
+  M4A:
+    NAME: MPEG-4 Media File (M4A)
+    MIME: audio/mp4
+    PUID: fmt/199, fmt/596
+    EXTENSIONS: m4a f4a
+
 VIDEO:
 
-  MPEG:
+  MPG:
     NAME: Moving Picture Experts Group (MPEG-1/MPEG-2)
     MIME: video/mpeg
+    PUID: x-fmt/385 x-fmt/386 fmt/425 fmt/649 fmt/640
     EXTENSIONS: mpg mpeg mp1 mp2 mp3 m1v m1a m2a mpa mpv
 
-  MPEG4:
+  MP4:
     NAME: Moving Picture Experts Group (MPEG-4)
-    MIME: video/mp4
-    EXTENSIONS: mp4,mpeg4
+    MIME: video/mp4, application/mp4
+    PUID: fmt/199
+    EXTENSIONS: mp4,m4v,mpeg4
+
+  WEBM:
+    NAME: Webm
+    PUID:  fmt/573
+    MIME:  video/webm
+    EXTENSIONS: webm
+
+  FLASH:
+    NAME: Macromedia Flash
+    MIME: application/x-shockwave-flash
+    PUID: fmt/104 fmt/105 fmt/106 fmt/107 fmt/108 fmt/109 fmt/110
+    EXTENSIONS: swf
+
+  MKV:
+    NAME: Matroska Multimedia Container (MKV)
+    MIME: video/x-matroska
+    PUID: fmt/569
+    EXTENSIONS: mkv, mk3d, mka, mks
 
   MJP2:
     NAME: Motion JPEG 2000 (MJP2)
     MIME: video/mj2
+    PUID: fmt/337
     EXTENSIONS: mj2, mjp2
 
   QTFF:
     NAME: QuickTime File Format (QTFF)
     MIME: video/quicktime
+    PUID: x-fmt/384
     EXTENSIONS: mov, qt
 
   AVI:
     NAME: Audio Video Interleave (AVI)
     MIME: video/x-msvideo
+    PUID: fmt/5
     EXTENSIONS: avi
 
   OGGV:
     NAME: OGG Video (OGGV)
     MIME: video/ogg
+    PUID:  fmt/945
     EXTENSIONS: ogv
 
   WMV:
     NAME: Windows Media Video (WMV)
     MIME: video/x-ms-wmv
+    PUID: fmt/133 fmt/441
     EXTENSIONS: wmv
 
   DV:
     NAME: Digital Video (DV)
     MIME: video/dv
+    PUID: x-fmt/152
     EXTENSIONS: dv
 
   FLASH:
     NAME: Flash video (FLV)
     MIME: video/x-flv
+    PUID: x-fmt/382
     EXTENSIONS: flv
 
 TEXT:

data/lib/libis/format.rb

@@ -19,6 +19,7 @@ module Libis
     autoload :PdfSplit, 'libis/format/pdf_split'
     autoload :PdfToPdfa, 'libis/format/pdf_to_pdfa'
     autoload :PdfaValidator, 'libis/format/pdfa_validator'
+    autoload :FFMpeg, 'libis/format/ffmpeg'
 
     autoload :Converter, 'libis/format/converter'
 

data/lib/libis/format/config.rb

@@ -15,11 +15,13 @@ module Libis
     Config[:ghostscript_path] = 'gs'
     Config[:droid_path] = '/opt/droid/droid.sh'
     Config[:fido_path] = '/usr/local/bin/fido'
+    Config[:ffmpeg_path] = 'ffmpeg'
     data_dir = File.absolute_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'data'))
     Config[:fido_formats] = [(File.join(data_dir, 'lias_formats.xml'))]
     # noinspection RubyStringKeysInHashInspection
     Config[:xml_validations] = [['archive/ead', File.join(data_dir, 'ead.xsd')]]
     Config[:type_database] = File.join(data_dir, 'types.yml')
+    Config[:raw_audio_convert_cmd] = 'sox %s -e signed -b 16 -t wav %s rate %d channels %d'
 
   end
 end

data/lib/libis/format/converter/audio_converter.rb

@@ -0,0 +1,181 @@
+require_relative 'base'
+require 'libis/format/identifier'
+require 'chromaprint'
+
+require 'fileutils'
+
+module Libis
+  module Format
+    module Converter
+
+      class AudioConverter < Libis::Format::Converter::Base
+
+        def self.input_types
+          [:MP3, :FLAC, :AC3, :AAC, :WMA, :ALAC, :WAV, :AIFF, :AMR, :AU, :M4A]
+        end
+
+        def self.output_types(format = nil)
+          return [] unless input_types.include?(format)
+          [:MP3, :FLAC, :AC3, :AAC, :WMA, :ALAC, :WAV, :AIFF, :AMR, :AU, :M4A]
+        end
+
+        def initialize
+          super
+        end
+
+        def quiet(v)
+          @flags[:quiet] = !!v
+        end
+
+        def format(format)
+          @options[:format] = format
+        end
+
+        def codec(codec)
+          @options[:codec] = codec
+        end
+
+        def start(seconds)
+          @options[:start] = seconds.to_s
+        end
+
+        def duration(seconds)
+          @options[:duration] = seconds.to_s
+        end
+
+        def quality(value)
+          @options[:quality] = value.to_s
+        end
+
+        def bit_rate(value)
+          @options[:bit_rate] = value.to_s
+        end
+
+        def sampling_freq(value)
+          @options[:sampling_freq] = value.to_s
+        end
+
+        def channels(value)
+          @options[:channels] = value.to_s
+        end
+
+        def web_stream(value)
+          if value
+            @options[:codec] = 'mp3'
+          end
+        end
+
+        def preset(stream, name)
+          (@options[:preset] ||= {})[stream] = name
+        end
+
+        # def encoder(value)
+        #   @options[:encoder] = value
+        # end
+        #
+        # def encoder_options(value)
+        #   @options[:encoder_options] = value
+        # end
+
+        def convert(source, target, _format, opts = {})
+          super
+
+          FileUtils.mkpath(File.dirname(target))
+
+          if source.is_a? Array
+
+            assemble_and_convert(source, target)
+
+          elsif File.directory?(source)
+
+            sources = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
+            assemble_and_convert(sources, target)
+
+          else
+
+            convert_file(source, target)
+
+          end
+
+          target
+
+        end
+
+        def assemble_and_convert(sources, target)
+          Tempfile.create(%w(list .txt)) do |f|
+            sources.each {|src| f.puts src}
+            opts[:global] ||= []
+            opts[:global] += %w(-f concat)
+            f.close
+            target = convert_file(f.to_path, target)
+          end
+          target
+        end
+
+        def sounds_like(file1, file2, threshold, rate, channels)
+          rate ||= 96000
+          channels ||= 2
+          threshold ||= 0.85
+
+          if File.exists?(file1) && File.exists?(file2)
+            # Convert input files into raw 16-bit signed audio (WAV) to process with Chramaprint
+            file1_raw = File.join('', 'tmp', File.basename(file1) + '.wav')
+            file2_raw = File.join('', 'tmp', File.basename(file2) + '.wav')
+            FileUtils.rm(file1_raw, force: true)
+            FileUtils.rm(file2_raw, force: true)
+            cvt_cmd = Libis::Format::Config[:raw_audio_convert_cmd]
+            %x"#{cvt_cmd % [file1, file1_raw, rate, channels]}"
+            %x"#{cvt_cmd % [file2, file2_raw, rate, channels]}"
+            file1_audio = File.binread(file1_raw)
+            file2_audio = File.binread(file2_raw)
+
+            # Get audio fingerprints
+            chromaprint = Chromaprint::Context.new(rate, channels, Chromaprint::ALGORITHM_TEST3)
+            file1_fp = chromaprint.get_fingerprint(file1_audio)
+            file2_fp = chromaprint.get_fingerprint(file2_audio)
+
+            # Cleanup files
+            FileUtils.rm(file1_raw, force: true)
+            FileUtils.rm(file2_raw, force: true)
+
+            # Compare fingerprints and compare result against threshold
+            cmp = file1_fp.compare(file2_fp)
+            # puts "Threshold[#{File.basename(exp_file)},#{File.basename(tgt_file)}: #{cmp}"
+            cmp > threshold
+          else
+            false
+          end
+
+        rescue Exception => e
+          error "Error comparing sound file #{file1} and #{file2}: #{e.message} @ #{e.backtrace.first}"
+          false
+        end
+
+        protected
+
+        def convert_file(source, target)
+          opts = {global: [], input: [], filter: [], output: []}
+          opts[:global] << '-hide_banner'
+          opts[:global] << '-loglevel' << (@options[:quiet] ? 'fatal' : 'warning')
+          opts[:output] << '-vn' # disable input video stream in case it exists
+          opts[:output] << '-codec:a' << @options[:codec] if @options[:codec]
+          opts[:output] << '-map_metadata:g' << '0:g' # Copy global metadata
+          opts[:output] << '-map_metadata:s:a' << '0:s:a' # Copy audio metadata
+          opts[:input] << '-accurate_seek' << (@options[:start].to_i < 0 ? '-sseof' : '-ss') << @options[:start] if @options[:start]
+          opts[:input] << '-t' << @options[:duration] if @options[:duration]
+          opts[:output] << '-q:a' << @options[:quality] if @options[:quality]
+          opts[:output] << '-b:a' << @options[:bit_rate] if @options[:bit_rate]
+          opts[:output] << '-ar' << @options[:sampling_freq] if @options[:sampling_freq]
+          opts[:output] << '-ac' << @options[:channels] if @options[:channels]
+          opts[:output] << '-f' << @options[:format] if @options[:format]
+          result = Libis::Format::FFMpeg.run(source, target, opts)
+          info "FFMpeg output: #{result}"
+          result
+          target
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/format/converter/video_converter.rb

@@ -0,0 +1,194 @@
+require_relative 'base'
+require 'libis/format/ffmpeg'
+
+require 'fileutils'
+
+module Libis
+  module Format
+    module Converter
+
+      class VideoConverter < Libis::Format::Converter::Base
+
+        def self.input_types
+          [:WEBM, :M4V, :MPEG, :MJP2, :QTFF, :AVI, :OGGV, :WMV, :DV, :FLASH]
+        end
+
+        def self.output_types(format = nil)
+          return [] unless input_types.include?(format)
+          [:GIF, :AVI, :FLV, :MKV, :MOV, :MP4, :MPG, :OGGV, :SWF, :WEBM, :WMV]
+        end
+
+        def initialize
+          super
+        end
+
+        def quiet(v)
+          @flags[:quiet] = !!v
+        end
+
+        def format(format)
+          @options[:format] = format
+        end
+
+        def audio_channels(value)
+          @options[:audio_channels] = value.to_s
+        end
+
+        def audio_codec(codec)
+          @options[:audio_codec] = codec
+        end
+
+        def video_codec(codec)
+          @options[:video_codec] = codec
+        end
+
+        def start(seconds)
+          @options[:start] = seconds.to_s
+        end
+
+        def duration(seconds)
+          @options[:duration] = seconds.to_s
+        end
+
+        def audio_quality(value)
+          @options[:audio_quality] = value.to_s
+        end
+
+        def video_quality(value)
+          @options[:video_quality] = value.to_s
+        end
+
+        def audio_bitrate(value)
+          @options[:audio_bitrate] = value.to_s
+        end
+
+        def video_bitrate(value)
+          @options[:video_bitrate] = value.to_s
+        end
+
+        def constant_rate_factor(value)
+          @options[:crf] = value.to_s
+        end
+
+        def frame_rate(value)
+          @options[:frame_rate] = value.to_s
+        end
+
+        def sampling_freq(value)
+          @options[:sampling_freq] = value.to_s
+        end
+
+        def scale(width_x_height)
+          @options[:scale] = width_x_height
+        end
+
+        def web_stream(value)
+          if value
+            @options[:video_codec] = 'h264'
+            @options[:audio_codec] = 'acc'
+          end
+        end
+
+        def preset(name)
+          @options[:preset] = name
+        end
+
+        def audio_preset(name)
+          @options[:audio_preset] = name
+        end
+
+        def video_preset(name)
+          @options[:video_preset] = name
+        end
+
+        # def video_encoder(value)
+        #   @options[:video_encoder] = value
+        # end
+        #
+        # def audio_encoder(value)
+        #   @options[:audio_encoder] = value
+        # end
+        #
+        # def encoder_options(value)
+        #   @options[:encoder_options] = value
+        # end
+
+        def convert(source, target, _format, opts = {})
+          super
+
+          FileUtils.mkpath(File.dirname(target))
+
+          if source.is_a? Array
+
+            assemble_and_convert(source, target)
+
+          elsif File.directory?(source)
+
+            sources = Dir[File.join(source, '**', '*')].reject {|p| File.directory? p}
+            assemble_and_convert(sources, target)
+
+          else
+
+            convert_file(source, target)
+
+          end
+
+          target
+
+        end
+
+        def assemble_and_convert(sources, target)
+          Tempfile.create(%w(list .txt)) do |f|
+            sources.each {|src| f.puts src}
+            opts[:global] ||= []
+            opts[:global] += %w(-f concat)
+            f.close
+            target = convert_file(f.to_path, target)
+          end
+          target
+        end
+
+        protected
+
+        def convert_file(source, target)
+          # FLV special: only supports aac and speex audio codecs
+          format = (@options[:format] || File.extname(target)[1..-1]).to_s.downcase
+          @options[:audio_codec] ||= 'aac' if %w'flv'.include?(format)
+
+          # FLV special: only supports aac and speex audio codecs
+          format = (@options[:format] || File.extname(target)[1..-1]).to_s.downcase
+          @options[:audio_codec] ||= 'mp3' if %w'swf'.include?(format)
+
+          # Set up FFMpeg command line parameters
+          opts = {global: [], input: [], filter: [], output: []}
+          opts[:global] << '-hide_banner'
+          opts[:global] << '-loglevel' << (@options[:quiet] ? 'fatal' : 'warning')
+          opts[:output] << '-ac' << @options[:audio_channels] if @options[:audio_channels]
+          opts[:output] << '-c:a' << @options[:audio_codec] if @options[:audio_codec]
+          opts[:output] << '-c:v' << @options[:video_codec] if @options[:video_codec]
+          opts[:output] << '-b:a' << @options[:audio_bitrate] if @options[:audio_bitrate]
+          opts[:output] << '-b:v' << @options[:video_bitrate] if @options[:video_bitrate]
+          opts[:output] << '-crf' << @options[:crf] if @options[:crf]
+          opts[:output] << '-map_metadata:g' << '0:g' # Copy global metadata
+          opts[:output] << '-map_metadata:s:a' << '0:s:a' # Copy audio metadata
+          opts[:output] << '-map_metadata:s:v' << '0:s:v' # Copy video metadata
+          opts[:input] << '-accurate_seek' << (@options[:start].to_i < 0 ? '-sseof' : '-ss') << @options[:start] if @options[:start]
+          opts[:input] << '-t' << @options[:duration] if @options[:duration]
+          opts[:output] << '-qscale' << @options[:video_quality] if @options[:video_quality]
+          opts[:output] << '-q:a' << @options[:audio_quality] if @options[:audio_quality]
+          opts[:output] << '-r' << @options[:frame_rate] if @options[:frame_rate]
+          opts[:output] << '-ar' << @options[:sampling_freq] if @options[:sampling_freq]
+          opts[:output] << '-f' << @options[:format] if @options[:format]
+          opts[:output] << '-pre' << @options[:preset] if @options[:preset]
+          opts[:output] << '-apre' << @options[:audio_preset] if @options[:audio_preset]
+          opts[:output] << '-vpre' << @options[:video_preset] if @options[:video_preset]
+          result = Libis::Format::FFMpeg.run(source, target, opts)
+          info "FFMpeg output: #{result}"
+          target
+        end
+
+      end
+
+    end
+  end
+end