video_converter 0.2.13 → 0.3.0

data/lib/video_converter.rb

@@ -8,15 +8,16 @@ require "video_converter/input"
 require "video_converter/input_array"
 require "video_converter/output"
 require "video_converter/output_array"
+require "video_converter/hls"
 require "fileutils"
 require "net/http"
 require "video_screenshoter"
+require "shellwords"
 
 module VideoConverter
   class << self
     attr_accessor :paral
   end
-  
   self.paral = true
 
   def self.new params
@@ -24,7 +25,7 @@ module VideoConverter
   end
 
   def self.find uid
-    VideoConverter::Process.new uid
+    VideoConverter::Process.find uid
   end
 end
 

data/lib/video_converter/base.rb

@@ -2,7 +2,7 @@
 
 module VideoConverter
   class Base
-    attr_accessor :input_array, :output_array, :log, :uid, :clear_tmp
+    attr_accessor :input_array, :output_array, :log, :uid, :clear_tmp, :process
 
     def initialize params
       self.uid = params[:uid] || (Socket.gethostname + object_id.to_s)
@@ -19,24 +19,13 @@ module VideoConverter
     end
 
     def run
-      process = VideoConverter::Process.new(uid)
-      process.status = 'started'
-      process.pid = `cat /proc/self/stat`.split[3]
-      actions = []
-      actions = [:convert, :segment, :screenshot]
-      actions << :clear if clear_tmp
-      actions.each do |action|
+      self.process = VideoConverter::Process.new(uid, output_array)
+      [:convert, :segment, :screenshot].each do |action|
         process.status = action.to_s
-        process.progress = 0
-        res = send action
-        if res
-          process.status = "#{action}_success"
-        else
-          process.status = "#{action}_error"
-          return false
-        end
+        process.status = "#{action}_error" and return false unless send(action)
       end
       process.status = 'finished'
+      clear if clear_tmp
       true
     end
 
@@ -44,7 +33,7 @@ module VideoConverter
 
     def convert
       params = {}
-      [:input_array, :output_array, :log].each do |param|
+      [:input_array, :output_array, :log, :process].each do |param|
         params[param] = self.send(param)
       end
       Ffmpeg.new(params).run
@@ -73,6 +62,7 @@ module VideoConverter
         FileUtils.rm(Dir.glob(File.join(output.work_dir, '*.log.mbtree')))
         FileUtils.rm(File.join(output.work_dir, output.filename.sub(/\.m3u8$/, '.ts'))) if output.type == :segmented
       end
+      FileUtils.rm_r(process.process_dir)
       true
     end
   end

data/lib/video_converter/command.rb

@@ -13,12 +13,23 @@ module VideoConverter
     def initialize command, params = {}
       res = command.clone
       params.each do |param, value|
-        res.gsub! "%{#{param}}", value.to_s
+        value = value.to_s.strip
+        res.gsub! "%{#{param}}", if value.empty?
+          ''
+        elsif matches = value.match(/^([\w-]+)\s+(.+)$/)
+          "#{matches[1]} #{escape(matches[2])}"
+        else
+          escape(value)
+        end
       end
       self.command = res
       raise ArgumentError.new("Command is not parsed '#{self.command}'") if self.command.match(/%{[\w\-.]+}/)
     end
 
+    def escape value
+      Shellwords.escape(value.to_s.gsub(/(?<!\\)(['+])/, '\\\\\1')).gsub("\\\\","\\")
+    end
+
     def execute params = {}
       puts command if params[:verbose] || self.class.verbose
       if params[:debug] || self.class.debug

data/lib/video_converter/ffmpeg.rb

@@ -11,39 +11,42 @@ module VideoConverter
     self.paral = true
     self.log = '/dev/null'
     
-    self.one_pass_command = "%{bin} -i %{input} -y -acodec copy -vcodec %{video_codec} -g 100 -keyint_min 50 -b:v %{video_bitrate}k -bt %{video_bitrate}k -f mp4 %{local_path} 1>%{log} 2>&1 || exit 1"
+    self.one_pass_command = "%{bin} -i %{input} -y -acodec copy -vcodec %{video_codec} -g 100 -keyint_min 50 -b:v %{video_bitrate}k -bt %{video_bitrate}k %{vf} %{frame_rate} -progress %{progressfile} -f mp4 %{local_path} 1>%{log} 2>&1 || exit 1"
 
-    self.first_pass_command = "%{bin} -i %{input} -y -an -vcodec %{video_codec} -g %{keyframe_interval} -keyint_min 25 -pass 1 -passlogfile %{passlogfile} -b:v 700k -threads %{threads} -f mp4 /dev/null 1>>%{log} 2>&1 || exit 1"
+    self.first_pass_command = "%{bin} -i %{input} -y -an -vcodec %{video_codec} -g %{keyframe_interval} -keyint_min 25 -pass 1 -passlogfile %{passlogfile} -progress %{progressfile} -b:v 3000k %{vf} %{frame_rate} -threads %{threads} -f mp4 /dev/null 1>>%{log} 2>&1 || exit 1"
 
-    self.second_pass_command = "%{bin} -i %{input} -y -pass 2 -passlogfile %{passlogfile} -c:a %{audio_codec} -b:a %{audio_bitrate}k -c:v %{video_codec} -g %{keyframe_interval} -keyint_min 25 %{frame_rate} -b:v %{video_bitrate}k %{size} -threads %{threads} -f mp4 %{local_path} 1>%{log} 2>&1 || exit 1"
+    self.second_pass_command = "%{bin} -i %{input} -y -pass 2 -passlogfile %{passlogfile} -progress %{progressfile} -c:a %{audio_codec} -b:a %{audio_bitrate}k -c:v %{video_codec} -g %{keyframe_interval} -keyint_min 25 %{frame_rate} -b:v %{video_bitrate}k %{vf} -threads %{threads} -f mp4 %{local_path} 1>%{log} 2>&1 || exit 1"
 
-    attr_accessor :input_array, :output_array, :one_pass, :paral, :log
+    attr_accessor :input_array, :output_array, :one_pass, :paral, :log, :process
 
     def initialize params
       [:input_array, :output_array].each do |param|
         self.send("#{param}=", params[param]) or raise ArgumentError.new("#{param} is needed")
       end
-      [:one_pass, :paral, :log].each do |param|
+      [:one_pass, :paral, :log, :process].each do |param|
         self.send("#{param}=", params[param] ? params[param] : self.class.send(param))
       end
     end
 
     def run
+      progress_thread = Thread.new { collect_progress }
       res = true
       input_array.inputs.each do |input|
         threads = []
         input.output_groups.each_with_index do |group, group_number|
-          passlogfile = File.join(File.dirname(group.first.local_path), "#{group_number}.log")
+          passlogfile = File.join(group.first.work_dir, "#{group_number}.log")
+          progressfile = File.join(process.progress_dir, "#{group_number}.log")
           one_pass = self.one_pass || group.first.video_codec == 'copy'
           unless one_pass
-            first_pass_command = Command.new self.class.first_pass_command, prepare_params(common_params.merge(group.first.to_hash).merge((group.first.playlist.to_hash rescue {})).merge(:passlogfile => passlogfile, :input => input))
+            first_pass_command = Command.new self.class.first_pass_command, prepare_params(common_params.merge((group.first.playlist.to_hash rescue {})).merge(group.first.to_hash).merge(:passlogfile => passlogfile, :input => input, :progressfile => progressfile))
             res &&= first_pass_command.execute
           end
-          group.each do |quality|
+          group.each_with_index do |quality, quality_number|
+            progressfile = File.join(process.progress_dir, "#{group_number}_#{quality_number}.log")
             if one_pass
-              quality_command = Command.new self.class.one_pass_command, prepare_params(common_params.merge(quality.to_hash).merge(:passlogfile => passlogfile, :input => input))
+              quality_command = Command.new self.class.one_pass_command, prepare_params(common_params.merge(quality.to_hash).merge(:passlogfile => passlogfile, :input => input, :progressfile => progressfile))
             else
-              quality_command = Command.new self.class.second_pass_command, prepare_params(common_params.merge(quality.to_hash).merge(:passlogfile => passlogfile, :input => input))
+              quality_command = Command.new self.class.second_pass_command, prepare_params(common_params.merge(quality.to_hash).merge(:passlogfile => passlogfile, :input => input, :progressfile => progressfile))
             end
             if paral
               threads << Thread.new { res &&= quality_command.execute }
@@ -54,9 +57,10 @@ module VideoConverter
         end
         threads.each { |t| t.join } if paral
       end
+      progress_thread.kill
       res
     end
-
+    
     private 
 
     def common_params
@@ -64,9 +68,34 @@ module VideoConverter
     end
 
     def prepare_params params
-      params[:size] = params[:size] ? "-s #{params[:size]}" : ''
+      width = params[:width] || params[:size].to_s.match(/^(\d+)x(\d*)$/).to_a[1] || 'trunc(oh*a/2)*2'
+      height = params[:height] || params[:size].to_s.match(/^(\d*)x(\d+)$/).to_a[2] || 'trunc(ow/a/2)*2'
+      if width.to_s.include?('trunc') && height.to_s.include?('trunc')
+        params[:vf] = ''
+      else
+        params[:vf] = "-vf scale=#{width}:#{height}"
+      end
       params[:frame_rate] = params[:frame_rate] ? "-r #{params[:frame_rate]}" : ''
       params
     end
+
+    def processes_count
+      (one_pass ? 0 : input_array.inputs.inject(0) { |sum, i| sum + i.output_groups.count }) +
+      input_array.inputs.map { |i| i.output_groups.map { |g| g.count } }.flatten.inject(:+)
+    end
+
+    def collect_progress
+      duration = input_array.inputs.first.metadata[:duration_in_ms]
+      loop do
+        sleep Process.collect_progress_interval
+        koefs = []
+        Dir.glob(File.join(process.progress_dir, '*.log')).each do |progressfile|
+          if matches = `tail -n 9 #{progressfile}`.match(/out_time_ms=(\d+).*?progress=(\w+)/m)
+            koefs << (matches[2] == 'end' ? 1 : (matches[1].to_f / 1000 / duration))
+          end
+        end
+        process.collect_progress(koefs.inject { |sum, k| sum + k } / processes_count)
+      end
+    end
   end
 end

data/lib/video_converter/hls.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+
+module VideoConverter
+  class Hls
+    attr_accessor :ffmpeg_bin, :input, :output, :chunks_dir, :replace_in_chunk, :log, :verbose
+
+    def initialize params
+      self.ffmpeg_bin = params[:ffmpeg_bin] || Ffmpeg.bin
+      %w(input output).each { |param| self.send("#{param}=", params[param.to_sym]) or raise "#{param} is needed" }
+      self.chunks_dir = if params[:chunks_dir]
+        params[:chunks_dir]
+      elsif !input.match(/^https?:\/\//)
+        File.dirname(input)
+      end
+      self.replace_in_chunk = params[:replace_in_chunk]
+      self.log = params[:log] || '/dev/null'
+      self.verbose = params[:verbose]
+    end
+
+    def concat
+      chunks = parse_playlist rescue []
+      raise "Empty or invalid playlist #{input}" if chunks.empty?
+      output_dir = File.dirname(output)
+      `mkdir -p #{output_dir}` unless Dir.exists?(output_dir)
+      concat_file = File.join(output_dir, 'concat.ts')
+      `rm #{concat_file}` if File.exists? concat_file
+      chunks.each do |chunk|
+        local_chunk = if chunks_dir
+          File.join(chunks_dir, chunk)
+        elsif replace_in_chunk
+          chunk.sub(replace_in_chunk.first, replace_in_chunk.last)
+        end
+        if local_chunk && File.exists?(local_chunk)
+          message = "Copy #{local_chunk} to #{concat_file}"
+          puts message if verbose
+          yield message if block_given?
+          `cat #{local_chunk} >> #{concat_file}`
+        else
+          chunk = File.join(File.dirname(input), chunk) unless chunk.match(/(^https?:\/\/)|(^\/)/)
+          message = "Download #{chunk} to #{concat_file}"
+          puts message if verbose
+          yield message if block_given?
+          `cd #{output_dir} && wget #{chunk} 1>>#{log} 2>&1 && cat #{File.basename(chunk)} >> #{concat_file} && rm #{File.basename(chunk)}` 
+        end
+      end
+      raise "Cannot download chunks from #{input}" unless File.exists?(concat_file) && File.size(concat_file) > 0
+      puts "Convert #{concat_file}" if verbose
+      `#{ffmpeg_bin} -i #{concat_file} -vcodec copy -acodec copy -f mpegts pipe:1 2>>/dev/null | #{ffmpeg_bin} -y -i - -acodec copy -vcodec copy -absf aac_adtstoasc -f mp4 #{output} 1>>#{log} 2>&1`
+      `rm #{concat_file}`
+      output
+    end
+
+    private
+
+    def parse_playlist
+      require 'open-uri'
+      open(input) { |f| f.read }.scan(/EXTINF:[\d.]+.*?\n(.*?)\n/).flatten
+    end
+  end
+end

data/lib/video_converter/input.rb

@@ -18,7 +18,7 @@ module VideoConverter
     end
 
     def to_s
-      input.gsub(/ /, "\\ ")
+      Shellwords.escape(input).gsub("\\\\","\\")
     end
 
     def exists?
@@ -37,7 +37,7 @@ module VideoConverter
 
     def metadata
       metadata = {}
-      s = `#{Command.new self.class.metadata_command, common_params}`
+      s = `#{Command.new self.class.metadata_command, common_params}`.encode!('UTF-8', 'UTF-8', :invalid => :replace)
       if (m = s.match(/Stream.*?Audio:\s*(\w+).*?(\d+)\s*Hz.*?(\d+)\s*kb\/s$/).to_a).any?
         metadata[:audio_codec] = m[1]
         metadata[:audio_sample_rate] = m[2].to_i
@@ -83,7 +83,7 @@ module VideoConverter
     private
 
     def common_params
-      { :bin => VideoConverter::Ffmpeg.bin, :input => to_s }
+      { :bin => VideoConverter::Ffmpeg.bin, :input => input }
     end
   end
 end

data/lib/video_converter/input_array.rb

@@ -8,7 +8,7 @@ module VideoConverter
       self.inputs = (inputs.is_a?(Array) ? inputs : [inputs]).map { |input| Input.new(input) }
       output_array.outputs.each do |output|
         if [:standard, :segmented].include? output.type
-          self.inputs[self.inputs.index { |input| input.to_s == output.path.to_s.gsub(/ /, "\\ ") }.to_i].outputs << output
+          self.inputs[self.inputs.index { |input| input.to_s == Shellwords.escape(output.path.to_s).gsub("\\\\","\\") }.to_i].outputs << output
         end
       end
       self.inputs.each do |input|

data/lib/video_converter/output.rb

@@ -16,7 +16,7 @@ module VideoConverter
     self.video_codec = 'libx264'
     self.audio_codec = 'libfaac'
 
-    attr_accessor :type, :url, :base_url, :filename, :format, :video_bitrate, :uid, :streams, :work_dir, :local_path, :playlist, :items, :segment_seconds, :chunks_dir, :audio_bitrate, :keyframe_interval, :threads, :video_codec, :audio_codec, :path, :thumbnails
+    attr_accessor :type, :url, :base_url, :filename, :format, :video_bitrate, :uid, :streams, :work_dir, :local_path, :playlist, :items, :segment_seconds, :chunks_dir, :audio_bitrate, :keyframe_interval, :threads, :video_codec, :audio_codec, :path, :thumbnails, :frame_rate, :size, :width, :height
 
     def initialize params = {}
       self.uid = params[:uid].to_s
@@ -59,17 +59,23 @@ module VideoConverter
 
       # Frame rate
       self.keyframe_interval = params[:keyframe_interval].to_i > 0 ? params[:keyframe_interval].to_i : self.class.keyframe_interval
+      self.frame_rate = params[:frame_rate].to_i if params[:frame_rate]
 
       # Format and codecs
       self.video_codec = (params[:copy_video] ? 'copy' : params[:video_codec]) || self.class.video_codec
       self.audio_codec = (params[:copy_audio] ? 'copy' : params[:audio_codec]) || self.class.audio_codec
 
+      # Resolution
+      self.size = params[:size]
+      self.width = params[:width]
+      self.height = params[:height]
+
       #Thumbnails
       self.thumbnails = params[:thumbnails]
     end
 
     def to_hash
-      keys = [:video_bitrate, :local_path, :segment_seconds, :chunks_dir, :audio_bitrate, :keyframe_interval, :threads, :video_codec, :audio_codec]
+      keys = [:video_bitrate, :local_path, :segment_seconds, :chunks_dir, :audio_bitrate, :keyframe_interval, :frame_rate, :threads, :video_codec, :audio_codec, :size, :width, :height]
       Hash[*keys.map{ |key| [key, self.send(key)] }.flatten]
     end
 

data/lib/video_converter/process.rb

@@ -2,31 +2,81 @@
 
 module VideoConverter
   class Process
-    attr_accessor :uid, :status, :progress, :pid
+    attr_accessor :uid, :status, :progress, :status_progress, :pid, :process_dir, :status_progress_koefs
 
     class << self
-      attr_accessor :path
+      attr_accessor :process_dir, :collect_progress_interval
     end
-    self.path = 'tmp/processes'
+    self.process_dir = '/tmp/video_converter_process'
+    self.collect_progress_interval = 10
 
-    def initialize uid
+    def self.find uid
+      if Dir.exists?(File.join(process_dir, uid))
+        new uid
+      else
+        nil
+      end
+    end
+
+    def initialize uid, output_array = nil
       self.uid = uid
-      Dir.mkdir(self.class.path) unless Dir.exists?(self.class.path)
+      self.process_dir = File.join(self.class.process_dir, uid)
+
+      unless Dir.exists? process_dir
+        FileUtils.mkdir_p process_dir
+        self.pid = `cat /proc/self/stat`.split[3]
+        self.status = 'started'
+        self.progress = self.status_progress = 0
+      end
+
+      if output_array
+        self.status_progress_koefs = {}
+        self.status_progress_koefs[:screenshot] = 0.05 if output_array.outputs.detect { |o| o.thumbnails }
+        self.status_progress_koefs[:segment] = 0.1 if output_array.playlists.any?
+        self.status_progress_koefs[:convert] = 1 - status_progress_koefs.values.inject(0, :+)
+      else
+        self.status_progress_koefs = { :screenshot => 0.05, :segment => 0.1, :convert => 0.85 }
+      end
     end
 
-    [:status, :progress, :pid].each do |attr|
-      define_method(attr) { File.open(send("#{attr}_file".to_sym), 'r') { |f| f.read } rescue nil }
-      define_method("#{attr}=") { |value| File.open(send("#{attr}_file".to_sym), 'w') { |f| f.write value } }
+    # attr are saved in local file
+    def get_attr attr
+      File.open(File.join(process_dir, "#{attr}.txt"), 'r') { |f| f.read } rescue nil
+    end
+    def set_attr attr, value
+      File.open(File.join(process_dir, "#{attr}.txt"), 'w') { |f| f.write value }
+    end
+
+    [:progress, :status_progress, :pid].each do |attr|
+      define_method attr do
+        get_attr attr
+      end
+      define_method "#{attr}=" do |value|
+        set_attr attr, value
+      end
+    end
+        
+    def status
+      get_attr :status
+    end
+
+    def status= value
+      set_attr :status, value
+      FileUtils.mkdir_p progress_dir if %w(convert).include?(value.to_s)
+      self.progress = self.status_progress = 1 if value.to_s == 'finished'
     end
 
     def stop
       `pkill -P #{pid}`
     end
 
-    private
+    def collect_progress progress
+      self.status_progress = progress
+      self.progress = progress * status_progress_koefs[status.to_sym].to_f
+    end
 
-    [:status, :progress, :pid].each do |attr|
-      define_method("#{attr}_file".to_sym) { File.join self.class.path, "#{uid}_#{attr}" }
+    def progress_dir
+      File.join process_dir, 'progress', status
     end
   end
 end

data/lib/video_converter/version.rb

@@ -1,3 +1,3 @@
 module VideoConverter
-  VERSION = "0.2.13"
+  VERSION = "0.3.0"
 end

data/test/video_converter_test.rb

@@ -9,7 +9,7 @@ class VideoConverterTest < Test::Unit::TestCase
 
     context 'with default type' do
       setup do
-        @c = VideoConverter.new('input' => @input_file, 'output' => [{'video_bitrate' => 300, 'filename' => 'tmp/test1.mp4'}, {'video_bitrate' => 700, :filename => 'tmp/test2.mp4'}], 'log' => 'tmp/test.log')
+        @c = VideoConverter.new('input' => @input_file, 'output' => [{'video_bitrate' => 300, 'filename' => 'tmp/test1.mp4'}, {'video_bitrate' => 700, :filename => 'tmp/test2.mp4'}], 'log' => 'tmp/test.log', :clear_tmp => false)
         @res = @c.run
       end
       should 'convert files' do

data/video_converter.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "video_screenshoter", "~> 0.0.6"
+  spec.add_runtime_dependency "video_screenshoter", "~> 0.2.1"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: video_converter
 version: !ruby/object:Gem::Version
-  version: 0.2.13
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-18 00:00:00.000000000 Z
+date: 2013-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: video_screenshoter
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.6
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.6
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -109,6 +109,7 @@ files:
 - lib/video_converter/base.rb
 - lib/video_converter/command.rb
 - lib/video_converter/ffmpeg.rb
+- lib/video_converter/hls.rb
 - lib/video_converter/input.rb
 - lib/video_converter/input_array.rb
 - lib/video_converter/live_segmenter.rb
@@ -136,7 +137,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2281938760435681386
+      hash: -805925364969050678
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -145,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2281938760435681386
+      hash: -805925364969050678
 requirements:
 - ffmpeg, version 1.2 or greated configured with libx264 and libfaac
 - live_segmenter to convert to hls