ffmprb 0.7.0 → 0.7.3

data/lib/ffmprb/util/{io_buffer.rb → threaded_io_buffer.rb}

@@ -4,7 +4,7 @@ module Ffmprb
 
     # XXX the events mechanism is currently unused (and commented out) => synchro mechanism not needed
     # XXX partially specc'ed in file_spec
-    class IoBuffer
+    class ThreadedIoBuffer
       # include Synchro
 
       class << self
@@ -20,126 +20,43 @@ module Ffmprb
       end
 
       # NOTE input/output can be lambdas for single asynchronic io evaluation
-      # NOTE both ios are closed as soon as possible
-      def initialize(input, output,
-        blocks_max: self.class.blocks_max, block_size: self.class.block_size)
+      #      the labdas must be timeout-interrupt-safe (since they are wrapped in timeout blocks)
+      # NOTE both ios are being opened and closed as soon as possible
+      def initialize(input, output)
 
         @input = input
         @output = output
-        @q = SizedQueue.new(blocks_max)
+        @q = SizedQueue.new(self.class.blocks_max)
         @stat_blocks_max = 0
         @terminate = false
         # @events = {}
 
-        # NOTE reads all of input, then closes the stream times out on buffer overflow
+        Thread.new "io buffer main" do
+          init_reader!
+          init_writer_output!
+          init_writer!
 
-        @reader = Util::Thread.new("buffer reader") do
-          begin
-            while s = reader_input!.read(block_size)
-              begin
-                Timeout::timeout(self.class.timeout) do
-                  @q.enq s
-                end
-              rescue Timeout::Error  # NOTE the queue is probably overflown
-                @terminate = Error.new("The reader has failed with timeout while queuing")
-                # timeout!
-                raise Error, "Looks like we're stuck (#{self.class.timeout}s idle) with #{blocks_max}x#{block_size}B blocks (buffering #{reader_input!.path}->...)..."
-              end
-              @stat_blocks_max = blocks_count  if blocks_count > @stat_blocks_max
-            end
-            @terminate = true
-            @q.enq nil
-          ensure
-            begin
-              reader_input!.close  if reader_input!.respond_to?(:close)
-            rescue
-              Ffmprb.logger.error "IoBuffer input closing error: #{$!.message}"
-            end
-            # reader_done!
-            Ffmprb.logger.debug "IoBuffer reader terminated (blocks max: #{@stat_blocks_max})"
-          end
-        end
-
-        init_writer_output!
-
-        # NOTE writes as much output as possible, then terminates when the reader dies
-
-        @writer = Util::Thread.new("buffer writer") do
-          broken = false
-          begin
-            while s = @q.deq
-              next  if broken
-              written = 0
-              tries = 1
-              logged_tries = 1/2
-              while !broken
-                raise @terminate  if @terminate.kind_of?(Exception)
-                begin
-                  output = writer_output!
-                  written = output.write_nonblock(s)  if output  # NOTE will only be nil if @terminate is an exception
-                  break  if written == s.length  # NOTE kinda optimisation
-                  s = s[written..-1]
-                rescue Errno::EAGAIN, Errno::EWOULDBLOCK
-                  if tries == 2 * logged_tries
-                    Ffmprb.logger.debug "IoBuffer writer (to #{output.path}) retrying... (#{tries} writes): #{$!.class}"
-                    logged_tries = tries
-                  end
-                  sleep 0.01
-                rescue Errno::EPIPE
-                  broken = true
-                  Ffmprb.logger.debug "IoBuffer writer (to #{output.path}) broken"
-                ensure
-                  tries += 1
-                end
-              end
-            end
-          ensure
-            # terminated!
-            begin
-              writer_output!.close  if !broken && writer_output!.respond_to?(:close)
-            rescue
-              Ffmprb.logger.error "IoBuffer output closing error: #{$!.message}"
-            end
-            Ffmprb.logger.debug "IoBuffer writer terminated (blocks max: #{@stat_blocks_max})"
-          end
-        end
-      end
-
-      def flush!  # NOTE blocking, closes ios
-        e = nil
-        # [@reader, @writer, @handler_thr].each do |thr|
-        [@reader, @writer, @output_thr].compact.each do |thr|
-          begin
-            thr.join
-          rescue
-            if e
-              Ffmprb.logger.debug "Additionally got (hidden): #{$!.message}"
-            else
-              e = $!
-            end
-          end
+          Thread.join_children!
         end
-        raise e  if e
       end
-      # handle_synchronously :flush!
       #
       # def once(event, &blk)
       #   event = event.to_sym
       #   wait_for_handler!
       #   if @events[event].respond_to? :call
-      #     raise Error, "Once upon a time (one #once(event) at a time) please"
+      #     fail Error, "Once upon a time (one #once(event) at a time) please"
       #   elsif @events[event]
-      #     Ffmprb.logger.debug "IoBuffer (post-)reacting to #{event}"
+      #     Ffmprb.logger.debug "ThreadedIoBuffer (post-)reacting to #{event}"
       #     @handler_thr = Util::Thread.new "#{event} handler", &blk
       #   else
-      #     Ffmprb.logger.debug "IoBuffer subscribing to #{event}"
+      #     Ffmprb.logger.debug "ThreadedIoBuffer subscribing to #{event}"
       #     @events[event] = blk
       #   end
       # end
       # handle_synchronously :once
       #
       # def reader_done!
-      #   Ffmprb.logger.debug "IoBuffer reader terminated (blocks max: #{@stat_blocks_max})"
+      #   Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (blocks max: #{@stat_blocks_max})"
       #   fire! :reader_done
       # end
       #
@@ -155,7 +72,7 @@ module Ffmprb
       #
       # def fire!(event)
       #   wait_for_handler!
-      #   Ffmprb.logger.debug "IoBuffer firing #{event}"
+      #   Ffmprb.logger.debug "ThreadedIoBuffer firing #{event}"
       #   if blk = @events.to_h[event.to_sym]
       #     @handler_thr = Util::Thread.new "#{event} handler", &blk
       #   end
@@ -189,23 +106,86 @@ module Ffmprb
       def init_writer_output!
         return  unless @output.respond_to?(:call)
 
-        @output_thr = Util::Thread.new("buffer writer output helper") do
+        @output_thr = Thread.new("buffer writer output helper") do
           Ffmprb.logger.debug "Opening buffer output"
-          tries = 1
-          logged_tries = 1/2
+          @output =
+            Thread.timeout_or_live nil, log: "in the buffer writer helper thread", timeout: self.class.timeout do |time|
+              fail Error, "giving up buffer writer init since the reader has failed (#{@terminate.message})"  if @terminate.kind_of?(Exception)
+              @output.call
+            end
+          Ffmprb.logger.debug "Opened buffer output: #{@output.path}"
+        end
+      end
+
+      # NOTE reads all of input, then closes the stream times out on buffer overflow
+      def init_reader!
+        Thread.new("buffer reader") do
           begin
-            Timeout::timeout(self.class.timeout/2) do
-              @output = @output.call
-              Ffmprb.logger.debug "Opened buffer output: #{@output.path}"
+            while s = reader_input!.read(self.class.block_size)
+              begin
+                Timeout.timeout(self.class.timeout) do
+                  @q.enq s
+                end
+              rescue Timeout::Error  # NOTE the queue is probably overflown
+                @terminate = Error.new("The reader has failed with timeout while queuing")
+                # timeout!
+                fail Error, "Looks like we're stuck (#{timeout}s idle) with #{self.class.blocks_max}x#{self.class.block_size}B blocks (buffering #{reader_input!.path}->...)..."
+              end
+              @stat_blocks_max = blocks_count  if blocks_count > @stat_blocks_max
             end
-          rescue Timeout::Error
-            if tries == 2 * logged_tries
-              Ffmprb.logger.info "A little bit of timeout in the buffer writer helper thread (##{tries})"
-              logged_tries = tries
+            @terminate = true
+            @q.enq nil
+          ensure
+            begin
+              reader_input!.close  if reader_input!.respond_to?(:close)
+            rescue
+              Ffmprb.logger.error "ThreadedIoBuffer input closing error: #{$!.message}"
+            end
+            # reader_done!
+            Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (blocks max: #{@stat_blocks_max})"
+          end
+        end
+      end
+
+      # NOTE writes as much output as possible, then terminates when the reader dies
+      def init_writer!
+        Thread.new("buffer writer") do
+          broken = false
+          begin
+            while s = @q.deq
+              next  if broken
+              written = 0
+              tries = 1
+              logged_tries = 1/2
+              while !broken
+                fail @terminate  if @terminate.kind_of?(Exception)
+                begin
+                  output = writer_output!
+                  written = output.write_nonblock(s)  if output  # NOTE will only be nil if @terminate is an exception
+                  break  if written == s.length  # NOTE kinda optimisation
+                  s = s[written..-1]
+                rescue Errno::EAGAIN, Errno::EWOULDBLOCK
+                  if tries == 2 * logged_tries
+                    Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output.path}) retrying... (#{tries} writes): #{$!.class}"
+                    logged_tries = tries
+                  end
+                  sleep 0.01
+                rescue Errno::EPIPE
+                  broken = true
+                  Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output.path}) broken"
+                ensure
+                  tries += 1
+                end
+              end
             end
-            retry  unless @terminate.kind_of?(Exception)
           ensure
-            tries += 1
+            # terminated!
+            begin
+              writer_output!.close  if !broken && writer_output!.respond_to?(:close)
+            rescue
+              Ffmprb.logger.error "ThreadedIoBuffer output closing error: #{$!.message}"
+            end
+            Ffmprb.logger.debug "ThreadedIoBuffer writer terminated (blocks max: #{@stat_blocks_max})"
           end
         end
       end

data/lib/ffmprb/util.rb

@@ -1,6 +1,6 @@
 # require 'ffmprb/util/synchro'
 require 'ffmprb/util/thread'
-require 'ffmprb/util/io_buffer'
+require 'ffmprb/util/threaded_io_buffer'
 
 require 'open3'
 
@@ -8,50 +8,81 @@ module Ffmprb
 
   module Util
 
+    class TimeLimitError < Error; end
+
     class << self
 
       attr_accessor :ffmpeg_cmd, :ffprobe_cmd
+      attr_accessor :cmd_timeout
 
-      def ffprobe(*args)
-        sh *ffprobe_cmd, *args
+      def ffprobe(*args, limit: nil, timeout: cmd_timeout)
+        sh *ffprobe_cmd, *args, limit: limit, timeout: timeout
       end
 
-      def ffmpeg(*args)
+      def ffmpeg(*args, limit: nil, timeout: cmd_timeout)
         args = ['-loglevel', 'debug'] + args  if Ffmprb.debug
-        sh *ffmpeg_cmd, '-y', *args, output: :stderr
+        sh *ffmpeg_cmd, '-y', *args, output: :stderr, limit: limit, timeout: timeout
       end
 
-      def sh(*cmd, output: :stdout, log: :stderr)
-        cmd = cmd.to_a.map(&:to_s)
-        cmd_str = cmd.join(' ')
-        Ffmprb.logger.info cmd_str
-        Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
-          stdin.close
+      def sh(*cmd, output: :stdout, log: :stderr, limit: nil, timeout: cmd_timeout)
+        cmd = cmd.map &:to_s  unless cmd.size == 1
+        cmd_str = cmd.size != 1 ? cmd.map{|c| "\"#{c}\""}.join(' ') : cmd.first
+        timeout = [timeout, limit].compact.min
+        thr = Thread.new "`#{cmd_str}`" do
+          Ffmprb.logger.info "Popening `#{cmd_str}`..."
+          Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
+            begin
+              stdin.close
 
-          # XXX process timeouting/cleanup here will be appreciated
+              log_cmd = cmd.first.upcase  if log
+              stdout_r = Reader.new(stdout, output == :stdout, log == :stdout && log_cmd)
+              stderr_r = Reader.new(stderr, true, log == :stderr && log_cmd)
 
-          begin
-            log_cmd = "#{(cmd.respond_to?(:first)? cmd : cmd.split(' ')).first.upcase}: "  if log
-            stdout_r = Reader.new(stdout, output == :stdout, log == :stdout && log_cmd)
-            stderr_r = Reader.new(stderr, true, log == :stderr && log_cmd)
+              Thread.timeout_or_live(limit, log: "while waiting for `#{cmd_str}`", timeout: timeout) do |time|
+                fail Error, "#{cmd_str}:\n#{stderr_r.read}"  unless
+                  wait_thr.value.exitstatus == 0  # NOTE blocking
+              end
+              Ffmprb.logger.debug "FINISHED: #{cmd_str}"
 
-            raise Error, "#{cmd_str}:\n#{stderr_r.read}"  unless
-              wait_thr.value.exitstatus == 0  # NOTE blocks
+              Thread.join_children! limit, timeout: timeout
 
-            # NOTE only one of them will return non-nil, see above
-            stdout_r.read || stderr_r.read
-          ensure
-            begin
-              stdout_r.join  if stdout_r
-              stdout_r = nil
-              stderr_r.join  if stderr_r
-            rescue
-              Ffmprb.logger.error "Thread joining error: #{$!.message}"
-              stderr_r.join  if stdout_r
+              # NOTE only one of them will return non-nil, see above
+              stdout_r.read || stderr_r.read
+            ensure
+              process_dead! wait_thr, cmd_str, limit
             end
-            Ffmprb.logger.debug "FINISHED: #{cmd_str}"
           end
         end
+        thr.value
+      end
+
+      protected
+
+      def process_dead!(wait_thr, cmd_str, limit)
+        grace = limit ? limit/4 : 1
+        return  unless wait_thr.alive?
+
+        # NOTE a simplistic attempt to gracefully terminate a child process
+        # the successful completion is via exception...
+        begin
+          Ffmprb.logger.info "Sorry it came to this, but I'm terminating `#{cmd_str}`(#{wait_thr.pid})..."
+          ::Process.kill 'TERM', wait_thr.pid
+          sleep grace
+          Ffmprb.logger.info "Very sorry it came to this, but I'm terminating `#{cmd_str}`(#{wait_thr.pid}) again..."
+          ::Process.kill 'TERM', wait_thr.pid
+          sleep grace
+          Ffmprb.logger.warn "Die `#{cmd_str}`(#{wait_thr.pid}), die!.. (killing amok)"
+          ::Process.kill 'KILL', wait_thr.pid
+          sleep grace
+          Ffmprb.logger.warn "Checking if `#{cmd_str}`(#{wait_thr.pid}) finally dead..."
+          ::Process.kill 0, wait_thr.pid
+          Ffmprb.logger.error "Still alive -- `#{cmd_str}`(#{wait_thr.pid}), giving up..."
+        rescue Errno::ESRCH
+          Ffmprb.logger.info "Apparently `#{cmd_str}`(#{wait_thr.pid}) is dead..."
+        end
+
+        fail Error, "System error or something: waiting for the thread running `#{cmd_str}`(#{wait_thr.pid})..." unless
+          wait_thr.join limit
       end
 
     end
@@ -65,7 +96,7 @@ module Ffmprb
         super "reader" do
           begin
             while s = input.gets
-              Ffmprb.logger.debug log + s.chomp  if log
+              Ffmprb.logger.debug "#{log}: #{s.chomp}"  if log
               @output << s  if store
             end
             @queue.enq @output
@@ -78,7 +109,7 @@ module Ffmprb
       def read
         case res = @queue.deq
         when Exception
-          raise res
+          fail res
         when ''
           nil
         else

data/lib/ffmprb/version.rb

@@ -1,3 +1,3 @@
 module Ffmprb
-  VERSION = "0.7.0"
+  VERSION = '0.7.3'
 end

data/lib/ffmprb.rb

@@ -1,41 +1,23 @@
-require 'ffmprb/execution'
-require 'ffmprb/file'
-require 'ffmprb/filter'
-require 'ffmprb/find_silence'
-require 'ffmprb/process'
-require 'ffmprb/util'
-require 'ffmprb/version'
-
 require 'logger'
 
+# IMPORTANT NOTE ffmprb uses threads internally, however, it is not "thread-safe"
+
 module Ffmprb
 
   ENV_VAR_FALSE_REGEX = /^(0|no?|false)?$/i
 
+  CGA = '320x200'
   QVGA = '320x240'
   HD_720p = '1280x720'
   HD_1080p = '1920x1080'
 
-  class Error < StandardError
-  end
-
-  Util.ffmpeg_cmd = ['ffmpeg']
-  Util.ffprobe_cmd = ['ffprobe']
-
-  Process.duck_audio_hi = 0.9
-  Process.duck_audio_lo = 0.1
-  Process.duck_audio_transition_sec = 1
-  Process.duck_audio_silent_min_sec = 3
-  Filter.silence_noise_max_db = -40
-
-  Util::IoBuffer.blocks_max = 1024
-  Util::IoBuffer.block_size = 64*1024
-  Util::IoBuffer.timeout = 9
+  class Error < StandardError; end
 
   class << self
 
     # NOTE the form with the block returns the result of #run
-    # NOTE the form without the block returns the process (before it is run)
+    # NOTE the form without the block returns the process (before it is run) - advanced use
+    # XXX is this clear enough? Do we really need the second form?
     def process(*args, &blk)
       logger.debug "Starting process with #{args} in #{blk.source_location}"
       process = Process.new
@@ -68,3 +50,7 @@ module Ffmprb
 end
 
 Ffmprb.debug = ENV.fetch('FFMPRB_DEBUG', '') !~ Ffmprb::ENV_VAR_FALSE_REGEX
+
+Dir["#{__FILE__.slice /(.*).rb$/, 1}/**/*.rb"].each{|f| require f}  # XXX require_sub  __FILE__  # or something
+
+require 'defaults'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ffmprb
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.3
 platform: ruby
 authors:
 - showbox.com
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-07-28 00:00:00.000000000 Z
+date: 2015-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mkfifo
@@ -147,18 +147,20 @@ files:
 - circle.yml
 - exe/ffmprb
 - ffmprb.gemspec
+- lib/defaults.rb
 - lib/ffmprb.rb
 - lib/ffmprb/execution.rb
 - lib/ffmprb/file.rb
+- lib/ffmprb/file/sample.rb
 - lib/ffmprb/filter.rb
 - lib/ffmprb/find_silence.rb
 - lib/ffmprb/process.rb
 - lib/ffmprb/process/input.rb
 - lib/ffmprb/process/output.rb
 - lib/ffmprb/util.rb
-- lib/ffmprb/util/io_buffer.rb
 - lib/ffmprb/util/synchro.rb
 - lib/ffmprb/util/thread.rb
+- lib/ffmprb/util/threaded_io_buffer.rb
 - lib/ffmprb/version.rb
 homepage: https://github.com/showbox-oss/ffmprb
 licenses: []