ffmprb 0.11.3 → 0.12.1

data/lib/ffmprb/util/threaded_io_buffer.rb

@@ -6,7 +6,7 @@ module Ffmprb
 
     # TODO the events mechanism is currently unused (and commented out) => synchro mechanism not needed
     class ThreadedIoBuffer
-      # XXX include Synchro
+      # TODO? include Synchro
       include ProcVis::Node
 
       class << self
@@ -29,7 +29,7 @@ module Ffmprb
       def initialize(input, *outputs, keep_outputs_open_on_input_idle_limit: nil)
         super()  # NOTE for the monitor, apparently
 
-        Ffmprb.logger.debug "ThreadedIoBuffer initializing with (#{ThreadedIoBuffer.blocks_max}x#{ThreadedIoBuffer.block_size})"
+        Ffmprb.logger.debug{"ThreadedIoBuffer initializing with (#{ThreadedIoBuffer.blocks_max}x#{ThreadedIoBuffer.block_size})"}
 
         @input = input
         @outputs = outputs.map do |outp|
@@ -47,10 +47,11 @@ module Ffmprb
           end
 
           Thread.join_children!.tap do
-            Ffmprb.logger.debug "ThreadedIoBuffer (#{@input.path}->#{@outputs.map(&:io).map(&:path)}) terminated successfully (#{stats})"
+            Ffmprb.logger.debug{"ThreadedIoBuffer (#{@input.path}->#{@outputs.map(&:io).map(&:path)}) terminated successfully (#{stats})"}
           end
         end
       end
+      # TODO?
       #
       # def once(event, &blk)
       #   event = event.to_sym
@@ -58,17 +59,17 @@ module Ffmprb
       #   if @events[event].respond_to? :call
       #     fail Error, "Once upon a time (one #once(event) at a time) please"
       #   elsif @events[event]
-      #     Ffmprb.logger.debug "ThreadedIoBuffer (post-)reacting to #{event}"
+      #     Ffmprb.logger.debug{"ThreadedIoBuffer (post-)reacting to #{event}"}
       #     @handler_thr = Util::Thread.new "#{event} handler", &blk
       #   else
-      #     Ffmprb.logger.debug "ThreadedIoBuffer subscribing to #{event}"
+      #     Ffmprb.logger.debug{"ThreadedIoBuffer subscribing to #{event}"}
       #     @events[event] = blk
       #   end
       # end
       # handle_synchronously :once
       #
       # def reader_done!
-      #   Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (#{stats})"
+      #   Ffmprb.logger.debug{"ThreadedIoBuffer reader terminated (#{stats})"}
       #   fire! :reader_done
       # end
       #
@@ -84,7 +85,7 @@ module Ffmprb
       #
       # def fire!(event)
       #   wait_for_handler!
-      #   Ffmprb.logger.debug "ThreadedIoBuffer firing #{event}"
+      #   Ffmprb.logger.debug{"ThreadedIoBuffer firing #{event}"}
       #   if blk = @events.to_h[event.to_sym]
       #     @handler_thr = Util::Thread.new "#{event} handler", &blk
       #   end
@@ -104,9 +105,9 @@ module Ffmprb
 
       def reader_input!  # NOTE just for reader thread
         if @input.respond_to?(:call)
-          Ffmprb.logger.debug "Opening buffer input"
+          Ffmprb.logger.debug{"Opening buffer input"}
           @input = @input.call
-          Ffmprb.logger.debug "Opened buffer input: #{@input.path}"
+          Ffmprb.logger.debug{"Opened buffer input: #{@input.path}"}
         end
         @input
       end
@@ -137,17 +138,17 @@ module Ffmprb
                 rescue IO::WaitReadable
                   if @keep_outputs_open_on_input_idle_limit && stats.bytes_in > 0 && stats.blocks_buff == 0 && timeouts * ThreadedIoBuffer.io_wait_timeout > @keep_outputs_open_on_input_idle_limit
                     if s.length > 0  # NOTE let's see if it helps outputting an incomplete block
-                      Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) giving a chance to write #{s.length}/#{ThreadedIoBuffer.block_size}b after waiting >#{@keep_outputs_open_on_input_idle_limit}s, after reading #{stats.bytes_in}b"
+                      Ffmprb.logger.debug{"ThreadedIoBuffer reader (from #{input_io.path}) giving a chance to write #{s.length}/#{ThreadedIoBuffer.block_size}b after waiting >#{@keep_outputs_open_on_input_idle_limit}s, after reading #{stats.bytes_in}b"}
                       break
                     else
-                      Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) giving up after waiting >#{@keep_outputs_open_on_input_idle_limit}s, after reading #{stats.bytes_in}b, closing outputs"
+                      Ffmprb.logger.debug{"ThreadedIoBuffer reader (from #{input_io.path}) giving up after waiting >#{@keep_outputs_open_on_input_idle_limit}s, after reading #{stats.bytes_in}b, closing outputs"}
                       raise EOFError
                     end
                   else
                     Thread.current.live!
                     timeouts += 1
                     if timeouts > 2 * logged_timeouts
-                      Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) retrying... (#{timeouts} reads): #{$!.class}"
+                      Ffmprb.logger.debug{"ThreadedIoBuffer reader (from #{input_io.path}) retrying... (#{timeouts} reads): #{$!.class}"}
                       logged_timeouts = timeouts
                     end
                     IO.select [input_io], nil, nil, ThreadedIoBuffer.io_wait_timeout
@@ -165,7 +166,7 @@ module Ffmprb
               output_enq! s
             end
           rescue EOFError
-            Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) breaking off"
+            Ffmprb.logger.debug{"ThreadedIoBuffer reader (from #{input_io.path}) breaking off"}
           rescue AllOutputsBrokenError
             Ffmprb.logger.info "All outputs broken"
           rescue Exception
@@ -182,7 +183,7 @@ module Ffmprb
               Ffmprb.logger.error "#{$!.class.name} closing ThreadedIoBuffer input: #{$!.message}"
             end
             # reader_done!
-            Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (#{stats})"
+            Ffmprb.logger.debug{"ThreadedIoBuffer reader terminated (#{stats})"}
           end
         end
       end
@@ -191,13 +192,13 @@ module Ffmprb
         return output.io = output._io  unless output._io.respond_to?(:call)
 
         output.thr = Thread.new("buffer writer output helper") do
-          Ffmprb.logger.debug "Opening buffer output"
+          Ffmprb.logger.debug{"Opening buffer output"}
           output.io =
             Thread.timeout_or_live nil, log: "in the buffer writer helper thread", timeout: ThreadedIoBuffer.timeout do |time|
               fail Error, "giving up buffer writer init since the reader has failed (#{@reader_failed.message})"  if @reader_failed
               output._io.call
             end
-          Ffmprb.logger.debug "Opened buffer output: #{output.io.path}"
+          Ffmprb.logger.debug{"Opened buffer output: #{output.io.path}"}
         end
       end
 
@@ -223,7 +224,7 @@ module Ffmprb
                 Thread.current.live!
                 timeouts += 1
                 if timeouts > 2 * logged_timeouts
-                  Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) retrying... (#{timeouts} writes): #{$!.class}"
+                  Ffmprb.logger.debug{"ThreadedIoBuffer writer (to #{output_io.path}) retrying... (#{timeouts} writes): #{$!.class}"}
                   logged_timeouts = timeouts
                 end
                 IO.select nil, [output_io], nil, ThreadedIoBuffer.io_wait_timeout
@@ -234,9 +235,9 @@ module Ffmprb
                 retry
               end
             end
-            Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) breaking off"
+            Ffmprb.logger.debug{"ThreadedIoBuffer writer (to #{output_io.path}) breaking off"}
           rescue Errno::EPIPE
-            Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) broken"
+            Ffmprb.logger.debug{"ThreadedIoBuffer writer (to #{output_io.path}) broken"}
             output.broken = true
           ensure
             # terminated!
@@ -246,7 +247,7 @@ module Ffmprb
               Ffmprb.logger.error "#{$!.class.name} closing ThreadedIoBuffer output: #{$!.message}"
             end
             output.broken = true
-            Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io && output_io.path}) terminated (#{stats})"
+            Ffmprb.logger.debug{"ThreadedIoBuffer writer (to #{output_io && output_io.path}) terminated (#{stats})"}
           end
         end
       end
@@ -279,9 +280,11 @@ module Ffmprb
                 logged_timeouts = timeouts
               end
 
-              retry  unless timeouts >= ThreadedIoBuffer.timeout_limit # NOTE the queue has probably overflown
+              retry  unless  # NOTE the queue has probably overflown
+                timeouts >= ThreadedIoBuffer.timeout_limit
 
-              @reader_failed ||= Error.new("the writer has failed with timeout limit while queuing")  # NOTE screw the race condition
+              @reader_failed ||=  # NOTE screw the race condition
+                Error.new("the writer has failed with timeout limit while queuing")
               # timeout!
               fail Error, "Looks like we're stuck (>#{ThreadedIoBuffer.timeout_limit*ThreadedIoBuffer.timeout}s idle) with #{ThreadedIoBuffer.blocks_max}x#{ThreadedIoBuffer.block_size}b blocks (buffering #{reader_input!.path}->...)..."
             end

data/lib/ffmprb/util.rb

@@ -9,6 +9,8 @@ module Ffmprb
     class BrokenPipeError < Error; end
     class TimeLimitError < Error; end
 
+    FFMPEG_BROKEN_PIPE_ERROR_RE = /^.*\berror\b.*:.*\bbroken pipe\b.*$/i
+
     class << self
 
       attr_accessor :ffmpeg_cmd, :ffmpeg_inputs_max, :ffprobe_cmd
@@ -18,17 +20,25 @@ module Ffmprb
         sh *ffprobe_cmd, *args, limit: limit, timeout: timeout
       end
 
+      # TODO un-colorise ffmpeg output for logging, also, convert ^M into something
       def ffmpeg(*args, limit: nil, timeout: cmd_timeout, ignore_broken_pipes: true)
-        args = ['-loglevel', 'debug'] + args  if Ffmprb.ffmpeg_debug
-        sh *ffmpeg_cmd, *args, output: :stderr, limit: limit, timeout: timeout, ignore_broken_pipes: ignore_broken_pipes
+        args = %w[-loglevel debug] + args  if
+          Ffmprb.ffmpeg_debug
+        sh *ffmpeg_cmd, *args,
+        output: :stderr,
+        limit: limit,
+        timeout: timeout,
+        ignore_broken_pipes: ignore_broken_pipes,
+        broken_pipe_error_re: FFMPEG_BROKEN_PIPE_ERROR_RE
       end
 
-      def sh(*cmd, input: nil, output: :stdout, limit: nil, timeout: cmd_timeout, ignore_broken_pipes: false)
+      def sh(*cmd, input: nil, output: :stdout, limit: nil, timeout: cmd_timeout, ignore_broken_pipes: false, broken_pipe_error_re: nil)
         cmd = cmd.map &:to_s  unless cmd.size == 1
         cmd_str = cmd.size != 1 ? cmd.map{|c| sh_escape c}.join(' ') : cmd.first
+        cmd_log_line = "#{log_hash cmd_str}: `#{cmd_str}`"
         timeout = [timeout, limit].compact.min
-        thr = Thread.new "`#{cmd_str}`" do
-          Ffmprb.logger.info "Popening `#{cmd_str}`..."
+        thr = Thread.new cmd_log_line do
+          Ffmprb.logger.info "Popening #{cmd_log_line}..."
           Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
             begin
               stdin.write input  if input
@@ -37,29 +47,33 @@ module Ffmprb
               log_cmd = cmd.first.upcase
               stdout_r = Reader.new(stdout, store: output == :stdout, log_with: log_cmd)
               stderr_r = Reader.new(stderr, store: true, log_with: log_cmd, log_as: output == :stderr && Logger::DEBUG || Logger::INFO)
+              stderr_s = nil
 
-              Thread.timeout_or_live(limit, log: "while waiting for `#{cmd_str}`", timeout: timeout) do |time|
+              Thread.timeout_or_live(limit, log: "while waiting for #{cmd_log_line}", timeout: timeout) do |time|
                 value = wait_thr.value
                 status = value.exitstatus  # NOTE blocking
                 if status != 0
-                  if value.signaled? && value.termsig == Signal.list['PIPE']
+                  stderr_s = stderr_r.read
+                  if (value.signaled? && value.termsig == Signal.list['PIPE']) ||
+                     # NOTE this doesn't seem to work for ffmpeg 4.x (it ignores SIGPIPEs)
+                     (broken_pipe_error_re && status == 1 && stderr_s =~ broken_pipe_error_re)
                     if ignore_broken_pipes
-                      Ffmprb.logger.info "Ignoring broken pipe: #{cmd_str}"
+                      Ffmprb.logger.info "Ignoring broken pipe: #{cmd_log_line}"
                     else
-                      fail BrokenPipeError, cmd_str
+                      fail BrokenPipeError, cmd_log_line
                     end
                   else
                     status ||= "sig##{value.termsig}"
-                    fail Error, "#{cmd_str} (#{status}):\n#{stderr_r.read}"
+                    fail Error, "#{cmd_log_line} (#{status}):\n#{stderr_s}"
                   end
                 end
               end
-              Ffmprb.logger.debug "FINISHED: #{cmd_str}"
+              Ffmprb.logger.debug{"FINISHED: #{cmd_log_line}"}
 
               Thread.join_children! limit, timeout: timeout
 
               # NOTE only one of them will return non-nil, see above
-              stdout_r.read || stderr_r.read
+              stdout_r.read || stderr_s || stderr_r.read
             ensure
               process_dead! wait_thr, cmd_str, limit
             end
@@ -71,7 +85,18 @@ module Ffmprb
       def assert_options_empty!(opts)
         fail ArgumentError, "Unknown options: #{opts}"  unless opts.empty?
       end
-      protected
+
+      private
+
+      def log_hash(s)
+        n = s.hash
+        %w[bcdfghk aeiuy mnprqstvwxz].reduce '' do |hash, chars|
+          hash + chars[n % chars.length]
+        end
+      end
+
+      def broken_pipe_error_printed?(s)
+      end
 
       # NOTE a best guess kinda method
       def sh_escape(str)

data/lib/ffmprb/version.rb

@@ -1,8 +1,8 @@
 module Ffmprb
 
-  VERSION = '0.11.3'
+  VERSION = '0.12.1'
 
-  GEM_GITHUB_URL = 'https://github.com/showbox-oss/ffmprb'
+  GEM_GITHUB_URL = 'https://github.com/costa/ffmprb'
 
   FIREBASE_AVAILABLE =
     begin

data/lib/ffmprb.rb

@@ -13,8 +13,10 @@ module Ffmprb
 
   CGA = '320x200'
   QVGA = '320x240'
+  HD_480p = '854x480'
   HD_720p = '1280x720'
   HD_1080p = '1920x1080'
+  HD_4K = '3840x2160'
 
   class << self
 
@@ -22,16 +24,17 @@ module Ffmprb
     def process(*args, name: nil, **opts, &blk)
       fail Error, "process: nothing ;( gimme a block!"  unless blk
 
-      name ||=  blk.source_location.map(&:to_s).map{ |s| ::File.basename s.to_s, ::File.extname(s) }.join(':')
+      name ||=  blk.source_location.map(&:to_s).map{ |s| File.basename s.to_s, File.extname(s) }.join(':')
       process = Process.new(name: name, **opts)
-      proc_vis_node process  if respond_to? :proc_vis_node  # XXX simply include the ProcVis if it makes into a gem
-      logger.debug "Starting process with #{args} #{opts} in #{blk.source_location}"
+      # TODO simply include the ProcVis if it makes into a gem
+      proc_vis_node process  if respond_to? :proc_vis_node
+      logger.debug{"Starting process with #{args} #{opts} in #{blk.source_location}"}
 
       process.instance_exec *args, &blk
-      logger.debug "Initialized process with #{args} #{opts} in #{blk.source_location}"
+      logger.debug{"Initialized process with #{args} #{opts} in #{blk.source_location}"}
 
       process.run.tap do
-        logger.debug "Finished process with #{args} #{opts} in #{blk.source_location}"
+        logger.debug{"Finished process with #{args} #{opts} in #{blk.source_location}"}
       end
     end
     alias :action! :process  # ;)

data/tmp/exp/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '2.1'
+services:
+  tester:
+    build: ../..
+    volumes:
+      - ../..:/ffmprb
+    environment:
+      - FFMPRB_DEBUG=1
+    command: tmp/exp/src