ffmprb 0.9.6 → 0.10.0

data/lib/ffmprb/util/thread.rb

@@ -2,8 +2,8 @@ module Ffmprb
 
   module Util
 
-    # NOTE doesn't have specs (and not too proud about it)
     class Thread < ::Thread
+      include ProcVis::Node
 
       class Error < Ffmprb::Error; end
       class ParentError < Error; end
@@ -14,19 +14,19 @@ module Ffmprb
 
         def timeout_or_live(limit=nil, log: "while doing this", timeout: self.timeout, &blk)
           started_at = Time.now
-          tries = 0
-          logged_tries = 0
+          timeouts = 0
+          logged_timeouts = 1
           begin
-            tries += 1
+            timeouts += 1
             time = Time.now - started_at
             fail TimeLimitError  if limit && time > limit
             Timeout.timeout timeout do
               blk.call time
             end
           rescue Timeout::Error
-            if tries > 2 * logged_tries
-              Ffmprb.logger.info "A little bit of timeout #{log.respond_to?(:call)? log.call : log} (##{tries})"
-              logged_tries = tries
+            if timeouts > 2 * logged_timeouts
+              Ffmprb.logger.info "A little bit of timeout #{log.respond_to?(:call)? log.call : log} (##{timeouts})"
+              logged_timeouts = timeouts
             end
             current.live!
             retry
@@ -41,7 +41,7 @@ module Ffmprb
 
       attr_reader :name
 
-      def initialize(name="some", &blk)
+      def initialize(name="some", main: false, &blk)
         @name = name
         @parent = Thread.current
         @live_children = []
@@ -50,7 +50,12 @@ module Ffmprb
         Ffmprb.logger.debug "about to launch #{name}"
         sync_q = Queue.new
         super() do
-          @parent.child_lives self  if @parent.respond_to? :child_lives
+          @parent.proc_vis_node self  if @parent.respond_to? :proc_vis_node
+          if @parent.respond_to? :child_lives
+            @parent.child_lives self
+          else
+            Ffmprb.logger.warn "Not the main: true thread run by a not #{self.class.name} thread"  unless main
+          end
           sync_q.enq :ok
           Ffmprb.logger.debug "#{name} thread launched"
           begin
@@ -58,13 +63,14 @@ module Ffmprb
               Ffmprb.logger.debug "#{name} thread done"
             end
           rescue Exception
-            Ffmprb.logger.warn "#{$!.class} raised in #{name} thread: #{$!.message}\nBacktrace:\n\t#{$!.backtrace.join("\n\t")}"
+            Ffmprb.logger.warn "#{$!.class.name} raised in #{name} thread: #{$!.message}\nBacktrace:\n\t#{$!.backtrace.join("\n\t")}"
             cause = $!
-            Ffmprb.logger.warn "...caused by #{cause.class}: #{cause.message}\nBacktrace:\n\t#{cause.backtrace.join("\n\t")}" while
+            Ffmprb.logger.warn "...caused by #{cause.class.name}: #{cause.message}\nBacktrace:\n\t#{cause.backtrace.join("\n\t")}" while
               cause = cause.cause
             fail $!  # XXX I have no idea why I need to give it `$!` -- the docs say I need not
           ensure
             @parent.child_dies self  if @parent.respond_to? :child_dies
+            @parent.proc_vis_node self, :remove  if @parent.respond_to? :proc_vis_node
           end
         end
         sync_q.deq
@@ -81,6 +87,7 @@ module Ffmprb
           Ffmprb.logger.debug "picking up #{thr.name} thread"
           @live_children << thr
         end
+        proc_vis_edge self, thr
       end
 
       def child_dies(thr)
@@ -89,9 +96,10 @@ module Ffmprb
           @dead_children_q.enq thr
           fail "System Error"  unless @live_children.delete thr
         end
+        proc_vis_edge self, thr, :remove
       end
 
-      def join_children!(limit=nil, timeout: self.class.timeout)
+      def join_children!(limit=nil, timeout: Thread.timeout)
         timeout = [timeout, limit].compact.min
         Ffmprb.logger.debug "joining threads: #{@live_children.size} live, #{@dead_children_q.size} dead"
         until @live_children.empty? && @dead_children_q.empty?

data/lib/ffmprb/util/threaded_io_buffer.rb

@@ -1,46 +1,52 @@
+require 'ostruct'
+
 module Ffmprb
 
   module Util
 
     # TODO the events mechanism is currently unused (and commented out) => synchro mechanism not needed
-    # XXX *partially* specc'ed in file_spec
     class ThreadedIoBuffer
-      # include Synchro
+      # XXX include Synchro
+      include ProcVis::Node
 
       class << self
 
         attr_accessor :blocks_max
         attr_accessor :block_size
         attr_accessor :timeout
-
-        def default_size
-          blocks_max * block_size
-        end
+        attr_accessor :timeout_limit
+        attr_accessor :io_wait_timeout
 
       end
 
+
       # NOTE input/output can be lambdas for single asynchronic io evaluation
-      #      the labdas must be timeout-interrupt-safe (since they are wrapped in timeout blocks)
-      # NOTE both ios are being opened and closed as soon as possible
-      def initialize(input, *outputs)  # XXX SPEC ME!!! multiple outputs!!
+      #      the lambdas must be timeout-interrupt-safe (since they are wrapped in timeout blocks)
+      # NOTE all ios are being opened and closed as soon as possible
+      def initialize(input, *outputs, keep_outputs_open_on_input_idle_limit: nil)
+        super()  # NOTE for the monitor, apparently
+
+        Ffmprb.logger.debug "ThreadedIoBuffer initializing with (#{ThreadedIoBuffer.blocks_max}x#{ThreadedIoBuffer.block_size})"
 
         @input = input
-        @outputs = outputs.inject({}) do |hash, out|
-          hash[out] = SizedQueue.new(self.class.blocks_max)
-          hash
+        @outputs = outputs.map do |outp|
+          OpenStruct.new _io: outp, q: SizedQueue.new(ThreadedIoBuffer.blocks_max)
         end
-        @stat_blocks_max = 0
+        @stats = Stats.new(self)
         @terminate = false
+        @keep_outputs_open_on_input_idle_limit = keep_outputs_open_on_input_idle_limit
         # @events = {}
 
         Thread.new "io buffer main" do
           init_reader!
-          outputs.each do |output|
+          @outputs.each do |output|
             init_writer_output! output
             init_writer! output
           end
 
-          Thread.join_children!
+          Thread.join_children!.tap do
+            Ffmprb.logger.debug "ThreadedIoBuffer (#{@input.path}->#{@outputs.map(&:io).map(&:path)}) terminated successfully (#{@stats})"
+          end
         end
       end
       #
@@ -60,7 +66,7 @@ module Ffmprb
       # handle_synchronously :once
       #
       # def reader_done!
-      #   Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (blocks max: #{@stat_blocks_max})"
+      #   Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (#{@stats})"
       #   fire! :reader_done
       # end
       #
@@ -72,7 +78,7 @@ module Ffmprb
       #   fire! :timeout
       # end
 
-      protected
+      # protected
       #
       # def fire!(event)
       #   wait_for_handler!
@@ -84,12 +90,16 @@ module Ffmprb
       # end
       # handle_synchronously :fire!
       #
-      def blocks_count
-        @outputs.values.map(&:size).max
+
+      def label
+        "IObuff: Curr/Peak/Max=#{@stats.blocks_buff}/#{@stats.blocks_max}/#{ThreadedIoBuffer.blocks_max} In/Out=#{@stats.bytes_in}/#{@stats.bytes_out}"
       end
 
       private
 
+      class AllOutputsBrokenError < Error
+      end
+
       def reader_input!  # NOTE just for reader thread
         if @input.respond_to?(:call)
           Ffmprb.logger.debug "Opening buffer input"
@@ -99,99 +109,139 @@ module Ffmprb
         @input
       end
 
+      # NOTE to be called after #init_writer_output! only
       def writer_output!(output)  # NOTE just for writer thread
-        if @output_thrs[output]
-          @output_thrs[output].join
-          @output_thrs[output] = nil
+        if output.thr
+          output.thr.join
+          output.thr = nil
         end
-        @output_ios[output]
+        output.io
       end
 
-      # NOTE reads all of input, then closes the stream times out on buffer overflow
+      # NOTE reads roughly as much input as writers can write, then closes the stream; times out on buffer overflow
       def init_reader!
         Thread.new("buffer reader") do
           begin
-            while s = reader_input!.read(self.class.block_size)
+            input_io = reader_input!
+            loop do
+              s = ''
               begin
-                Timeout.timeout(self.class.timeout) do
-                  output_enq s
+                while s.length < ThreadedIoBuffer.block_size
+                  timeouts = 0
+                  logged_timeouts = 1
+                  begin
+                    ss = input_io.read_nonblock(ThreadedIoBuffer.block_size - s.length)
+                    @stats.add_bytes_in ss.length
+                    s += ss
+                  rescue IO::WaitReadable
+                    if !@terminate && @stats.bytes_in > 0 && @stats.blocks_buff == 0 && @keep_outputs_open_on_input_idle_limit && timeouts * ThreadedIoBuffer.io_wait_timeout > @keep_outputs_open_on_input_idle_limit
+                      if s.length > 0
+                        output_enq! s
+                        s = ''  # NOTE let's see if it helps outputting an incomplete block
+                      else
+                        Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) giving up after waiting >#{@keep_outputs_open_on_input_idle_limit}s, after reading #{@stats.bytes_in}b closing outputs"
+                        @terminate = true
+                        output_enq! nil  # NOTE EOF signal
+                      end
+                    else
+                      timeouts += 1
+                      if !@terminate && timeouts > 2 * logged_timeouts
+                        Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) retrying... (#{timeouts} reads): #{$!.class}"
+                        logged_timeouts = timeouts
+                      end
+                      IO.select [input_io], nil, nil, ThreadedIoBuffer.io_wait_timeout
+                      retry
+                    end
+                  rescue IO::WaitWritable  # NOTE should not really happen, so just for conformance
+                    Ffmprb.logger.error "ThreadedIoBuffer reader (from #{input_io.path}) gets a #{$!} - should not really happen."
+                    IO.select nil, [input_io], nil, ThreadedIoBuffer.io_wait_timeout
+                    retry
+                  end
                 end
-              rescue Timeout::Error  # NOTE the queue is probably overflown
-                @terminate = Error.new("The reader has failed with timeout while queuing")
-                # timeout!
-                fail Error, "Looks like we're stuck (#{timeout}s idle) with #{self.class.blocks_max}x#{self.class.block_size}B blocks (buffering #{reader_input!.path}->...)..."
+              ensure
+                output_enq! s  unless @terminate
               end
-              @stat_blocks_max = blocks_count  if blocks_count > @stat_blocks_max
             end
-            @terminate = true
-            output_enq nil
+          rescue EOFError
+            unless @terminate
+              Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) breaking off"
+              @terminate = true
+              output_enq! nil  # NOTE EOF signal
+            end
+          rescue AllOutputsBrokenError
+            Ffmprb.logger.info "All outputs broken"
           ensure
             begin
               reader_input!.close  if reader_input!.respond_to?(:close)
             rescue
-              Ffmprb.logger.error "ThreadedIoBuffer input closing error: #{$!.message}"
+              Ffmprb.logger.error "#{$!.class.name} closing ThreadedIoBuffer input: #{$!.message}"
             end
             # reader_done!
-            Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (blocks max: #{@stat_blocks_max})"
+            Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (#{@stats})"
           end
         end
       end
 
       def init_writer_output!(output)
-        @output_ios ||= {}
-        return @output_ios[output] = output  unless output.respond_to?(:call)
+        return output.io = output._io  unless output._io.respond_to?(:call)
 
-        @output_thrs ||= {}
-        @output_thrs[output] = Thread.new("buffer writer output helper") do
+        output.thr = Thread.new("buffer writer output helper") do
           Ffmprb.logger.debug "Opening buffer output"
-          @output_ios[output] =
-            Thread.timeout_or_live nil, log: "in the buffer writer helper thread", timeout: self.class.timeout do |time|
-              fail Error, "giving up buffer writer init since the reader has failed (#{@terminate.message})"  if @terminate.kind_of?(Exception)
-              output.call
+          output.io =
+            Thread.timeout_or_live nil, log: "in the buffer writer helper thread", timeout: ThreadedIoBuffer.timeout do |time|
+              fail Error, "giving up buffer writer init since the reader has failed (#{@terminate.message})"  if @terminate.kind_of? Exception
+              output._io.call
             end
-          Ffmprb.logger.debug "Opened buffer output: #{@output_ios[output].path}"
+          Ffmprb.logger.debug "Opened buffer output: #{output.io.path}"
         end
       end
 
       # NOTE writes as much output as possible, then terminates when the reader dies
       def init_writer!(output)
         Thread.new("buffer writer") do
-          broken = false
           begin
-            while s = @outputs[output].deq
-              next  if broken
-              written = 0
-              tries = 1
-              logged_tries = 1/2
-              while !broken
-                fail @terminate  if @terminate.kind_of?(Exception)
-                begin
-                  output_io = writer_output!(output)
-                  written = output_io.write_nonblock(s)  if output  # NOTE will only be nil if @terminate is an exception
-                  break  if written == s.length  # NOTE kinda optimisation
+            output_io = writer_output!(output)
+            while s = output.q.deq  # NOTE until EOF signal
+              @stats.blocks_for output, output.q.length
+              timeouts = 0
+              logged_timeouts = 1
+              begin
+                fail @terminate  if @terminate.kind_of? Exception
+                written = output_io.write_nonblock(s)  if output_io  # NOTE will only be nil if @terminate is an exception
+                @stats.add_bytes_out written
+
+                if written != s.length  # NOTE kinda optimisation
                   s = s[written..-1]
-                rescue Errno::EAGAIN, Errno::EWOULDBLOCK
-                  if tries == 2 * logged_tries
-                    Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) retrying... (#{tries} writes): #{$!.class}"
-                    logged_tries = tries
-                  end
-                  sleep 0.01
-                rescue Errno::EPIPE
-                  broken = true
-                  Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) broken"
-                ensure
-                  tries += 1
+                  raise IO::EAGAINWaitWritable
+                end
+
+              rescue IO::WaitWritable
+                timeouts += 1
+                if timeouts > 2 * logged_timeouts
+                  Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) retrying... (#{timeouts} writes): #{$!.class}"
+                  logged_timeouts = timeouts
                 end
+                IO.select nil, [output_io], nil, ThreadedIoBuffer.io_wait_timeout
+                retry
+              rescue IO::WaitReadable  # NOTE should not really happen, so just for conformance
+                Ffmprb.logger.error "ThreadedIoBuffer writer (to #{output_io.path}) gets a #{$!} - should not really happen."
+                IO.select [output_io], nil, ThreadedIoBuffer.io_wait_timeout
+                retry
               end
             end
+            Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) breaking off"
+          rescue Errno::EPIPE
+            Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) broken"
+            output.broken = true
           ensure
             # terminated!
             begin
-              writer_output!(output).close  if !broken && writer_output!(output).respond_to?(:close)
+              writer_output!(output).close  if !output.broken && writer_output!(output).respond_to?(:close)
+              output.broken = true
             rescue
-              Ffmprb.logger.error "ThreadedIoBuffer output closing error: #{$!.message}"
+              Ffmprb.logger.error "#{$!.class.name} closing ThreadedIoBuffer output: #{$!.message}"
             end
-            Ffmprb.logger.debug "ThreadedIoBuffer writer terminated (blocks max: #{@stat_blocks_max})"
+            Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io && output_io.path}) terminated (#{@stats})"
           end
         end
       end
@@ -201,10 +251,72 @@ module Ffmprb
       #   @handler_thr = nil
       # end
 
-      def output_enq(item)
-        @outputs.values.each do |q|
-          q.enq item
+      def output_enq!(item)
+        fail AllOutputsBrokenError  if
+          @outputs.select do |output|
+            next  if output.broken
+
+            timeouts = 0
+            logged_timeouts = 1
+            begin
+              # NOTE let's assume there's no race condition here between the possible timeout exception and enq
+              Timeout.timeout(ThreadedIoBuffer.timeout) do
+                output.q.enq item
+              end
+              @stats.blocks_for output, output.q.length
+              true
+
+            rescue Timeout::Error
+              next  if output.broken
+
+              timeouts += 1
+              if timeouts == 2 * logged_timeouts
+                Ffmprb.logger.warn "A little bit of timeout (>#{timeouts*ThreadedIoBuffer.timeout}s idle) with #{ThreadedIoBuffer.blocks_max}x#{ThreadedIoBuffer.block_size}b blocks (buffering #{reader_input!.path}->...; #{@outputs.reject(&:io).size}/#{@outputs.size} unopen/total)"
+                logged_timeouts = timeouts
+              end
+
+              retry  unless timeouts >= ThreadedIoBuffer.timeout_limit # NOTE the queue has probably overflown
+
+              @terminate = Error.new("the writer has failed with timeout limit while queuing")
+              # timeout!
+              fail Error, "Looks like we're stuck (>#{ThreadedIoBuffer.timeout_limit*ThreadedIoBuffer.timeout}s idle) with #{ThreadedIoBuffer.blocks_max}x#{ThreadedIoBuffer.block_size}b blocks (buffering #{reader_input!.path}->...)..."
+            end
+        end.empty?
+      end
+
+      class Stats < OpenStruct
+        include MonitorMixin
+
+        def initialize(proc)
+          @proc = proc
+          super blocks_max: 0, bytes_in: 0, bytes_out: 0
         end
+
+        def add_bytes_in(n)
+          synchronize do
+            self.bytes_in += n
+            @proc.proc_vis_node @proc  # NOTE update
+          end
+        end
+
+        def add_bytes_out(n)
+          synchronize do
+            self.bytes_out += n
+            @proc.proc_vis_node @proc  # NOTE update
+          end
+        end
+
+        def blocks_for(outp, n)
+          synchronize do
+            if n > blocks_max
+              self.blocks_max = n
+              @proc.proc_vis_node @proc  # NOTE update
+            end
+            (@_outp_blocks ||= {})[outp] = n
+            self.blocks_buff = @_outp_blocks.values.reduce(0, :+)
+          end
+        end
+
       end
 
     end