ffmprb 0.9.6 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,8 +2,8 @@ module Ffmprb
2
2
 
3
3
  module Util
4
4
 
5
- # NOTE doesn't have specs (and not too proud about it)
6
5
  class Thread < ::Thread
6
+ include ProcVis::Node
7
7
 
8
8
  class Error < Ffmprb::Error; end
9
9
  class ParentError < Error; end
@@ -14,19 +14,19 @@ module Ffmprb
14
14
 
15
15
  def timeout_or_live(limit=nil, log: "while doing this", timeout: self.timeout, &blk)
16
16
  started_at = Time.now
17
- tries = 0
18
- logged_tries = 0
17
+ timeouts = 0
18
+ logged_timeouts = 1
19
19
  begin
20
- tries += 1
20
+ timeouts += 1
21
21
  time = Time.now - started_at
22
22
  fail TimeLimitError if limit && time > limit
23
23
  Timeout.timeout timeout do
24
24
  blk.call time
25
25
  end
26
26
  rescue Timeout::Error
27
- if tries > 2 * logged_tries
28
- Ffmprb.logger.info "A little bit of timeout #{log.respond_to?(:call)? log.call : log} (##{tries})"
29
- logged_tries = tries
27
+ if timeouts > 2 * logged_timeouts
28
+ Ffmprb.logger.info "A little bit of timeout #{log.respond_to?(:call)? log.call : log} (##{timeouts})"
29
+ logged_timeouts = timeouts
30
30
  end
31
31
  current.live!
32
32
  retry
@@ -41,7 +41,7 @@ module Ffmprb
41
41
 
42
42
  attr_reader :name
43
43
 
44
- def initialize(name="some", &blk)
44
+ def initialize(name="some", main: false, &blk)
45
45
  @name = name
46
46
  @parent = Thread.current
47
47
  @live_children = []
@@ -50,7 +50,12 @@ module Ffmprb
50
50
  Ffmprb.logger.debug "about to launch #{name}"
51
51
  sync_q = Queue.new
52
52
  super() do
53
- @parent.child_lives self if @parent.respond_to? :child_lives
53
+ @parent.proc_vis_node self if @parent.respond_to? :proc_vis_node
54
+ if @parent.respond_to? :child_lives
55
+ @parent.child_lives self
56
+ else
57
+ Ffmprb.logger.warn "Not the main: true thread run by a not #{self.class.name} thread" unless main
58
+ end
54
59
  sync_q.enq :ok
55
60
  Ffmprb.logger.debug "#{name} thread launched"
56
61
  begin
@@ -58,13 +63,14 @@ module Ffmprb
58
63
  Ffmprb.logger.debug "#{name} thread done"
59
64
  end
60
65
  rescue Exception
61
- Ffmprb.logger.warn "#{$!.class} raised in #{name} thread: #{$!.message}\nBacktrace:\n\t#{$!.backtrace.join("\n\t")}"
66
+ Ffmprb.logger.warn "#{$!.class.name} raised in #{name} thread: #{$!.message}\nBacktrace:\n\t#{$!.backtrace.join("\n\t")}"
62
67
  cause = $!
63
- Ffmprb.logger.warn "...caused by #{cause.class}: #{cause.message}\nBacktrace:\n\t#{cause.backtrace.join("\n\t")}" while
68
+ Ffmprb.logger.warn "...caused by #{cause.class.name}: #{cause.message}\nBacktrace:\n\t#{cause.backtrace.join("\n\t")}" while
64
69
  cause = cause.cause
65
70
  fail $! # XXX I have no idea why I need to give it `$!` -- the docs say I need not
66
71
  ensure
67
72
  @parent.child_dies self if @parent.respond_to? :child_dies
73
+ @parent.proc_vis_node self, :remove if @parent.respond_to? :proc_vis_node
68
74
  end
69
75
  end
70
76
  sync_q.deq
@@ -81,6 +87,7 @@ module Ffmprb
81
87
  Ffmprb.logger.debug "picking up #{thr.name} thread"
82
88
  @live_children << thr
83
89
  end
90
+ proc_vis_edge self, thr
84
91
  end
85
92
 
86
93
  def child_dies(thr)
@@ -89,9 +96,10 @@ module Ffmprb
89
96
  @dead_children_q.enq thr
90
97
  fail "System Error" unless @live_children.delete thr
91
98
  end
99
+ proc_vis_edge self, thr, :remove
92
100
  end
93
101
 
94
- def join_children!(limit=nil, timeout: self.class.timeout)
102
+ def join_children!(limit=nil, timeout: Thread.timeout)
95
103
  timeout = [timeout, limit].compact.min
96
104
  Ffmprb.logger.debug "joining threads: #{@live_children.size} live, #{@dead_children_q.size} dead"
97
105
  until @live_children.empty? && @dead_children_q.empty?
@@ -1,46 +1,52 @@
1
+ require 'ostruct'
2
+
1
3
  module Ffmprb
2
4
 
3
5
  module Util
4
6
 
5
7
  # TODO the events mechanism is currently unused (and commented out) => synchro mechanism not needed
6
- # XXX *partially* specc'ed in file_spec
7
8
  class ThreadedIoBuffer
8
- # include Synchro
9
+ # XXX include Synchro
10
+ include ProcVis::Node
9
11
 
10
12
  class << self
11
13
 
12
14
  attr_accessor :blocks_max
13
15
  attr_accessor :block_size
14
16
  attr_accessor :timeout
15
-
16
- def default_size
17
- blocks_max * block_size
18
- end
17
+ attr_accessor :timeout_limit
18
+ attr_accessor :io_wait_timeout
19
19
 
20
20
  end
21
21
 
22
+
22
23
  # NOTE input/output can be lambdas for single asynchronic io evaluation
23
- # the labdas must be timeout-interrupt-safe (since they are wrapped in timeout blocks)
24
- # NOTE both ios are being opened and closed as soon as possible
25
- def initialize(input, *outputs) # XXX SPEC ME!!! multiple outputs!!
24
+ # the lambdas must be timeout-interrupt-safe (since they are wrapped in timeout blocks)
25
+ # NOTE all ios are being opened and closed as soon as possible
26
+ def initialize(input, *outputs, keep_outputs_open_on_input_idle_limit: nil)
27
+ super() # NOTE for the monitor, apparently
28
+
29
+ Ffmprb.logger.debug "ThreadedIoBuffer initializing with (#{ThreadedIoBuffer.blocks_max}x#{ThreadedIoBuffer.block_size})"
26
30
 
27
31
  @input = input
28
- @outputs = outputs.inject({}) do |hash, out|
29
- hash[out] = SizedQueue.new(self.class.blocks_max)
30
- hash
32
+ @outputs = outputs.map do |outp|
33
+ OpenStruct.new _io: outp, q: SizedQueue.new(ThreadedIoBuffer.blocks_max)
31
34
  end
32
- @stat_blocks_max = 0
35
+ @stats = Stats.new(self)
33
36
  @terminate = false
37
+ @keep_outputs_open_on_input_idle_limit = keep_outputs_open_on_input_idle_limit
34
38
  # @events = {}
35
39
 
36
40
  Thread.new "io buffer main" do
37
41
  init_reader!
38
- outputs.each do |output|
42
+ @outputs.each do |output|
39
43
  init_writer_output! output
40
44
  init_writer! output
41
45
  end
42
46
 
43
- Thread.join_children!
47
+ Thread.join_children!.tap do
48
+ Ffmprb.logger.debug "ThreadedIoBuffer (#{@input.path}->#{@outputs.map(&:io).map(&:path)}) terminated successfully (#{@stats})"
49
+ end
44
50
  end
45
51
  end
46
52
  #
@@ -60,7 +66,7 @@ module Ffmprb
60
66
  # handle_synchronously :once
61
67
  #
62
68
  # def reader_done!
63
- # Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (blocks max: #{@stat_blocks_max})"
69
+ # Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (#{@stats})"
64
70
  # fire! :reader_done
65
71
  # end
66
72
  #
@@ -72,7 +78,7 @@ module Ffmprb
72
78
  # fire! :timeout
73
79
  # end
74
80
 
75
- protected
81
+ # protected
76
82
  #
77
83
  # def fire!(event)
78
84
  # wait_for_handler!
@@ -84,12 +90,16 @@ module Ffmprb
84
90
  # end
85
91
  # handle_synchronously :fire!
86
92
  #
87
- def blocks_count
88
- @outputs.values.map(&:size).max
93
+
94
+ def label
95
+ "IObuff: Curr/Peak/Max=#{@stats.blocks_buff}/#{@stats.blocks_max}/#{ThreadedIoBuffer.blocks_max} In/Out=#{@stats.bytes_in}/#{@stats.bytes_out}"
89
96
  end
90
97
 
91
98
  private
92
99
 
100
+ class AllOutputsBrokenError < Error
101
+ end
102
+
93
103
  def reader_input! # NOTE just for reader thread
94
104
  if @input.respond_to?(:call)
95
105
  Ffmprb.logger.debug "Opening buffer input"
@@ -99,99 +109,139 @@ module Ffmprb
99
109
  @input
100
110
  end
101
111
 
112
+ # NOTE to be called after #init_writer_output! only
102
113
  def writer_output!(output) # NOTE just for writer thread
103
- if @output_thrs[output]
104
- @output_thrs[output].join
105
- @output_thrs[output] = nil
114
+ if output.thr
115
+ output.thr.join
116
+ output.thr = nil
106
117
  end
107
- @output_ios[output]
118
+ output.io
108
119
  end
109
120
 
110
- # NOTE reads all of input, then closes the stream times out on buffer overflow
121
+ # NOTE reads roughly as much input as writers can write, then closes the stream; times out on buffer overflow
111
122
  def init_reader!
112
123
  Thread.new("buffer reader") do
113
124
  begin
114
- while s = reader_input!.read(self.class.block_size)
125
+ input_io = reader_input!
126
+ loop do
127
+ s = ''
115
128
  begin
116
- Timeout.timeout(self.class.timeout) do
117
- output_enq s
129
+ while s.length < ThreadedIoBuffer.block_size
130
+ timeouts = 0
131
+ logged_timeouts = 1
132
+ begin
133
+ ss = input_io.read_nonblock(ThreadedIoBuffer.block_size - s.length)
134
+ @stats.add_bytes_in ss.length
135
+ s += ss
136
+ rescue IO::WaitReadable
137
+ if !@terminate && @stats.bytes_in > 0 && @stats.blocks_buff == 0 && @keep_outputs_open_on_input_idle_limit && timeouts * ThreadedIoBuffer.io_wait_timeout > @keep_outputs_open_on_input_idle_limit
138
+ if s.length > 0
139
+ output_enq! s
140
+ s = '' # NOTE let's see if it helps outputting an incomplete block
141
+ else
142
+ Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) giving up after waiting >#{@keep_outputs_open_on_input_idle_limit}s, after reading #{@stats.bytes_in}b closing outputs"
143
+ @terminate = true
144
+ output_enq! nil # NOTE EOF signal
145
+ end
146
+ else
147
+ timeouts += 1
148
+ if !@terminate && timeouts > 2 * logged_timeouts
149
+ Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) retrying... (#{timeouts} reads): #{$!.class}"
150
+ logged_timeouts = timeouts
151
+ end
152
+ IO.select [input_io], nil, nil, ThreadedIoBuffer.io_wait_timeout
153
+ retry
154
+ end
155
+ rescue IO::WaitWritable # NOTE should not really happen, so just for conformance
156
+ Ffmprb.logger.error "ThreadedIoBuffer reader (from #{input_io.path}) gets a #{$!} - should not really happen."
157
+ IO.select nil, [input_io], nil, ThreadedIoBuffer.io_wait_timeout
158
+ retry
159
+ end
118
160
  end
119
- rescue Timeout::Error # NOTE the queue is probably overflown
120
- @terminate = Error.new("The reader has failed with timeout while queuing")
121
- # timeout!
122
- fail Error, "Looks like we're stuck (#{timeout}s idle) with #{self.class.blocks_max}x#{self.class.block_size}B blocks (buffering #{reader_input!.path}->...)..."
161
+ ensure
162
+ output_enq! s unless @terminate
123
163
  end
124
- @stat_blocks_max = blocks_count if blocks_count > @stat_blocks_max
125
164
  end
126
- @terminate = true
127
- output_enq nil
165
+ rescue EOFError
166
+ unless @terminate
167
+ Ffmprb.logger.debug "ThreadedIoBuffer reader (from #{input_io.path}) breaking off"
168
+ @terminate = true
169
+ output_enq! nil # NOTE EOF signal
170
+ end
171
+ rescue AllOutputsBrokenError
172
+ Ffmprb.logger.info "All outputs broken"
128
173
  ensure
129
174
  begin
130
175
  reader_input!.close if reader_input!.respond_to?(:close)
131
176
  rescue
132
- Ffmprb.logger.error "ThreadedIoBuffer input closing error: #{$!.message}"
177
+ Ffmprb.logger.error "#{$!.class.name} closing ThreadedIoBuffer input: #{$!.message}"
133
178
  end
134
179
  # reader_done!
135
- Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (blocks max: #{@stat_blocks_max})"
180
+ Ffmprb.logger.debug "ThreadedIoBuffer reader terminated (#{@stats})"
136
181
  end
137
182
  end
138
183
  end
139
184
 
140
185
  def init_writer_output!(output)
141
- @output_ios ||= {}
142
- return @output_ios[output] = output unless output.respond_to?(:call)
186
+ return output.io = output._io unless output._io.respond_to?(:call)
143
187
 
144
- @output_thrs ||= {}
145
- @output_thrs[output] = Thread.new("buffer writer output helper") do
188
+ output.thr = Thread.new("buffer writer output helper") do
146
189
  Ffmprb.logger.debug "Opening buffer output"
147
- @output_ios[output] =
148
- Thread.timeout_or_live nil, log: "in the buffer writer helper thread", timeout: self.class.timeout do |time|
149
- fail Error, "giving up buffer writer init since the reader has failed (#{@terminate.message})" if @terminate.kind_of?(Exception)
150
- output.call
190
+ output.io =
191
+ Thread.timeout_or_live nil, log: "in the buffer writer helper thread", timeout: ThreadedIoBuffer.timeout do |time|
192
+ fail Error, "giving up buffer writer init since the reader has failed (#{@terminate.message})" if @terminate.kind_of? Exception
193
+ output._io.call
151
194
  end
152
- Ffmprb.logger.debug "Opened buffer output: #{@output_ios[output].path}"
195
+ Ffmprb.logger.debug "Opened buffer output: #{output.io.path}"
153
196
  end
154
197
  end
155
198
 
156
199
  # NOTE writes as much output as possible, then terminates when the reader dies
157
200
  def init_writer!(output)
158
201
  Thread.new("buffer writer") do
159
- broken = false
160
202
  begin
161
- while s = @outputs[output].deq
162
- next if broken
163
- written = 0
164
- tries = 1
165
- logged_tries = 1/2
166
- while !broken
167
- fail @terminate if @terminate.kind_of?(Exception)
168
- begin
169
- output_io = writer_output!(output)
170
- written = output_io.write_nonblock(s) if output # NOTE will only be nil if @terminate is an exception
171
- break if written == s.length # NOTE kinda optimisation
203
+ output_io = writer_output!(output)
204
+ while s = output.q.deq # NOTE until EOF signal
205
+ @stats.blocks_for output, output.q.length
206
+ timeouts = 0
207
+ logged_timeouts = 1
208
+ begin
209
+ fail @terminate if @terminate.kind_of? Exception
210
+ written = output_io.write_nonblock(s) if output_io # NOTE will only be nil if @terminate is an exception
211
+ @stats.add_bytes_out written
212
+
213
+ if written != s.length # NOTE kinda optimisation
172
214
  s = s[written..-1]
173
- rescue Errno::EAGAIN, Errno::EWOULDBLOCK
174
- if tries == 2 * logged_tries
175
- Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) retrying... (#{tries} writes): #{$!.class}"
176
- logged_tries = tries
177
- end
178
- sleep 0.01
179
- rescue Errno::EPIPE
180
- broken = true
181
- Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) broken"
182
- ensure
183
- tries += 1
215
+ raise IO::EAGAINWaitWritable
216
+ end
217
+
218
+ rescue IO::WaitWritable
219
+ timeouts += 1
220
+ if timeouts > 2 * logged_timeouts
221
+ Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) retrying... (#{timeouts} writes): #{$!.class}"
222
+ logged_timeouts = timeouts
184
223
  end
224
+ IO.select nil, [output_io], nil, ThreadedIoBuffer.io_wait_timeout
225
+ retry
226
+ rescue IO::WaitReadable # NOTE should not really happen, so just for conformance
227
+ Ffmprb.logger.error "ThreadedIoBuffer writer (to #{output_io.path}) gets a #{$!} - should not really happen."
228
+ IO.select [output_io], nil, ThreadedIoBuffer.io_wait_timeout
229
+ retry
185
230
  end
186
231
  end
232
+ Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) breaking off"
233
+ rescue Errno::EPIPE
234
+ Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io.path}) broken"
235
+ output.broken = true
187
236
  ensure
188
237
  # terminated!
189
238
  begin
190
- writer_output!(output).close if !broken && writer_output!(output).respond_to?(:close)
239
+ writer_output!(output).close if !output.broken && writer_output!(output).respond_to?(:close)
240
+ output.broken = true
191
241
  rescue
192
- Ffmprb.logger.error "ThreadedIoBuffer output closing error: #{$!.message}"
242
+ Ffmprb.logger.error "#{$!.class.name} closing ThreadedIoBuffer output: #{$!.message}"
193
243
  end
194
- Ffmprb.logger.debug "ThreadedIoBuffer writer terminated (blocks max: #{@stat_blocks_max})"
244
+ Ffmprb.logger.debug "ThreadedIoBuffer writer (to #{output_io && output_io.path}) terminated (#{@stats})"
195
245
  end
196
246
  end
197
247
  end
@@ -201,10 +251,72 @@ module Ffmprb
201
251
  # @handler_thr = nil
202
252
  # end
203
253
 
204
- def output_enq(item)
205
- @outputs.values.each do |q|
206
- q.enq item
254
+ def output_enq!(item)
255
+ fail AllOutputsBrokenError if
256
+ @outputs.select do |output|
257
+ next if output.broken
258
+
259
+ timeouts = 0
260
+ logged_timeouts = 1
261
+ begin
262
+ # NOTE let's assume there's no race condition here between the possible timeout exception and enq
263
+ Timeout.timeout(ThreadedIoBuffer.timeout) do
264
+ output.q.enq item
265
+ end
266
+ @stats.blocks_for output, output.q.length
267
+ true
268
+
269
+ rescue Timeout::Error
270
+ next if output.broken
271
+
272
+ timeouts += 1
273
+ if timeouts == 2 * logged_timeouts
274
+ Ffmprb.logger.warn "A little bit of timeout (>#{timeouts*ThreadedIoBuffer.timeout}s idle) with #{ThreadedIoBuffer.blocks_max}x#{ThreadedIoBuffer.block_size}b blocks (buffering #{reader_input!.path}->...; #{@outputs.reject(&:io).size}/#{@outputs.size} unopen/total)"
275
+ logged_timeouts = timeouts
276
+ end
277
+
278
+ retry unless timeouts >= ThreadedIoBuffer.timeout_limit # NOTE the queue has probably overflown
279
+
280
+ @terminate = Error.new("the writer has failed with timeout limit while queuing")
281
+ # timeout!
282
+ fail Error, "Looks like we're stuck (>#{ThreadedIoBuffer.timeout_limit*ThreadedIoBuffer.timeout}s idle) with #{ThreadedIoBuffer.blocks_max}x#{ThreadedIoBuffer.block_size}b blocks (buffering #{reader_input!.path}->...)..."
283
+ end
284
+ end.empty?
285
+ end
286
+
287
+ class Stats < OpenStruct
288
+ include MonitorMixin
289
+
290
+ def initialize(proc)
291
+ @proc = proc
292
+ super blocks_max: 0, bytes_in: 0, bytes_out: 0
207
293
  end
294
+
295
+ def add_bytes_in(n)
296
+ synchronize do
297
+ self.bytes_in += n
298
+ @proc.proc_vis_node @proc # NOTE update
299
+ end
300
+ end
301
+
302
+ def add_bytes_out(n)
303
+ synchronize do
304
+ self.bytes_out += n
305
+ @proc.proc_vis_node @proc # NOTE update
306
+ end
307
+ end
308
+
309
+ def blocks_for(outp, n)
310
+ synchronize do
311
+ if n > blocks_max
312
+ self.blocks_max = n
313
+ @proc.proc_vis_node @proc # NOTE update
314
+ end
315
+ (@_outp_blocks ||= {})[outp] = n
316
+ self.blocks_buff = @_outp_blocks.values.reduce(0, :+)
317
+ end
318
+ end
319
+
208
320
  end
209
321
 
210
322
  end