ffmprb 0.9.6 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,19 +4,12 @@ module Ffmprb
4
4
 
5
5
  class Input
6
6
 
7
- def temporise!(extname=nil)
8
- extname ||= io.extname
9
- self.io = nil
10
- extend Temp
11
- @extname = extname
12
- end
13
-
14
- module Temp
15
-
16
- def io
17
- @io ||= File.temp_fifo(@extname)
7
+ def temporise_io!(extname=nil)
8
+ process.proc_vis_edge @io, process, :remove
9
+ @io.tap do
10
+ @io = File.temp_fifo(extname || io.extname)
11
+ process.proc_vis_edge @io, process
18
12
  end
19
-
20
13
  end
21
14
 
22
15
  end
@@ -8,27 +8,42 @@ module Ffmprb
8
8
 
9
9
  # XXX check for unknown options
10
10
 
11
- def video_cmd_options(video=nil)
12
- video = Process.output_video_options.merge(video.to_h || {})
13
- [].tap do |options|
14
- options.concat %W[-c:v #{video[:encoder]}] if video[:encoder]
15
- options.concat %W[-pix_fmt #{video[:pixel_format]}] if video[:pixel_format]
11
+ def video_args(video=nil)
12
+ video = Process.output_video_options.merge(video.to_h)
13
+ [].tap do |args|
14
+ encoder = pixel_format = nil # NOTE ah, ruby
15
+ args.concat %W[-c:v #{encoder}] if (encoder = video.delete(:encoder))
16
+ args.concat %W[-pix_fmt #{pixel_format}] if (pixel_format = video.delete(:pixel_format))
17
+ video.delete :resolution # NOTE is handled otherwise
18
+ video.delete :fps # NOTE is handled otherwise
19
+ fail "Unknown output video options: #{video}" unless video.empty?
16
20
  end
17
21
  end
18
22
 
19
- def audio_cmd_options(audio=nil)
20
- audio = Process.output_audio_options.merge(audio.to_h || {})
21
- [].tap do |options|
22
- options.concat %W[-c:a #{audio[:encoder]}] if audio[:encoder]
23
+ def audio_args(audio=nil)
24
+ audio = Process.output_audio_options.merge(audio.to_h)
25
+ [].tap do |args|
26
+ encoder = nil
27
+ args.concat %W[-c:a #{encoder}] if (encoder = audio.delete(:encoder))
28
+ fail "Unknown output audio options: #{audio}" unless audio.empty?
29
+ end
30
+ end
31
+
32
+ def resolve(io)
33
+ return io unless io.is_a? String # XXX XXX
34
+
35
+ File.create(io).tap do |file|
36
+ Ffmprb.logger.warn "Output file exists (#{file.path}), will probably overwrite" if file.exist?
23
37
  end
24
38
  end
25
39
 
26
40
  end
27
41
 
42
+ attr_reader :io
28
43
  attr_reader :process
29
44
 
30
45
  def initialize(io, process, video:, audio:)
31
- @io = resolve(io)
46
+ @io = self.class.resolve(io)
32
47
  @process = process
33
48
  @channels = {
34
49
  video: video && @io.channel?(:video) && OpenStruct.new(video),
@@ -179,14 +194,18 @@ module Ffmprb
179
194
 
180
195
  segments.compact!
181
196
 
182
- lbl_out = "o#{idx}o"
197
+ lbl_out = segments[0]
198
+
199
+ if segments.size > 1
200
+ lbl_out = "o#{idx}o"
183
201
 
184
- @filters.concat(
185
- Filter.concat_v segments.map{|s| "#{s}:v"}, "#{lbl_out}:v"
186
- ) if channel?(:video)
187
- @filters.concat(
188
- Filter.concat_a segments.map{|s| "#{s}:a"}, "#{lbl_out}:a"
189
- ) if channel?(:audio)
202
+ @filters.concat(
203
+ Filter.concat_v segments.map{|s| "#{s}:v"}, "#{lbl_out}:v"
204
+ ) if channel?(:video)
205
+ @filters.concat(
206
+ Filter.concat_a segments.map{|s| "#{s}:a"}, "#{lbl_out}:a"
207
+ ) if channel?(:audio)
208
+ end
190
209
 
191
210
  # Overlays
192
211
 
@@ -218,8 +237,8 @@ module Ffmprb
218
237
  # NOTE multi-process overlays last
219
238
 
220
239
  @channel_lbl_ios = {} # XXX this is a spaghetti machine
221
- @channel_lbl_ios["#{lbl_out}:v"] = @io if channel?(:video)
222
- @channel_lbl_ios["#{lbl_out}:a"] = @io if channel?(:audio)
240
+ @channel_lbl_ios["#{lbl_out}:v"] = io if channel?(:video)
241
+ @channel_lbl_ios["#{lbl_out}:a"] = io if channel?(:audio)
223
242
 
224
243
  # TODO supporting just "full" overlays for now, see exception in #add_reel
225
244
  @overlays.to_a.each_with_index do |over_reel, i|
@@ -228,44 +247,46 @@ module Ffmprb
228
247
  if over_reel.duck
229
248
  fail Error, "Don't know how to duck video... yet" if over_reel.duck != :audio
230
249
 
250
+ Ffmprb.logger.info "ATTENTION: ducking audio (due to the absence of a simple ffmpeg filter) does not support streaming main input. yet."
251
+
231
252
  # So ducking just audio here, ye?
232
253
  # XXX check if we're on audio channel
233
254
 
234
255
  main_av_o = @channel_lbl_ios["#{lbl_out}:a"]
235
256
  fail Error, "Main output does not contain audio to duck" unless main_av_o
236
- # XXX#181845 must really seperate channels for streaming (e.g. mp4 wouldn't stream through the fifo)
237
- # NOTE what really must be done here (optimisation & compatibility):
238
- # - output v&a through non-compressed pipes
239
- # - v-output will be input to the new v+a merging+encoding process
240
- # - a-output will go through the ducking process below and its output will be input to the m+e process above
241
- # - v-output will have to use another thread-buffered pipe
242
- main_av_inter_o = File.temp_fifo(main_av_o.extname)
257
+
258
+ intermediate_extname = Process.intermediate_channel_extname video: main_av_o.channel?(:video), audio: main_av_o.channel?(:audio)
259
+ main_av_inter_i, main_av_inter_o = File.threaded_buffered_fifo(intermediate_extname, reader_open_on_writer_idle_limit: Util::ThreadedIoBuffer.timeout * 2, proc_vis: process)
243
260
  @channel_lbl_ios.each do |channel_lbl, io|
244
- @channel_lbl_ios[channel_lbl] = main_av_inter_o if io == main_av_o # XXX ~~~spaghetti
261
+ @channel_lbl_ios[channel_lbl] = main_av_inter_i if io == main_av_o # XXX ~~~spaghetti
245
262
  end
246
- Ffmprb.logger.debug "Re-routed the main audio output (#{main_av_inter_o.path}->...->#{main_av_o.path}) through the process of audio ducking"
263
+ process.proc_vis_edge process, main_av_o, :remove
264
+ process.proc_vis_edge process, main_av_inter_i
265
+ Ffmprb.logger.debug "Re-routed the main audio output (#{main_av_inter_i.path}->...->#{main_av_o.path}) through the process of audio ducking"
247
266
 
248
- over_a_i, over_a_o = File.threaded_buffered_fifo(Process.intermediate_channel_extname :audio)
267
+ over_a_i, over_a_o = File.threaded_buffered_fifo(Process.intermediate_channel_extname(audio: true, video: false), proc_vis: process)
249
268
  lbl_over = "o#{idx}l#{i}"
250
269
  @filters.concat(
251
270
  over_reel.reel.filters_for lbl_over, video: false, audio: channel(:audio)
252
271
  )
253
272
  @channel_lbl_ios["#{lbl_over}:a"] = over_a_i
254
- Ffmprb.logger.debug "Routed and buffering an auxiliary output fifos (#{over_a_i.path}>#{over_a_o.path}) for overlay"
273
+ process.proc_vis_edge process, over_a_i
274
+ Ffmprb.logger.debug "Routed and buffering auxiliary output fifos (#{over_a_i.path}>#{over_a_o.path}) for overlay"
255
275
 
256
- inter_i, inter_o = File.threaded_buffered_fifo(main_av_inter_o.extname)
276
+ inter_i, inter_o = File.threaded_buffered_fifo(intermediate_extname, proc_vis: process)
257
277
  Ffmprb.logger.debug "Allocated fifos to buffer media (#{inter_i.path}>#{inter_o.path}) while finding silence"
258
278
 
259
- ignore_broken_pipe_was = process.ignore_broken_pipe
260
- process.ignore_broken_pipe = true # NOTE audio ducking process may break the overlay pipe
279
+ ignore_broken_pipes_was = process.ignore_broken_pipes # XXX maybe throw an exception instead?
280
+ process.ignore_broken_pipes = true # NOTE audio ducking process may break the overlay pipe
261
281
 
262
282
  Util::Thread.new "audio ducking" do
283
+ process.proc_vis_edge main_av_inter_o, inter_i # XXX mark it better
263
284
  silence = Ffmprb.find_silence(main_av_inter_o, inter_i)
264
285
 
265
286
  Ffmprb.logger.debug "Audio ducking with silence: [#{silence.map{|s| "#{s.start_at}-#{s.end_at}"}.join ', '}]"
266
287
 
267
288
  Process.duck_audio inter_o, over_a_o, silence, main_av_o,
268
- process_options: {ignore_broken_pipe: ignore_broken_pipe_was, timeout: process.timeout},
289
+ process_options: {parent: process, ignore_broken_pipes: ignore_broken_pipes_was, timeout: process.timeout},
269
290
  video: channel(:video), audio: channel(:audio)
270
291
  end
271
292
  end
@@ -275,25 +296,23 @@ module Ffmprb
275
296
  @filters
276
297
  end
277
298
 
278
- def options
299
+ def args
279
300
  fail Error, "Must generate filters first." unless @channel_lbl_ios
280
301
 
281
- options = []
282
-
283
- io_channel_lbls = {} # XXX ~~~spaghetti
284
- @channel_lbl_ios.each do |channel_lbl, io|
285
- (io_channel_lbls[io] ||= []) << channel_lbl
286
- end
287
- io_channel_lbls.each do |io, channel_lbls|
288
- channel_lbls.each do |channel_lbl|
289
- options << '-map' << "[#{channel_lbl}]"
302
+ [].tap do |args|
303
+ io_channel_lbls = {} # XXX ~~~spaghetti
304
+ @channel_lbl_ios.each do |channel_lbl, io|
305
+ (io_channel_lbls[io] ||= []) << channel_lbl
306
+ end
307
+ io_channel_lbls.each do |io, channel_lbls|
308
+ channel_lbls.each do |channel_lbl|
309
+ args.concat ['-map', "[#{channel_lbl}]"]
310
+ end
311
+ args.concat self.class.video_args(channel :video) if channel? :video
312
+ args.concat self.class.audio_args(channel :audio) if channel? :audio
313
+ args << io.path
290
314
  end
291
- options.concat self.class.video_cmd_options(channel :video) if channel? :video
292
- options.concat self.class.audio_cmd_options(channel :audio) if channel? :audio
293
- options << io.path
294
315
  end
295
-
296
- options
297
316
  end
298
317
 
299
318
  def roll(
@@ -330,21 +349,6 @@ module Ffmprb
330
349
  !!channel(medium)
331
350
  end
332
351
 
333
- protected
334
-
335
- def resolve(io)
336
- return io unless io.is_a? String
337
-
338
- case io
339
- when /^\/\w/
340
- File.create(io).tap do |file|
341
- Ffmprb.logger.warn "Output file exists (#{file.path}), will probably overwrite" if file.exist?
342
- end
343
- else
344
- fail Error, "Cannot resolve output: #{io}"
345
- end
346
- end
347
-
348
352
  private
349
353
 
350
354
  def reels_channel?(medium)
data/lib/ffmprb/util.rb CHANGED
@@ -1,51 +1,51 @@
1
- # require 'ffmprb/util/synchro'
2
- require 'ffmprb/util/thread'
3
- require 'ffmprb/util/threaded_io_buffer'
4
-
5
1
  require 'open3'
6
2
 
7
3
  module Ffmprb
8
4
 
5
+ class Error < StandardError; end
6
+
9
7
  module Util
10
8
 
11
9
  class TimeLimitError < Error; end
12
10
 
13
11
  class << self
14
12
 
15
- attr_accessor :ffmpeg_cmd, :ffprobe_cmd
13
+ attr_accessor :ffmpeg_cmd, :ffmpeg_inputs_max, :ffprobe_cmd
16
14
  attr_accessor :cmd_timeout
17
15
 
18
16
  def ffprobe(*args, limit: nil, timeout: cmd_timeout)
19
17
  sh *ffprobe_cmd, *args, limit: limit, timeout: timeout
20
18
  end
21
19
 
22
- def ffmpeg(*args, limit: nil, timeout: cmd_timeout, ignore_broken_pipe: false)
23
- args = ['-loglevel', 'debug'] + args if Ffmprb.debug
24
- sh *ffmpeg_cmd, *args, output: :stderr, limit: limit, timeout: timeout, ignore_broken_pipe: ignore_broken_pipe
20
+ def ffmpeg(*args, limit: nil, timeout: cmd_timeout, ignore_broken_pipes: true)
21
+ args = ['-loglevel', 'debug'] + args if Ffmprb.ffmpeg_debug
22
+ sh *ffmpeg_cmd, *args, output: :stderr, limit: limit, timeout: timeout, ignore_broken_pipes: ignore_broken_pipes
25
23
  end
26
24
 
27
- def sh(*cmd, output: :stdout, log: :stderr, limit: nil, timeout: cmd_timeout, ignore_broken_pipe: false)
25
+ def sh(*cmd, input: nil, output: :stdout, limit: nil, timeout: cmd_timeout, ignore_broken_pipes: false)
28
26
  cmd = cmd.map &:to_s unless cmd.size == 1
29
- cmd_str = cmd.size != 1 ? cmd.map{|c| "\"#{c}\""}.join(' ') : cmd.first
27
+ cmd_str = cmd.size != 1 ? cmd.map{|c| sh_escape c}.join(' ') : cmd.first
30
28
  timeout = [timeout, limit].compact.min
31
29
  thr = Thread.new "`#{cmd_str}`" do
32
30
  Ffmprb.logger.info "Popening `#{cmd_str}`..."
33
31
  Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
34
32
  begin
33
+ stdin.write input if input
35
34
  stdin.close
36
35
 
37
- log_cmd = cmd.first.upcase if log
38
- stdout_r = Reader.new(stdout, output == :stdout, log == :stdout && log_cmd)
39
- stderr_r = Reader.new(stderr, true, log == :stderr && log_cmd)
36
+ log_cmd = cmd.first.upcase
37
+ stdout_r = Reader.new(stdout, store: output == :stdout, log_with: log_cmd)
38
+ stderr_r = Reader.new(stderr, store: true, log_with: log_cmd, log_as: output == :stderr && Logger::DEBUG || Logger::INFO)
40
39
 
41
40
  Thread.timeout_or_live(limit, log: "while waiting for `#{cmd_str}`", timeout: timeout) do |time|
42
41
  value = wait_thr.value
43
42
  status = value.exitstatus # NOTE blocking
44
43
  if status != 0
45
- if ignore_broken_pipe && value.signaled? && value.termsig == Signal.list['PIPE']
46
- Ffmprb.logger.debug "Ignoring broken pipe: #{cmd_str}"
44
+ if ignore_broken_pipes && value.signaled? && value.termsig == Signal.list['PIPE']
45
+ Ffmprb.logger.info "Ignoring broken pipe: #{cmd_str}"
47
46
  else
48
- fail Error, "#{cmd_str} (#{status || "sig##{value.termsig}"}):\n#{stderr_r.read}"
47
+ status ||= "sig##{value.termsig}"
48
+ fail Error, "#{cmd_str} (#{status}):\n#{stderr_r.read}"
49
49
  end
50
50
  end
51
51
  end
@@ -65,6 +65,15 @@ module Ffmprb
65
65
 
66
66
  protected
67
67
 
68
+ # NOTE a best guess kinda method
69
+ def sh_escape(str)
70
+ if str !~ /^[a-z0-9\/.:_-]*$/i && str !~ /"/
71
+ "\"#{str}\""
72
+ else
73
+ str
74
+ end
75
+ end
76
+
68
77
  def process_dead!(wait_thr, cmd_str, limit)
69
78
  grace = limit ? limit/4 : 1
70
79
  return unless wait_thr.alive?
@@ -97,13 +106,13 @@ module Ffmprb
97
106
 
98
107
  class Reader < Thread
99
108
 
100
- def initialize(input, store=false, log=nil)
109
+ def initialize(input, store: false, log_with: nil, log_as: Logger::DEBUG)
101
110
  @output = ''
102
111
  @queue = Queue.new
103
112
  super "reader" do
104
113
  begin
105
114
  while s = input.gets
106
- Ffmprb.logger.debug "#{log}: #{s.chomp}" if log
115
+ Ffmprb.logger.log log_as, "#{log_with}: #{s.chomp}" if log_with
107
116
  @output << s if store
108
117
  end
109
118
  @queue.enq @output
@@ -129,3 +138,8 @@ module Ffmprb
129
138
  end
130
139
 
131
140
  end
141
+
142
+ # require 'ffmprb/util/synchro'
143
+ require_relative 'util/proc_vis'
144
+ require_relative 'util/thread'
145
+ require_relative 'util/threaded_io_buffer'
@@ -0,0 +1,163 @@
1
+ require 'set'
2
+ require 'monitor'
3
+
4
+ module Ffmprb
5
+
6
+ module Util
7
+
8
+ module ProcVis
9
+
10
+ UPDATE_PERIOD_SEC = 1
11
+
12
+ module Node
13
+
14
+ attr_accessor :_proc_vis
15
+
16
+ def proc_vis_name
17
+ lbl = respond_to?(:label) && label ||
18
+ short_name ||
19
+ to_s
20
+ # ).gsub(/\W+/, '_').sub(/^[^[:alpha:]]*/, '')
21
+ "#{object_id} [labelType=\"html\" label=#{lbl.to_json}]"
22
+ end
23
+
24
+ def proc_vis_node(node, op=:upsert)
25
+ _proc_vis.proc_vis_node node, op if _proc_vis
26
+ end
27
+
28
+ def proc_vis_edge(from, to, op=:upsert)
29
+ _proc_vis.proc_vis_edge from, to, op if _proc_vis
30
+ end
31
+
32
+ private
33
+
34
+ def short_name
35
+ return unless respond_to? :name
36
+
37
+ short =
38
+ if name.length <= 30
39
+ name
40
+ else
41
+ "#{name[0..13]}..#{name[-14..-1]}"
42
+ end
43
+ "#{self.class.name.split('::').last}: #{short}"
44
+ end
45
+
46
+ end
47
+
48
+ module ClassMethods
49
+
50
+ attr_accessor :proc_vis_firebase
51
+
52
+ def proc_vis_node(obj, op=:upsert)
53
+ return unless proc_vis_init?
54
+ fail Error, "Must be a #{Node.name}" unless obj.kind_of? Node # XXX duck typing FTW
55
+
56
+ obj._proc_vis = self
57
+ obj.proc_vis_name.tap do |lbl|
58
+ proc_vis_sync do
59
+ @_proc_vis_nodes ||= {}
60
+ if op == :remove
61
+ @_proc_vis_nodes.delete obj
62
+ else
63
+ @_proc_vis_nodes[obj] = lbl
64
+ end
65
+ end
66
+ proc_vis_update # XXX optimise
67
+ end
68
+ end
69
+
70
+ def proc_vis_edge(from, to, op=:upsert)
71
+ return unless proc_vis_init?
72
+
73
+ if op == :upsert
74
+ proc_vis_node from
75
+ proc_vis_node to
76
+ end
77
+ "#{from.object_id} -> #{to.object_id}".tap do |edge|
78
+ proc_vis_sync do
79
+ @_proc_vis_edges ||= SortedSet.new
80
+ if op == :remove
81
+ @_proc_vis_edges.delete edge
82
+ else
83
+ @_proc_vis_edges << edge
84
+ end
85
+ end
86
+ proc_vis_update
87
+ end
88
+ end
89
+
90
+ private
91
+
92
+ def proc_vis_update
93
+ @_proc_vis_upq.enq 1
94
+ end
95
+
96
+
97
+ def proc_vis_do_update
98
+ nodes = @_proc_vis_nodes.map{ |_, node| "#{node};"}.join("\n") if @_proc_vis_nodes
99
+ edges = @_proc_vis_edges.map{ |edge| "#{edge};"}.join("\n") if @_proc_vis_edges
100
+ proc_vis_firebase_client.set proc_vis_pid, dot: [*nodes, *edges].join("\n")
101
+ end
102
+
103
+ def proc_vis_pid
104
+ @proc_vis_pid ||= object_id.tap do |pid|
105
+ Ffmprb.logger.info "You may view your process visualised at: https://#{proc_vis_firebase}.firebaseapp.com/?pid=#{pid}"
106
+ end
107
+ end
108
+
109
+ def proc_vis_init?
110
+ !!proc_vis_firebase_client
111
+ end
112
+
113
+ def proc_vis_up_init
114
+ @_proc_vis_thr ||= Thread.new do # NOTE update throttling
115
+ prev_t = Time.now
116
+ while @_proc_vis_upq.deq # NOTE currently, runs forever (nil terminator needed)
117
+ proc_vis_do_update
118
+ while Time.now - prev_t < UPDATE_PERIOD_SEC
119
+ @_proc_vis_upq.deq # NOTE drains the queue
120
+ end
121
+ @_proc_vis_upq.enq 1
122
+ end
123
+ end
124
+ end
125
+
126
+ def proc_vis_sync_init
127
+ @_proc_vis_mon ||= Monitor.new
128
+ @_proc_vis_upq ||= Queue.new
129
+ end
130
+ def proc_vis_sync(&blk)
131
+ @_proc_vis_mon.synchronize &blk if blk
132
+ end
133
+
134
+ def proc_vis_firebase_client
135
+ return @proc_vis_firebase_client if defined? @proc_vis_firebase_client
136
+ @proc_vis_firebase_client =
137
+ if proc_vis_firebase
138
+ url = "https://#{proc_vis_firebase}.firebaseio.com/proc/"
139
+ Ffmprb.logger.debug "Connecting to #{url}"
140
+ begin
141
+ Firebase::Client.new(url).tap do
142
+ Ffmprb.logger.info "Connected to #{url}"
143
+ proc_vis_up_init
144
+ end
145
+ rescue
146
+ Ffmprb.logger.error "Could not connect to #{url}"
147
+ end
148
+ end
149
+ end
150
+
151
+ end
152
+
153
+ def self.included(klass)
154
+ klass.extend ClassMethods
155
+ klass.send :proc_vis_sync_init
156
+ end
157
+
158
+
159
+ end
160
+
161
+ end
162
+
163
+ end