staugaard-transcoding_machine 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +20 -0
- data/README +6 -0
- data/VERSION.yml +4 -0
- data/bin/transcoding_machine +43 -0
- data/bin/transcoding_machine_ec2_server +26 -0
- data/lib/transcoding_machine.rb +45 -0
- data/lib/transcoding_machine/client/job_queue.rb +17 -0
- data/lib/transcoding_machine/client/result_queue.rb +47 -0
- data/lib/transcoding_machine/client/server_manager.rb +107 -0
- data/lib/transcoding_machine/media_format.rb +91 -0
- data/lib/transcoding_machine/media_format_criterium.rb +50 -0
- data/lib/transcoding_machine/media_player.rb +17 -0
- data/lib/transcoding_machine/server.rb +6 -0
- data/lib/transcoding_machine/server/ec2_environment.rb +79 -0
- data/lib/transcoding_machine/server/file_storage.rb +23 -0
- data/lib/transcoding_machine/server/media_file_attributes.rb +582 -0
- data/lib/transcoding_machine/server/s3_storage.rb +35 -0
- data/lib/transcoding_machine/server/transcoder.rb +157 -0
- data/lib/transcoding_machine/server/transcoding_event_listener.rb +59 -0
- data/lib/transcoding_machine/server/worker.rb +137 -0
- data/test/deserialze_test.rb +7 -0
- data/test/fixtures/serialized_models.json +52 -0
- data/test/media_format_criterium_test.rb +55 -0
- data/test/media_format_test.rb +42 -0
- data/test/media_player_test.rb +7 -0
- data/test/test_helper.rb +9 -0
- metadata +106 -0
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2008 Mick Staugaard
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README
ADDED
data/VERSION.yml
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'rubygems'
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
OPTIONS = {
|
6
|
+
:config_file => nil
|
7
|
+
}
|
8
|
+
|
9
|
+
ARGV.options do |o|
|
10
|
+
script_name = File.basename($0)
|
11
|
+
|
12
|
+
o.set_summary_indent(' ')
|
13
|
+
o.banner = "Usage: #{script_name} [OPTIONS]"
|
14
|
+
o.define_head "Transcodes a video or audio file into multible different formats"
|
15
|
+
o.separator ""
|
16
|
+
|
17
|
+
o.on("-c [CONFIG FILE]", "The configuration file to use") do |config_file|
|
18
|
+
OPTIONS[:config_file] = config_file
|
19
|
+
end
|
20
|
+
|
21
|
+
o.separator ""
|
22
|
+
|
23
|
+
o.on_tail("-h", "--help", "Show this help message.") { puts o; exit }
|
24
|
+
|
25
|
+
o.parse!
|
26
|
+
|
27
|
+
OPTIONS[:source_file] = ARGV[0]
|
28
|
+
|
29
|
+
unless OPTIONS[:config_file] && OPTIONS[:source_file]
|
30
|
+
puts o; exit
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
|
36
|
+
require 'transcoding_machine'
|
37
|
+
|
38
|
+
TranscodingMachine::Transcoder.options[:work_directory] = '/tmp/transcoding_machine'
|
39
|
+
|
40
|
+
media_players = TranscodingMachine.load_models_from_json(File.read(OPTIONS[:config_file])).first.values
|
41
|
+
|
42
|
+
transcoder = TranscodingMachine::Transcoder.new(OPTIONS[:source_file], media_players)
|
43
|
+
transcoder.start
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require File.expand_path('../lib/transcoding_machine/server/worker', File.dirname(__FILE__))
|
3
|
+
|
4
|
+
if ARGV.size >= 1
|
5
|
+
log = File.new(ARGV[0], "a")
|
6
|
+
end
|
7
|
+
log = log || STDERR
|
8
|
+
|
9
|
+
begin
|
10
|
+
TranscodingMachine::Server::Ec2Environment.logger = log
|
11
|
+
TranscodingMachine::Server::Ec2Environment.load
|
12
|
+
|
13
|
+
transcoding_machine = TranscodingMachine::Server::Worker.new(log)
|
14
|
+
|
15
|
+
# Catch interrupts
|
16
|
+
Signal.trap("INT") do
|
17
|
+
transcoding_machine.shutdown
|
18
|
+
end
|
19
|
+
|
20
|
+
# Run the transcoder.
|
21
|
+
transcoding_machine.run
|
22
|
+
rescue
|
23
|
+
log.puts "error #{$!}"
|
24
|
+
pp "#{$@}"
|
25
|
+
exit 1
|
26
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'activesupport'
|
3
|
+
require 'transcoding_machine/media_format'
|
4
|
+
require 'transcoding_machine/media_format_criterium'
|
5
|
+
require 'transcoding_machine/media_player'
|
6
|
+
require 'transcoding_machine/client/job_queue'
|
7
|
+
require 'transcoding_machine/client/result_queue'
|
8
|
+
require 'transcoding_machine/client/server_manager'
|
9
|
+
|
10
|
+
module TranscodingMachine
|
11
|
+
module_function
|
12
|
+
def load_models_from_json(json_string)
|
13
|
+
load_models_from_hash(ActiveSupport::JSON.decode(json_string))
|
14
|
+
end
|
15
|
+
|
16
|
+
def load_models_from_hash(models_hash)
|
17
|
+
models_hash.symbolize_keys!
|
18
|
+
media_formats = Hash.new
|
19
|
+
models_hash[:media_formats].each do |id, attributes|
|
20
|
+
attributes[:id] = id
|
21
|
+
attributes.symbolize_keys!
|
22
|
+
if attributes[:criteria]
|
23
|
+
attributes[:criteria].map! {|c| MediaFormatCriterium.new(:key => c['key'],
|
24
|
+
:operator => c['operator'],
|
25
|
+
:value => c['value'])}
|
26
|
+
end
|
27
|
+
case attributes[:type]
|
28
|
+
when 'video'
|
29
|
+
media_formats[id] = VideoFormat.new(attributes)
|
30
|
+
when 'audio'
|
31
|
+
media_formats[id] = AudioFormat.new(attributes)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
media_players = Hash.new
|
36
|
+
models_hash[:media_players].each do |id, attributes|
|
37
|
+
attributes[:id] = id
|
38
|
+
attributes.symbolize_keys!
|
39
|
+
attributes[:formats].map! {|format_id| media_formats[format_id] }
|
40
|
+
media_players[id] = MediaPlayer.new(attributes)
|
41
|
+
end
|
42
|
+
|
43
|
+
[media_players, media_formats.values.sort {|f1, f2| f2.priority <=> f1.priority}]
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'right_aws'
|
2
|
+
require 'transcoding_machine/server/s3_storage'
|
3
|
+
|
4
|
+
module TranscodingMachine
|
5
|
+
module Client
|
6
|
+
class JobQueue
|
7
|
+
def initialize
|
8
|
+
@sqs = RightAws::SqsGen2.new
|
9
|
+
end
|
10
|
+
|
11
|
+
def push(queue_name, bucket, key, media_player_ids, result_queue_name)
|
12
|
+
msg = {:bucket => bucket, :key => key, :media_players => media_player_ids, :result_queue => result_queue_name}
|
13
|
+
@sqs.queue(queue_name).push(msg.to_yaml)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'right_aws'
|
2
|
+
|
3
|
+
module TranscodingMachine
|
4
|
+
class ResultQueue
|
5
|
+
def initialize
|
6
|
+
@sqs = RightAws::SqsGen2.new
|
7
|
+
@consuming = false
|
8
|
+
end
|
9
|
+
|
10
|
+
def start_consuming(queue_names, &block)
|
11
|
+
@queue_names = queue_names.compact.uniq
|
12
|
+
@queues = @queue_names.map {|name| @sqs.queue(name) }
|
13
|
+
@consuming = true
|
14
|
+
|
15
|
+
while(@consuming)
|
16
|
+
@queues.map do |queue|
|
17
|
+
consume_queue(queue, &block)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def consume_queue(queue, &block)
|
23
|
+
puts "consuming queue #{queue.name}"
|
24
|
+
number_of_consumed_messages = 0
|
25
|
+
|
26
|
+
while message = queue.pop
|
27
|
+
consume_message(message, &block)
|
28
|
+
number_of_consumed_messages += 1
|
29
|
+
end
|
30
|
+
sleep(5) if number_of_consumed_messages == 0
|
31
|
+
number_of_consumed_messages
|
32
|
+
end
|
33
|
+
|
34
|
+
def consume_message(message, &block)
|
35
|
+
message_properties = YAML.load(message.body)
|
36
|
+
puts "consuming message #{message_properties.inspect}"
|
37
|
+
|
38
|
+
begin
|
39
|
+
yield(message_properties)
|
40
|
+
rescue Exception => e
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
@last_active_at = Time.now
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'right_aws'
|
2
|
+
|
3
|
+
module TranscodingMachine
|
4
|
+
module Client
|
5
|
+
class ServerManager
|
6
|
+
|
7
|
+
# Sets up a new ServerManager
|
8
|
+
# queue_settings are a in the following format:
|
9
|
+
# {'transcoding_job_queue' => {:ami => 'ami-e444444d',
|
10
|
+
# :location => 'us-east-1c',
|
11
|
+
# :key => 'my_awesome_key',
|
12
|
+
# :type => 'm1.large'}
|
13
|
+
# }
|
14
|
+
#
|
15
|
+
# options are:
|
16
|
+
# * <tt>:sleep_time</tt> the time to sleep between queue checks (default 30)
|
17
|
+
# * <tt>:transcoding_settings</tt> a string or lambda with the userdata to send to new transcoding servers
|
18
|
+
# * <tt>:server_count</tt> a lambda returning the needed number of transcoding servers for a given queue (defaults to the queue size)
|
19
|
+
def initialize(queue_settings, options)
|
20
|
+
@sqs = RightAws::SqsGen2.new
|
21
|
+
@ec2 = RightAws::Ec2.new
|
22
|
+
|
23
|
+
@queues = Hash.new
|
24
|
+
queue_settings.each do |queue_name, settings|
|
25
|
+
@queues[@sqs.queue(queue_name.to_s)] = settings
|
26
|
+
end
|
27
|
+
|
28
|
+
@server_count = options[:server_count] || lambda {|queue| queue.size}
|
29
|
+
|
30
|
+
@transcoding_settings = options[:transcoding_settings]
|
31
|
+
|
32
|
+
@sleep_time = options[:sleep_time] || 20
|
33
|
+
|
34
|
+
@running = false
|
35
|
+
end
|
36
|
+
|
37
|
+
def needed_server_count(queue)
|
38
|
+
@server_count.call(queue)
|
39
|
+
end
|
40
|
+
|
41
|
+
def transcoding_settings(queue)
|
42
|
+
"test"
|
43
|
+
#@transcoding_settings.respond_to?(:call) ? @transcoding_settings.call(queue) : @transcoding_settings
|
44
|
+
end
|
45
|
+
|
46
|
+
def running_server_count(queue)
|
47
|
+
@ec2.describe_instances.find_all do |instance|
|
48
|
+
state = instance[:aws_state_code].to_i
|
49
|
+
zone = instance[:aws_availability_zone]
|
50
|
+
ami = instance[:aws_image_id]
|
51
|
+
|
52
|
+
matches = state <= 16
|
53
|
+
matches &&= ami == ec2_ami(queue)
|
54
|
+
matches &&= zone == ec2_location(queue) if ec2_location(queue)
|
55
|
+
matches
|
56
|
+
end.size
|
57
|
+
end
|
58
|
+
|
59
|
+
def ec2_ami(queue)
|
60
|
+
@queues[queue][:ami]
|
61
|
+
end
|
62
|
+
|
63
|
+
def ec2_location(queue)
|
64
|
+
@queues[queue][:location]
|
65
|
+
end
|
66
|
+
|
67
|
+
def ec2_key(queue)
|
68
|
+
@queues[queue][:key]
|
69
|
+
end
|
70
|
+
|
71
|
+
def ec2_instance_type(queue)
|
72
|
+
@queues[queue][:type]
|
73
|
+
end
|
74
|
+
|
75
|
+
def ec2_security_groups(queue)
|
76
|
+
@queues[queue][:security_groups]
|
77
|
+
end
|
78
|
+
|
79
|
+
def manage_servers(options = {})
|
80
|
+
@running = true
|
81
|
+
|
82
|
+
while @running
|
83
|
+
@queues.keys.each do |queue|
|
84
|
+
needed = needed_server_count(queue)
|
85
|
+
running = running_server_count(queue)
|
86
|
+
|
87
|
+
#if needed > 0 || running > 0
|
88
|
+
puts "#{running} of #{needed} needed servers are running for queue #{queue.name}"
|
89
|
+
#end
|
90
|
+
|
91
|
+
if running < needed
|
92
|
+
puts "requesting #{needed - running} new servers for queue #{queue.name}"
|
93
|
+
puts [ec2_ami(queue), 1, needed - running, ec2_security_groups(queue),
|
94
|
+
ec2_key(queue), transcoding_settings(queue),
|
95
|
+
nil, ec2_instance_type(queue), nil, nil, ec2_location(queue)].inspect
|
96
|
+
|
97
|
+
new_servers = @ec2.run_instances(ec2_ami(queue), 1, needed - running, ec2_security_groups(queue),
|
98
|
+
ec2_key(queue), transcoding_settings(queue),
|
99
|
+
nil, ec2_instance_type(queue), nil, nil, ec2_location(queue))
|
100
|
+
end
|
101
|
+
end
|
102
|
+
sleep(@sleep_time)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'transcoding_machine/media_player'
|
2
|
+
require 'transcoding_machine/server/media_file_attributes'
|
3
|
+
|
4
|
+
module TranscodingMachine
|
5
|
+
class MediaFormat
|
6
|
+
attr_reader :criteria, :priority, :id, :suffix, :mime_type, :command
|
7
|
+
|
8
|
+
def initialize(args)
|
9
|
+
@fixed_criteria = []
|
10
|
+
@priority = args[:priority]
|
11
|
+
@id = args[:id]
|
12
|
+
@suffix = args[:suffix]
|
13
|
+
@mime_type = args[:mime_type]
|
14
|
+
@command = args[:command]
|
15
|
+
@criteria = args[:criteria] || []
|
16
|
+
end
|
17
|
+
|
18
|
+
def matches(media_file_attributes)
|
19
|
+
(@fixed_criteria + @criteria).all? { |c| c.matches(media_file_attributes) }
|
20
|
+
end
|
21
|
+
|
22
|
+
def can_transcode?(media_file_attributes)
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.type_cast_attribute_value(key, value)
|
27
|
+
case Server::MediaFileAttributes::FIELD_TYPES[key]
|
28
|
+
when :boolean
|
29
|
+
(value.to_s.downcase == 'true' || value.to_s == '1')
|
30
|
+
when :string
|
31
|
+
value.to_s
|
32
|
+
when :integer
|
33
|
+
value.to_i
|
34
|
+
when :float
|
35
|
+
value.to_f
|
36
|
+
when :codec
|
37
|
+
value.to_sym
|
38
|
+
else
|
39
|
+
raise "unknown key (#{key}) for MediaFormat attribute"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.best_match_for(media_file_attributes, sorted_formats)
|
44
|
+
sorted_formats.first {|f| f.can_transcode?(media_file_attributes)}
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
class AudioFormat < MediaFormat
|
49
|
+
attr_reader :bitrate
|
50
|
+
def initialize(args)
|
51
|
+
super
|
52
|
+
@bitrate = args[:bitrate]
|
53
|
+
|
54
|
+
@fixed_criteria << MediaFormatCriterium.new(:key => :video,
|
55
|
+
:operator => :not_equals,
|
56
|
+
:value => true)
|
57
|
+
|
58
|
+
@fixed_criteria << MediaFormatCriterium.new(:key => :audio,
|
59
|
+
:operator => :equals,
|
60
|
+
:value => true)
|
61
|
+
end
|
62
|
+
|
63
|
+
def can_transcode?(media_file_attributes)
|
64
|
+
!media_file_attributes[:video] && media_file_attributes[:audio] && media_file_attributes[:bitrate] >= @bitrate
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class VideoFormat < MediaFormat
|
69
|
+
attr_reader :width, :height, :aspect_ratio
|
70
|
+
def initialize(args)
|
71
|
+
super
|
72
|
+
@width = args[:width]
|
73
|
+
@height = args[:height]
|
74
|
+
|
75
|
+
case args[:aspect_ratio]
|
76
|
+
when String
|
77
|
+
Server::MediaFileAttributes::ASPECT_RATIO_VALUES[args[:aspect_ratio]]
|
78
|
+
when Float
|
79
|
+
@aspect_ratio = args[:aspect_ratio]
|
80
|
+
end
|
81
|
+
|
82
|
+
@fixed_criteria << MediaFormatCriterium.new(:key => :video,
|
83
|
+
:operator => :equals,
|
84
|
+
:value => true)
|
85
|
+
end
|
86
|
+
|
87
|
+
def can_transcode?(media_file_attributes)
|
88
|
+
media_file_attributes[:video] && media_file_attributes[:width] >= @width && media_file_attributes[:aspect_ratio] == @aspect_ratio
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'transcoding_machine/media_format'
|
2
|
+
require 'transcoding_machine/server/media_file_attributes'
|
3
|
+
|
4
|
+
module TranscodingMachine
|
5
|
+
class MediaFormatCriterium
|
6
|
+
TYPE_OPERATORS = {
|
7
|
+
:boolean => [:equals, :not_equals],
|
8
|
+
:string => [:equals, :not_equals],
|
9
|
+
:integer => [:equals, :not_equals, :lt, :lte, :gt, :gte],
|
10
|
+
:float => [:equals, :not_equals, :lt, :lte, :gt, :gte],
|
11
|
+
:codec => [:equals, :not_equals]
|
12
|
+
}
|
13
|
+
|
14
|
+
attr_reader :key, :operator, :value
|
15
|
+
def initialize(args)
|
16
|
+
@key = args[:key].to_sym
|
17
|
+
|
18
|
+
@operator = (args[:operator] || :equals).to_sym
|
19
|
+
|
20
|
+
@value = MediaFormat.type_cast_attribute_value(@key, args[:value])
|
21
|
+
|
22
|
+
unless MediaFormatCriterium::TYPE_OPERATORS[value_type].include?(@operator)
|
23
|
+
raise "invalid operator (#{@operator}) for MediaFormatCriterium with key #{@key}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def value_type
|
28
|
+
Server::MediaFileAttributes::FIELD_TYPES[@key]
|
29
|
+
end
|
30
|
+
|
31
|
+
def matches(media_file_attributes)
|
32
|
+
attr_value = MediaFormat.type_cast_attribute_value(@key, media_file_attributes[@key])
|
33
|
+
case @operator
|
34
|
+
when :equals
|
35
|
+
attr_value == @value
|
36
|
+
when :lt
|
37
|
+
attr_value < @value
|
38
|
+
when :gt
|
39
|
+
attr_value > @value
|
40
|
+
when :lte
|
41
|
+
attr_value <= @value
|
42
|
+
when :gte
|
43
|
+
attr_value >= @value
|
44
|
+
when :not_equals
|
45
|
+
attr_value != @value
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|