staugaard-transcoding_machine 0.0.2

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Mick Staugaard
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,6 @@
+transcoding_machine===================
+
+COPYRIGHT
+=========
+
+Copyright (c) 2009 Mick Staugaard. See LICENSE for details.

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 2
+:major: 0
+:minor: 0

data/bin/transcoding_machine

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'optparse'
+
+OPTIONS = {
+  :config_file => nil
+}
+
+ARGV.options do |o|
+  script_name = File.basename($0)
+  
+  o.set_summary_indent('  ')
+  o.banner =    "Usage: #{script_name} [OPTIONS]"
+  o.define_head "Transcodes a video or audio file into multible different formats"
+  o.separator   ""
+  
+  o.on("-c [CONFIG FILE]", "The configuration file to use") do |config_file|
+    OPTIONS[:config_file] = config_file
+  end
+  
+  o.separator ""
+
+  o.on_tail("-h", "--help", "Show this help message.") { puts o; exit }
+  
+  o.parse!
+  
+  OPTIONS[:source_file] = ARGV[0]
+
+  unless OPTIONS[:config_file] && OPTIONS[:source_file]
+    puts o; exit
+  end
+  
+end
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require 'transcoding_machine'
+
+TranscodingMachine::Transcoder.options[:work_directory] = '/tmp/transcoding_machine'
+
+media_players = TranscodingMachine.load_models_from_json(File.read(OPTIONS[:config_file])).first.values
+
+transcoder = TranscodingMachine::Transcoder.new(OPTIONS[:source_file], media_players)
+transcoder.start

data/bin/transcoding_machine_ec2_server

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+require File.expand_path('../lib/transcoding_machine/server/worker', File.dirname(__FILE__))
+
+if ARGV.size >= 1
+  log = File.new(ARGV[0], "a")
+end
+log = log || STDERR
+
+begin
+  TranscodingMachine::Server::Ec2Environment.logger = log
+  TranscodingMachine::Server::Ec2Environment.load
+  
+  transcoding_machine = TranscodingMachine::Server::Worker.new(log)
+
+  # Catch interrupts
+  Signal.trap("INT") do
+    transcoding_machine.shutdown
+  end
+
+  # Run the transcoder.
+  transcoding_machine.run
+rescue
+  log.puts "error #{$!}"
+  pp "#{$@}"
+  exit 1
+end

data/lib/transcoding_machine.rb

@@ -0,0 +1,45 @@
+require 'rubygems'
+require 'activesupport'
+require 'transcoding_machine/media_format'
+require 'transcoding_machine/media_format_criterium'
+require 'transcoding_machine/media_player'
+require 'transcoding_machine/client/job_queue'
+require 'transcoding_machine/client/result_queue'
+require 'transcoding_machine/client/server_manager'
+
+module TranscodingMachine
+  module_function
+  def load_models_from_json(json_string)
+    load_models_from_hash(ActiveSupport::JSON.decode(json_string))
+  end
+  
+  def load_models_from_hash(models_hash)
+    models_hash.symbolize_keys!
+    media_formats = Hash.new
+    models_hash[:media_formats].each do |id, attributes|
+      attributes[:id] = id
+      attributes.symbolize_keys!
+      if attributes[:criteria]
+        attributes[:criteria].map! {|c| MediaFormatCriterium.new(:key => c['key'],
+                                                                 :operator => c['operator'],
+                                                                 :value => c['value'])}
+      end
+      case attributes[:type]
+      when 'video'
+        media_formats[id] = VideoFormat.new(attributes)
+      when 'audio'
+        media_formats[id] = AudioFormat.new(attributes)
+      end
+    end
+    
+    media_players = Hash.new
+    models_hash[:media_players].each do |id, attributes|
+      attributes[:id] = id
+      attributes.symbolize_keys!
+      attributes[:formats].map! {|format_id| media_formats[format_id] }
+      media_players[id] = MediaPlayer.new(attributes)
+    end
+    
+    [media_players, media_formats.values.sort {|f1, f2| f2.priority <=> f1.priority}]
+  end
+end

data/lib/transcoding_machine/client/job_queue.rb

@@ -0,0 +1,17 @@
+require 'right_aws'
+require 'transcoding_machine/server/s3_storage'
+
+module TranscodingMachine
+  module Client
+    class JobQueue
+      def initialize
+        @sqs = RightAws::SqsGen2.new
+      end
+
+      def push(queue_name, bucket, key, media_player_ids, result_queue_name)
+        msg = {:bucket => bucket, :key => key, :media_players => media_player_ids, :result_queue => result_queue_name}
+        @sqs.queue(queue_name).push(msg.to_yaml)
+      end
+    end
+  end
+end

data/lib/transcoding_machine/client/result_queue.rb

@@ -0,0 +1,47 @@
+require 'right_aws'
+
+module TranscodingMachine
+  class ResultQueue
+    def initialize
+      @sqs = RightAws::SqsGen2.new
+      @consuming = false
+    end
+
+    def start_consuming(queue_names, &block)
+      @queue_names = queue_names.compact.uniq
+      @queues = @queue_names.map {|name| @sqs.queue(name) }
+      @consuming = true
+      
+      while(@consuming)
+        @queues.map do |queue|
+          consume_queue(queue, &block)
+        end
+      end
+    end
+    
+    def consume_queue(queue, &block)
+      puts "consuming queue #{queue.name}"
+      number_of_consumed_messages = 0
+      
+      while message = queue.pop
+        consume_message(message, &block)
+        number_of_consumed_messages += 1
+      end
+      sleep(5) if number_of_consumed_messages == 0
+      number_of_consumed_messages
+    end
+    
+    def consume_message(message, &block)
+      message_properties = YAML.load(message.body)
+      puts "consuming message #{message_properties.inspect}"
+      
+      begin
+        yield(message_properties)
+      rescue Exception => e
+        
+      end
+      
+      @last_active_at = Time.now
+    end
+  end
+end

data/lib/transcoding_machine/client/server_manager.rb

@@ -0,0 +1,107 @@
+require 'right_aws'
+
+module TranscodingMachine
+  module Client
+    class ServerManager
+      
+      # Sets up a new ServerManager
+      # queue_settings are a in the following format:
+      # {'transcoding_job_queue' => {:ami => 'ami-e444444d',
+      #                              :location => 'us-east-1c',
+      #                              :key => 'my_awesome_key',
+      #                              :type => 'm1.large'}
+      # }
+      # 
+      # options are:
+      # * <tt>:sleep_time</tt> the time to sleep between queue checks (default 30)
+      # * <tt>:transcoding_settings</tt> a string or lambda with the userdata to send to new transcoding servers
+      # * <tt>:server_count</tt> a lambda returning the needed number of transcoding servers for a given queue (defaults to the queue size)
+      def initialize(queue_settings, options)
+        @sqs = RightAws::SqsGen2.new
+        @ec2 = RightAws::Ec2.new
+        
+        @queues = Hash.new
+        queue_settings.each do |queue_name, settings|
+          @queues[@sqs.queue(queue_name.to_s)] = settings
+        end
+        
+        @server_count = options[:server_count] || lambda {|queue| queue.size}
+        
+        @transcoding_settings = options[:transcoding_settings]
+        
+        @sleep_time = options[:sleep_time] || 20
+        
+        @running = false
+      end
+      
+      def needed_server_count(queue)
+        @server_count.call(queue)
+      end
+      
+      def transcoding_settings(queue)
+        "test"
+        #@transcoding_settings.respond_to?(:call) ? @transcoding_settings.call(queue) : @transcoding_settings
+      end
+      
+      def running_server_count(queue)
+        @ec2.describe_instances.find_all do |instance|
+          state = instance[:aws_state_code].to_i
+          zone = instance[:aws_availability_zone]
+          ami = instance[:aws_image_id]
+          
+          matches = state <= 16
+          matches &&= ami == ec2_ami(queue)
+          matches &&= zone == ec2_location(queue) if ec2_location(queue)
+          matches
+        end.size
+      end
+      
+      def ec2_ami(queue)
+        @queues[queue][:ami]
+      end
+
+      def ec2_location(queue)
+        @queues[queue][:location]
+      end
+      
+      def ec2_key(queue)
+        @queues[queue][:key]
+      end
+
+      def ec2_instance_type(queue)
+        @queues[queue][:type]
+      end
+      
+      def ec2_security_groups(queue)
+        @queues[queue][:security_groups]
+      end
+
+      def manage_servers(options = {})
+        @running = true
+        
+        while @running
+          @queues.keys.each do |queue|
+            needed = needed_server_count(queue)
+            running = running_server_count(queue)
+            
+            #if needed > 0 || running > 0
+              puts "#{running} of #{needed} needed servers are running for queue #{queue.name}"
+            #end
+            
+            if running < needed
+              puts "requesting #{needed - running} new servers for queue #{queue.name}"
+              puts [ec2_ami(queue), 1, needed - running, ec2_security_groups(queue),
+                                               ec2_key(queue), transcoding_settings(queue),
+                                               nil, ec2_instance_type(queue), nil, nil, ec2_location(queue)].inspect
+                                               
+              new_servers = @ec2.run_instances(ec2_ami(queue), 1, needed - running, ec2_security_groups(queue),
+                                               ec2_key(queue), transcoding_settings(queue),
+                                               nil, ec2_instance_type(queue), nil, nil, ec2_location(queue))
+            end
+          end
+          sleep(@sleep_time)
+        end
+      end
+    end
+  end
+end

data/lib/transcoding_machine/media_format.rb

@@ -0,0 +1,91 @@
+require 'transcoding_machine/media_player'
+require 'transcoding_machine/server/media_file_attributes'
+
+module TranscodingMachine
+  class MediaFormat
+    attr_reader :criteria, :priority, :id, :suffix, :mime_type, :command
+    
+    def initialize(args)
+      @fixed_criteria = []
+      @priority = args[:priority]
+      @id = args[:id]
+      @suffix = args[:suffix]
+      @mime_type = args[:mime_type]
+      @command = args[:command]
+      @criteria = args[:criteria] || []
+    end
+    
+    def matches(media_file_attributes)
+      (@fixed_criteria + @criteria).all? { |c| c.matches(media_file_attributes) }
+    end
+    
+    def can_transcode?(media_file_attributes)
+      
+    end
+    
+    def self.type_cast_attribute_value(key, value)
+      case Server::MediaFileAttributes::FIELD_TYPES[key]
+      when :boolean
+        (value.to_s.downcase == 'true' || value.to_s == '1')
+      when :string
+        value.to_s
+      when :integer
+        value.to_i
+      when :float
+        value.to_f
+      when :codec
+        value.to_sym
+      else
+        raise "unknown key (#{key}) for MediaFormat attribute"
+      end
+    end
+    
+    def self.best_match_for(media_file_attributes, sorted_formats)
+      sorted_formats.first {|f| f.can_transcode?(media_file_attributes)}
+    end
+  end
+  
+  class AudioFormat < MediaFormat
+    attr_reader :bitrate
+    def initialize(args)
+      super
+      @bitrate = args[:bitrate]
+      
+      @fixed_criteria << MediaFormatCriterium.new(:key => :video,
+                                                  :operator => :not_equals,
+                                                  :value => true)
+      
+      @fixed_criteria << MediaFormatCriterium.new(:key => :audio,
+                                                  :operator => :equals,
+                                                  :value => true)
+    end
+    
+    def can_transcode?(media_file_attributes)
+      !media_file_attributes[:video] && media_file_attributes[:audio] && media_file_attributes[:bitrate] >= @bitrate
+    end
+  end
+  
+  class VideoFormat < MediaFormat
+    attr_reader :width, :height, :aspect_ratio
+    def initialize(args)
+      super
+      @width = args[:width]
+      @height = args[:height]
+      
+      case args[:aspect_ratio]
+      when String
+        Server::MediaFileAttributes::ASPECT_RATIO_VALUES[args[:aspect_ratio]]
+      when Float
+        @aspect_ratio = args[:aspect_ratio]
+      end
+      
+      @fixed_criteria << MediaFormatCriterium.new(:key => :video,
+                                                  :operator => :equals,
+                                                  :value => true)
+    end
+
+    def can_transcode?(media_file_attributes)
+      media_file_attributes[:video] && media_file_attributes[:width] >= @width && media_file_attributes[:aspect_ratio] == @aspect_ratio
+    end
+  end
+end

data/lib/transcoding_machine/media_format_criterium.rb

@@ -0,0 +1,50 @@
+require 'transcoding_machine/media_format'
+require 'transcoding_machine/server/media_file_attributes'
+
+module TranscodingMachine
+  class MediaFormatCriterium
+    TYPE_OPERATORS = {
+      :boolean => [:equals, :not_equals],
+      :string  => [:equals, :not_equals],
+      :integer => [:equals, :not_equals, :lt, :lte, :gt, :gte],
+      :float   => [:equals, :not_equals, :lt, :lte, :gt, :gte],
+      :codec   => [:equals, :not_equals]
+    }
+
+    attr_reader :key, :operator, :value
+    def initialize(args)
+      @key = args[:key].to_sym
+      
+      @operator = (args[:operator] || :equals).to_sym
+      
+      @value = MediaFormat.type_cast_attribute_value(@key, args[:value])
+      
+      unless MediaFormatCriterium::TYPE_OPERATORS[value_type].include?(@operator)
+        raise "invalid operator (#{@operator}) for MediaFormatCriterium with key #{@key}"
+      end
+    end
+    
+    def value_type
+      Server::MediaFileAttributes::FIELD_TYPES[@key]
+    end
+    
+    def matches(media_file_attributes)
+      attr_value = MediaFormat.type_cast_attribute_value(@key, media_file_attributes[@key])
+      case @operator
+      when :equals
+        attr_value == @value
+      when :lt
+        attr_value < @value
+      when :gt
+        attr_value > @value
+      when :lte
+        attr_value <= @value
+      when :gte
+        attr_value >= @value
+      when :not_equals
+        attr_value != @value
+      end
+    end
+    
+  end
+end