staugaard-transcoding_machine 0.0.2

data/lib/transcoding_machine/server/s3_storage.rb

@@ -0,0 +1,35 @@
+require 'right_aws'
+
+module TranscodingMachine
+  module Server
+    class S3Storage
+      def initialize
+        @s3 = RightAws::S3.new(nil, nil, :server => 's3.amazonaws.com', :port => 80, :protocol => 'http')
+      end
+
+      def get_file(key_name, destination_file_path, options)
+        file = File.new(destination_file_path, File::CREAT|File::RDWR)
+        rhdr = @s3.interface.get(options[:bucket], key_name) do |chunk|
+          file.write(chunk)
+        end
+        file.close
+      end
+
+      def put_file(source_file_path, destination_file_name, media_format, options)
+        destination_key = RightAws::S3::Key.create(@s3.bucket(options[:bucket]), destination_file_name)
+        s3_headers = { 'Content-Type' => media_format.mime_type,
+                       'Content-Disposition' => "attachment; filename=#{File.basename(destination_file_name)}"
+                     }
+
+        destination_key.put(File.new(source_file_path), 'public-read', s3_headers)
+        FileUtils.rm(source_file_path)
+      end
+
+      def put_thumbnail_file(thumbnail_file, source_file_key_name, options)
+        destination_key = RightAws::S3::Key.create(@s3.bucket(options[:bucket]), "#{source_file_key_name}.thumb.jpg")
+        destination_key.put(thumbnail_file, 'public-read', 'Content-Type' => 'image/jpg')
+        FileUtils.rm(thumbnail_file.path)
+      end
+    end
+  end
+end

data/lib/transcoding_machine/server/transcoder.rb

@@ -0,0 +1,157 @@
+require 'transcoding_machine/server/media_file_attributes'
+require 'transcoding_machine/server/file_storage'
+
+module TranscodingMachine
+  module Server
+    class Transcoder
+      @@options = {:work_directory => '/tmp', :storage => FileStorage.new}
+      def self.options
+        @@options
+      end
+
+      def work_directory
+        self.class.options[:work_directory]
+      end
+
+      def storage
+        self.class.options[:storage]
+      end
+
+      def initialize(source_file_name, media_players, media_formats, event_handler = nil, storage_options = {})
+        @source_file_name = source_file_name
+        @event_handler = event_handler
+        @media_players = media_players
+        @media_formats = media_formats
+        @storage_options = storage_options
+      end
+
+      def source_file_path
+        @source_file_path ||= File.expand_path(@source_file_name, work_directory)
+      end
+
+      def source_file_directory
+        @source_file_directory ||= File.dirname(source_file_path)
+      end
+
+      def destination_file_name(media_format)
+        @source_file_name + media_format.suffix
+      end
+
+      def destination_file_path(media_format)
+        File.expand_path(destination_file_name(media_format), work_directory)
+      end
+
+      def start
+        prepare_working_directory
+        get_source_file
+        source_file_attributes, source_media_format = analyze_source_file
+
+        thumbnail_file_path = generate_thumbnail(source_file_attributes)
+        storage.put_thumbnail_file(thumbnail_file_path, @source_file_name, @storage_options) if thumbnail_file_path
+
+        media_formats = @media_players.map {|mp| mp.best_format_for(source_file_attributes)}.compact.uniq
+
+        media_formats -= [source_media_format]
+
+        media_formats.each do |media_format|
+          destination_file_path = transcode(source_file_attributes, media_format)
+          put_destination_file(destination_file_path, media_format)
+        end
+        clear
+      end
+
+      def prepare_working_directory
+        if !File.exist?(source_file_directory)
+          puts "creating directory #{source_file_directory}"
+          FileUtils.mkdir_p(source_file_directory)
+        end
+      end
+
+      def get_source_file
+        @event_handler.getting_source_file if @event_handler
+        storage.get_file(@source_file_name, source_file_path, @storage_options)
+        @event_handler.got_source_file if @event_handler
+      end
+
+      def analyze_source_file
+        @event_handler.analyzing_source_file if @event_handler
+
+        source_file_attributes = MediaFileAttributes.new(source_file_path)
+
+        source_media_format = @media_formats.find {|mf| mf.matches(source_file_attributes)}
+
+        @event_handler.analyzed_source_file(source_file_attributes, source_media_format) if @event_handler
+        [source_file_attributes, source_media_format]
+      end
+
+      def generate_thumbnail(source_file_attributes)
+        if source_file_attributes.video?
+          @event_handler.generating_thumbnail_file if @event_handler
+          file = source_file_attributes.thumbnail_file
+          @event_handler.generated_thumbnail_file if @event_handler
+          file
+        else
+          nil
+        end
+      end
+
+      def transcode(source_file_attributes, media_format)
+        @event_handler.transcoding(media_format) if @event_handler
+
+        dst_file_path = destination_file_path(media_format)
+        cmd = command_for(source_file_attributes, media_format, dst_file_path)
+        commands = cmd.split(' && ')
+        puts "Number of commands to run: #{commands.size}"
+
+        commands.each do |command|
+          puts "Running command: #{command}"
+          result = begin
+            timeout(1000 * 60 * 55) do
+              puts IO.popen(command).read
+            end
+          rescue Timeout::Error => e
+            puts "TIMEOUT REACHED WHEN RUNNING COMMAND"
+          end
+        end
+
+        @event_handler.transcoded(media_format) if @event_handler
+        dst_file_path
+      end
+
+      def put_destination_file(file_path, media_format)
+        @event_handler.putting_destination_file(file_path, media_format) if @event_handler
+        storage.put_file(destination_file_path(media_format), destination_file_name(media_format), media_format, @storage_options)
+        @event_handler.put_destination_file(file_path, media_format) if @event_handler
+      end
+
+      def clear
+        FileUtils.rm(source_file_path)
+      end
+
+      def command_for(source_file_attributes, media_format, destination_file_path)
+        command_variables = {
+          'in_file_name' => "\"#{source_file_path}\"",
+          'in_directory' => "#{source_file_directory}",
+          'out_file_name' => "\"#{destination_file_path}\"",
+          'out_directory' => "#{source_file_directory}"
+        }
+
+        command_variables['fps'] = target_fps(source_file_attributes) if source_file_attributes.video_fps
+
+        cmd = media_format.command.strip
+
+        command_variables.each {|key, value| cmd.gsub!("{{#{key}}}", value.to_s) }
+
+        cmd
+      end
+
+      def target_fps(source_file_attributes)
+        if fps = source_file_attributes.video_fps
+          fps > 30 ? 25 : fps
+        else
+          nil
+        end
+      end
+    end 
+  end
+end

data/lib/transcoding_machine/server/transcoding_event_listener.rb

@@ -0,0 +1,59 @@
+require 'right_aws'
+
+module TranscodingMachine
+  module Server
+    class TranscodingEventListener
+      def initialize(message_properties)
+        @message_properties = message_properties
+        @result_queue = RightAws::SqsGen2.new.queue(message_properties[:result_queue])
+      end
+
+      def getting_source_file
+        push_status(:downloading)
+      end
+
+      def got_source_file
+
+      end
+
+      def analyzing_source_file
+        push_status(:analyzing)
+      end
+
+      def analyzed_source_file(source_file_attributes, source_media_format)
+        push_status(:analyzed, :media_format => source_media_format, :media_attributes => source_file_attributes)
+      end
+
+      def generating_thumbnail_file
+        push_status(:creating_thumbnail)
+      end
+
+      def generated_thumbnail_file
+
+      end
+
+      def transcoding(media_format)
+        push_status(:transcoding, :media_format => media_format)
+      end
+
+      def transcoded(media_format)
+        push_status(:transcoded, :media_format => media_format)
+      end
+
+      def putting_destination_file(file_path, media_format)
+        push_status(:uploading, :media_format => media_format, :destination_key => file_path)
+      end
+
+      def put_destination_file(file_path, media_format)
+        push_status(:uploaded, :media_format => media_format, :destination_key => file_path)
+      end
+
+      def push_status(status, options = {})
+        msg = @message_properties.clone
+        msg[:status] = status
+        msg.merge!(options)
+        @result_queue.push(msg.to_yaml)
+      end
+    end
+  end
+end

data/lib/transcoding_machine/server/worker.rb

@@ -0,0 +1,137 @@
+require File.expand_path('../server', File.dirname(__FILE__))
+require 'right_aws'
+require 'transcoding_machine/server/ec2_environment'
+require 'transcoding_machine/server/transcoding_event_listener'
+
+module TranscodingMachine
+  module Server
+    class Worker
+      # initialize queues from names
+      def initialize(log)
+        @log = log
+        @shutdown = false
+        @state = "none"
+        @last_status_time = Time.now
+        @sqs = RightAws::SqsGen2.new
+        @s3 = RightAws::S3.new(nil, nil, :server => 's3.amazonaws.com', :port => 80, :protocol => 'http')
+
+        begin
+          @work_queue = @sqs.queue(Ec2Environment.work_queue_name)
+          if @work_queue.nil?
+            @log.puts "error #{$!} #{Ec2Environment.work_queue_name}"
+            raise "no work queue"
+          end
+          @status_queue = @sqs.queue(Ec2Environment.status_queue_name)
+          if @status_queue.nil?
+            @log.puts "error #{$!} #{Ec2Environment.status_queue_name}"
+            raise "no status queue"
+          end
+        rescue
+          @log.puts "error #{$!}"
+          raise "cannot list queues"
+        end
+
+        @media_players, @media_formats = TranscodingMachine.load_models_from_hash(Ec2Environment.transcoding_settings)
+        Transcoder.options[:storage] = S3Storage.new
+
+        set_state("idle")
+      end
+
+      def send_status_message(status)
+        now = Time.now
+        msg = { :type => 'status',
+          :instance_id => Ec2Environment.instance_id,
+          :state => 'active',
+          :load_estimate => status == 'busy' ? 1 : 0,
+          :timestamp => now}
+        @status_queue.push(msg.to_yaml)
+        @last_status_time = now
+      end
+
+      def send_log_message(message)
+        @log.puts message
+        msg = { :type => 'log',
+          :instance_id => Ec2Environment.instance_id,
+          :message => message,
+          :timestamp => Time.now}
+        @status_queue.push(msg.to_yaml)
+      end
+
+      # Send status when state changes, when state becomes busy, or
+      # every minute (even if there is no state change).
+      def set_state(new_state)
+        if new_state != @state ||
+            new_state == "busy" ||
+            @last_status_time + 60 < Time.now
+          @state = new_state
+          send_status_message(new_state)
+        end
+      end
+
+      def handle_message(msg)
+        #pp msg
+        set_state("busy")
+
+        start_time = Time.now
+
+        send_log_message "Transcoding: BLA BLA BLA"
+
+        message_properties = YAML.load(msg.body)
+        puts "consuming message #{message_properties.inspect}"
+        selected_media_players = find_media_players(message_properties[:media_players])
+        if selected_media_players.any?
+          if bucket = @s3.bucket(message_properties[:bucket].to_s)
+            key = bucket.key(message_properties[:key].to_s)
+            if key.exists?
+              transcoder = Transcoder.new(key.name,
+                                          selected_media_players,
+                                          @media_formats,
+                                          TranscodingEventListener.new(message_properties),
+                                          :bucket => bucket.name)
+              transcoder.start
+            else
+              send_log_message "Input file not found (bucket: #{message_properties[:bucket]} key: #{message_properties[:key]})"
+            end
+          else
+            send_log_message "Input bucket not found (bucket: #{message_properties[:bucket]})"
+          end
+        else
+          send_log_message "No media players found #{message_properties[:media_players].inspect}"
+        end
+
+        end_time = Time.now
+
+        if true #test if transcode was successful
+          msg.delete
+          send_log_message "Transcoded: BLA BLA BLA"
+        end
+      end
+
+      def find_media_players(media_player_ids)
+        @media_players.slice(*media_player_ids).values
+      end
+
+      def message_loop
+        msg = @work_queue.pop
+        if msg.nil?
+          #@log.puts "no messages"
+          set_state("idle")
+          sleep 5
+        else
+           handle_message(msg)
+        end
+      end
+
+      def shutdown
+        @shutdown = true
+      end
+
+      def run
+        while ! @shutdown
+          message_loop
+        end
+      end
+
+    end
+  end
+end

data/test/deserialze_test.rb

@@ -0,0 +1,7 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class DeserializeTest < Test::Unit::TestCase
+  def test_deserialization
+    puts TranscodingMachine.load_models_from_json(File.read('test/fixtures/serialized_models.json')).inspect
+  end
+end

data/test/fixtures/serialized_models.json

@@ -0,0 +1,52 @@
+{
+  "media_formats": {
+    "1": {
+      "name": "h264 Video 640x352",
+      "type": "video",
+      "priority": 3000,
+      "width": 640,
+      "height": 352,
+      "aspect_ratio": "16/9",
+      "mime_type": "video/mp4",
+      "suffix": "_h264_640_480.mp4",
+      "command": "cp {{in_file_name}} {{out_file_name}}",
+      "criteria": [
+        {"key": "video_codec", "operator": "equals", "value": "h264"},
+        {"key": "width", "operator": "equals", "value": 640},
+        {"key": "height", "operator": "equals", "value": 352},
+        {"key": "ipod_uuid", "operator": "equals", "value": true},
+        {"key": "audio_codec", "operator": "equals", "value": "aac"}
+      ]
+    },
+    "2": {
+      "name": "h264 HD 1280x720p",
+      "type": "video",
+      "priority": 10000,
+      "width": 1280,
+      "height": 720,
+      "aspect_ratio": "16/9",
+      "mime_type": "video/mp4",
+      "suffix": "_h264_HD1280_720p.mp4",
+      "command": "cp {{in_file_name}} {{out_file_name}}",
+      "criteria": [
+        {"key": "video_codec", "operator": "equals", "value": "h264"},
+        {"key": "width", "operator": "equals", "value": 1280},
+        {"key": "height", "operator": "equals", "value": 720},
+        {"key": "audio_codec", "operator": "equals", "value": "aac"}
+      ]
+    }
+  },
+  
+  "media_players": {
+    "appletv": {
+      "name": "AppleTV",
+      "description": "Damn cool media player",
+      "formats": ["1", "2"]
+    },
+    "ipod": {
+      "name": "iPod",
+      "description": "you know this thing!",
+      "formats": ["1"]
+    }
+  }
+}