tootsie 0.9.0

data/lib/tootsie/application.rb

@@ -0,0 +1,48 @@
+require 's3'
+require 'sqs'
+
+module Tootsie
+    
+  class Application
+    
+    def initialize(options = {})
+      @@instance = self
+      @environment = options[:environment] || :development
+      @logger = options[:logger] || Logger.new($stderr)
+      @configuration = Configuration.new
+    end
+    
+    def configure!
+      @configuration.load_from_file(File.join(Dir.pwd, "config/#{@environment}.yml"))
+      @queue = Tootsie::SqsQueue.new(@configuration.sqs_queue_name, sqs_service)
+      @task_manager = TaskManager.new(@queue)
+    end
+    
+    def s3_service
+      return @s3_service ||= ::S3::Service.new(
+        :access_key_id => @configuration.aws_access_key_id,
+        :secret_access_key => @configuration.aws_secret_access_key)
+    end
+
+    def sqs_service
+      return @sqs_service ||= ::Sqs::Service.new(
+        :access_key_id => @configuration.aws_access_key_id,
+        :secret_access_key => @configuration.aws_secret_access_key)
+    end
+    
+    class << self
+      def get
+        @@instance
+      end
+    end
+    
+    attr_accessor :environment
+    
+    attr_reader :configuration
+    attr_reader :task_manager
+    attr_reader :queue
+    attr_reader :logger
+    
+  end
+  
+end

data/lib/tootsie/client.rb

@@ -0,0 +1,12 @@
+module Tootsie
+  
+  class Client
+    
+    # TODO: Implement client access control.
+    def self.generate_signature(k)
+      'signature'
+    end
+    
+  end
+  
+end

data/lib/tootsie/command_runner.rb

@@ -0,0 +1,58 @@
+module Tootsie
+
+  class CommandExecutionFailed < StandardError; end
+
+  class CommandRunner
+
+    def initialize(command_line, options = {})
+      @options = options.symbolize_keys
+      @options.assert_valid_keys(:ignore_exit_code, :output_encoding)
+      @command_line = command_line
+      @logger = Application.get.logger
+    end
+
+    def run(params = {}, &block)
+      command_line = @command_line
+      if params.any?
+        params = params.with_indifferent_access
+        command_line = command_line.gsub(/(^|\s):(\w+)/) do
+          pre, key, all = $1, $2, $~[0]
+          if params.include?(key)
+            value = params[key]
+            value = "'#{value}'" if value =~ /\s/
+            "#{pre}#{value}"
+          else
+            all
+          end
+        end
+      end
+      command_line = "#{command_line} 2>&1"
+
+      @logger.info("Running command: #{command_line}") if @logger.info?
+      IO.popen(command_line, "r:#{@options[:output_encoding] || 'utf-8'}") do |output|
+        output.each_line do |line|
+          @logger.info("[Command output] #{line.strip}") if @logger.info?
+          yield line if block_given?
+        end
+      end
+      status = $?
+      if status.exited?
+        if status.exitstatus != 0
+          if @options[:ignore_exit_code]
+            return false
+          else
+            raise CommandExecutionFailed, "Command failed with exit code #{status.exitstatus}: #{command_line}"
+          end
+        end
+      elsif status.stopped?
+        raise CommandExecutionFailed, "Command stopped unexpectedly with signal #{status.stopsig}: #{command_line}"
+      elsif status.signaled?
+        raise CommandExecutionFailed, "Command died unexpectedly by signal #{status.termsig}: #{command_line}"
+      else
+        raise CommandExecutionFailed, "Command died unexpectedly: #{command_line}"
+      end
+      true
+    end
+  
+  end
+end

data/lib/tootsie/configuration.rb

@@ -0,0 +1,29 @@
+module Tootsie
+  
+  class Configuration
+    
+    def initialize
+      @ffmpeg_thread_count = 1
+      @sqs_queue_name = 'tootsie'
+    end
+    
+    def load_from_file(file_name)
+      config = (YAML.load(File.read(file_name)) || {}).with_indifferent_access
+      [:aws_access_key_id, :aws_secret_access_key, :ffmpeg_thread_count,
+        :sqs_queue_name].each do |key|
+        if config.include?(key)
+          value = config[key]
+          value = $1.to_i if value =~ /\A\s*(\d+)\s*\z/
+          instance_variable_set("@#{key}", value)
+        end
+      end
+    end
+    
+    attr_accessor :aws_access_key_id
+    attr_accessor :aws_secret_access_key
+    attr_accessor :ffmpeg_thread_count
+    attr_accessor :sqs_queue_name
+    
+  end
+  
+end

data/lib/tootsie/daemon.rb

@@ -0,0 +1,282 @@
+require "logger"
+require "fileutils"
+require 'optparse'
+
+module Tootsie
+
+  class DaemonError < Exception; end
+  class DaemonAlreadyRunning < DaemonError; end
+  class DaemonNotRunning < DaemonError; end
+  class DaemonTerminationFailed < DaemonError; end
+  class DaemonNotConfigured < DaemonError; end
+  class DaemonStartFailed < DaemonError; end 
+
+  # Daemon controller class that encapsulates a running daemon and a remote interface to it.
+  class Daemon
+  
+    # Initializes the daemon controller.
+    def initialize(options = {})
+      @root = options[:root] || Dir.pwd
+      @pid_file = options[:pid_file]
+      @logger = options[:logger] || Logger.new('/dev/null')
+      @on_spawn = nil
+    end
+
+    # Specifies a block to execute to run the actual daemon. Each call overrides the previous one.  
+    def on_spawn(&block)
+      @on_spawn = block
+    end
+
+    # Specifies a block to execute to termiantion. Each call overrides the previous one.  
+    def on_terminate(&block)
+      @on_terminate = block
+    end
+  
+    # Control the daemon through command-line arguments.
+    def control(args, title = nil)
+      $stderr.sync = true
+      title ||= File.basename($0)
+      command = args.shift
+      control_with_command(command, args, title)
+    end
+  
+    # Control the daemon through a specific command.
+    def control_with_command(command, args, title = nil)
+      case command
+        when "start"
+          $stderr << "Starting #{title}: "
+          handle_errors do
+            start
+            $stderr << "started\n"
+          end
+        when "stop"
+          $stderr << "Stopping #{title}: "
+          options = {}
+          handle_errors do
+            stop({:signal => "TERM"}.merge(options))
+            $stderr << "stopped\n"
+          end
+        when "restart"
+          $stderr << "Restarting #{title}: "
+          handle_errors do
+            restart
+            $stderr << "restarted\n"
+          end
+        when "status"
+          if running?
+            $stderr << "#{title} is running\n"
+          else
+            $stderr << "#{title} is not running\n"
+          end
+      else
+        if command
+          $stderr << "#{File.basename($0)}: Invalid command #{command}\n"
+        else
+          puts "Usage: #{File.basename($0)} [start | stop | restart | status]"
+        end
+      end
+    end
+  
+    # Starts daemon.
+    def start
+      raise DaemonNotConfigured, "Daemon not configured" unless @on_spawn
+      FileUtils.mkdir_p(File.dirname(@pid_file)) rescue nil
+      pid = self.pid
+      if pid
+        if pid_running?(pid)
+          raise DaemonAlreadyRunning, "Process is already running with pid #{pid}"
+        end
+      end
+      File.delete(@pid_file) rescue nil
+      child_pid = Process.fork
+      if child_pid
+        sleep(1)      
+        unless running?
+          pid = self.pid
+          if pid == child_pid
+            raise DaemonStartFailed, "Daemon started, but failed prematurely"
+          else
+            raise DaemonStartFailed, "Daemon failed to start for some unknown reason"
+          end
+        end      
+        return
+      end
+      class << logger
+        def format_message(severity, timestamp, progname, msg)
+          "[#{timestamp}] #{msg}\n"
+        end
+      end
+      logger.info("Starting")
+      begin
+        Process.setsid
+        0.upto(255) do |n|
+          File.for_fd(n, "r").close rescue nil
+        end
+        File.umask(27)
+        Dir.chdir(@root)
+        $stdin = File.open("/dev/null", File::RDWR)
+        $stdout = File.open("/dev/null", File::RDWR)
+        $stderr = File.open("/dev/null", File::RDWR)
+        @pid = Process.pid
+        File.open(@pid_file, "w") do |file|
+          file << Process.pid
+        end
+        Signal.trap("HUP") do
+          logger.debug("Ignoring SIGHUP")
+        end
+        Signal.trap("TERM") do
+          if $$ == @pid
+            logger.info("Terminating (#{$$})")
+            @on_terminate.call if @on_terminate
+            File.delete(@pid_file) rescue nil
+          else
+            # Was sent to a child
+          end
+          exit(0)
+        end
+        @on_spawn.call
+        exit(0)
+      rescue SystemExit
+        # Do nothing
+      rescue Exception => e
+        message = "#{e.message}\n"
+        message << e.backtrace.map { |line| "\tfrom #{line}\n" }.join
+        logger.error(message)
+        exit(1)
+      ensure
+        logger.close
+      end
+    end
+  
+    # Stops daemon.
+    def stop(options = {})
+      stopped = false
+      found = false
+      pid = self.pid
+      if pid
+        # Send TERM to process
+        begin
+          Process.kill(options[:signal] || "TERM", pid)
+        rescue Errno::ESRCH
+          stopped = true
+        rescue Exception => e
+          raise DaemonTerminationFailed, "Could not stop process #{pid}: #{e.message}"
+        end
+        unless stopped
+          # Process was signaled, now wait for it to die
+          found = true
+          30.times do
+            begin
+              if not pid_running?(pid)
+                stopped = true
+                break
+              end
+              sleep(1)
+            rescue Exception => e
+              raise DaemonTerminationFailed, "Could not stop process #{pid}: #{e.message}"
+            end
+          end
+          if found and not stopped
+            # Process still running after wait, force kill and wait
+            begin
+              Process.kill("KILL", pid)
+            rescue Errno::ESRCH
+              stopped = true
+            end
+            30.times do
+              begin
+                if not pid_running?(pid)
+                  stopped = true
+                  break
+                end
+                sleep(1)
+              rescue Exception => e
+                raise DaemonTerminationFailed, "Could not stop process #{pid}: #{e.message}"
+              end
+            end
+            if not stopped
+              raise DaemonTerminationFailed, "Timeout attempting to stop process #{pid}"
+            end
+          end
+        end
+      end
+      unless found
+        raise DaemonNotRunning, "Process is not running"
+      end
+    end
+  
+    # Restarts daemon.
+    def restart
+      if running?
+        begin
+          stop
+        rescue DaemonNotRunning
+          # Ignore
+        end
+      end
+      start
+    end
+  
+    # Is the daemon running?
+    def running?
+      !pid.nil?
+    end
+  
+    # Returns the daemon pid.
+    def pid
+      pid = nil
+      maybe_pid = File.read(@pid_file) rescue nil
+      if maybe_pid =~ /([0-9]+)/
+        maybe_pid = $1.to_i
+        begin
+          Process.kill(0, maybe_pid)
+        rescue Errno::ESRCH
+        else
+          pid = maybe_pid
+        end
+      end
+      pid
+    end
+
+    # Signals the daemon.  
+    def signal(signal)
+      pid = self.pid
+      if pid
+        Process.kill(signal, pid)
+      else
+        raise DaemonNotRunning, "Process is not running"
+      end
+    end
+  
+    attr_reader :root
+    attr_reader :pid_file
+    attr_reader :logger
+  
+    private
+
+      def pid_running?(pid)
+        begin
+          Process.kill(0, pid)
+        rescue Errno::ESRCH
+          return false
+        end
+        return true
+      end
+        
+      def handle_errors(&block)
+        begin
+          yield
+        rescue DaemonError => e
+          $stderr << "#{e.message}\n"
+          if e.is_a?(DaemonAlreadyRunning) or e.is_a?(DaemonNotRunning)
+            exit_code = 0
+          else
+            exit_code = 1
+          end
+          exit(exit_code)
+        end
+      end
+
+  end
+
+end

data/lib/tootsie/ffmpeg_adapter.rb

@@ -0,0 +1,132 @@
+module Tootsie
+
+  class FfmpegAdapter
+    
+    def initialize(options = {})
+      @logger = Application.get.logger
+      @ffmpeg_binary = 'ffmpeg'
+      @ffmpeg_arguments = {}
+      @ffmpeg_arguments['threads'] = (options[:thread_count] || 1)
+      @ffmpeg_arguments['v'] = 1
+      if false
+        # TODO: Only in newer FFmpeg versions
+        @ffmpeg_arguments['xerror'] = true
+        @ffmpeg_arguments['loglevel'] = 'verbose'
+      end
+      @ffmpeg_arguments['y'] = true
+    end
+    
+    # Transcode a file by taking an input file and writing an output file.
+    def transcode(input_filename, output_filename, options = {})
+      arguments = @ffmpeg_arguments.dup
+      if options[:audio_codec].to_s == 'none'
+        arguments['an'] = true
+      else
+        case options[:audio_codec].try(:to_s)
+          when 'aac'
+            arguments['acodec'] = 'libfaac'
+          when String
+            arguments['acodec'] = options[:audio_codec]
+        end
+        arguments['ar'] = options[:audio_sample_rate] if options[:audio_sample_rate]
+        arguments['ab'] = options[:audio_bitrate] if options[:audio_bitrate]
+      end
+      if options[:video_codec].to_s == 'none'
+        arguments['vn'] = true
+      else
+        case options[:video_codec].try(:to_s)
+          when 'h264'
+            arguments['vcodec'] = 'libx264'
+            arguments['vpre'] = ['medium', 'main']  # TODO: Allow override
+            arguments['crf'] = 15                   # TODO: Allow override
+            arguments['threads'] = 0
+          when String
+            arguments['vcodec'] = options[:video_codec]
+        end
+        arguments['b'] = options[:video_bitrate] if options[:video_bitrate]
+        arguments['r'] = options[:video_frame_rate] if options[:video_frame_rate]
+        arguments['s'] = "#{options[:width]}x#{options[:height]}" if options[:width] or options[:height]
+        arguments['sameq'] = true
+      end
+      arguments['f'] = options[:format] if options[:format]
+
+      progress, expected_duration = @progress, nil
+      result_width, result_height = nil
+      run_ffmpeg(input_filename, output_filename, arguments) do |line|
+        if progress
+          case line
+            when /^\s*Duration: (\d+):(\d+):(\d+)\./
+              unless expected_duration
+                hours, minutes, seconds = $1.to_i, $2.to_i, $3.to_i
+                expected_duration = seconds + minutes * 60 + hours * 60 * 60
+              end
+            when /^frame=.* time=(\d+)\./
+              if expected_duration
+                elapsed_time = $1.to_i
+              end
+            when /Stream.*Video: .*, (\d+)x(\d+)\s/
+              unless result_width and result_height
+                result_width, result_height = $1.to_i, $2.to_i
+              end
+          end
+          progress.call(elapsed_time, expected_duration) if elapsed_time
+        end
+      end
+
+      thumbnail_options = options[:thumbnail]
+      if thumbnail_options
+        thumb_width = thumbnail_options[:width].try(:to_i) || options[:width].try(:to_i)
+        thumb_height = thumbnail_options[:height].try(:to_i) || options[:height].try(:to_i)
+        if not thumbnail_options[:force_aspect_ratio] and result_width and result_height
+          thumb_height = (thumb_width / (result_width / result_height.to_f)).to_i
+        end
+        at_seconds = thumbnail_options[:at_seconds].try(:to_f)
+        at_seconds ||= (expected_duration || 0) * (thumbnail_options[:at_fraction].try(:to_f) || 0.5)
+        @logger.info("Getting thumbnail frame (#{thumb_width}x#{thumb_height}) with FFmpeg at #{at_seconds} seconds")
+        begin
+          run_ffmpeg(input_filename, thumbnail_options[:filename], @ffmpeg_arguments.merge(
+            :ss => at_seconds,
+            :vcodec => :mjpeg,
+            :vframes => 1,
+            :an => true,
+            :f => :rawvideo,
+            :s => "#{thumb_width}x#{thumb_height}"))
+        rescue FfmpegAdapterExecutionFailed => e
+          @logger.error("Thumbnail rendering failed, ignoring: #{e}")
+        end
+      end
+    end
+    
+    attr_accessor :ffmpeg_binary
+    attr_accessor :ffmpeg_arguments
+    
+    # Output captured from FFmpeg command line tool so far.
+    attr_reader :output
+
+    # Progress reporter that implements +call(seconds, total_seconds)+ to record
+    # transcoding progress.
+    attr_accessor :progress
+    
+    private
+    
+      def run_ffmpeg(input_filename, output_filename, arguments, &block)
+        command_line = @ffmpeg_binary.dup
+        command_line << " -i '#{input_filename}' "
+        command_line << arguments.map { |k, v|
+          case v
+            when TrueClass, FalseClass
+              "-#{k}"
+            when Array
+              v.map { |w| "-#{k} '#{w}'" }.join(' ')
+            else
+              "-#{k} '#{v}'"
+          end
+        }.join(' ')
+        command_line << ' '
+        command_line << "'#{output_filename}'"
+        CommandRunner.new(command_line).run(&block)
+      end
+
+  end
+    
+end