tootsie 0.9.0

data/lib/tootsie/queues/file_system_queue.rb

@@ -0,0 +1,65 @@
+require 'json'
+require 'tempfile'
+require 'fileutils'
+
+module Tootsie
+
+  # A simple, naive queue implementation that stores items as JSON files 
+  # in the file system.
+  class FileSystemQueue
+    
+    def initialize(directory)
+      @directory = directory
+      FileUtils.mkdir_p(@directory)
+    end
+    
+    def count
+      Dir.glob(File.join(@directory, "*.json")).length
+    end
+    
+    def push(item)
+      Tempfile.open('tootsie') do |tempfile|
+        tempfile << item.to_json
+        tempfile.close
+        FileUtils.mv(tempfile.path, File.join(@directory, "#{Time.now.to_f}.json"))
+      end
+    end
+    
+    def pop(options = {})
+      loop do
+        lock do
+          file_name = Dir.glob(File.join(@directory, "*.json")).sort.first
+          if file_name
+            item = JSON.parse(File.read(file_name))
+            FileUtils.rm(file_name)
+            return item
+          end
+        end
+        if options[:wait]
+          sleep(1.0)
+        else
+          return nil
+        end
+      end
+    end
+    
+    private
+    
+      def lock
+        lock_file_name = File.join(@directory, "lock");
+        begin
+          FileUtils.mkdir(lock_file_name)
+        rescue Errno::EEXIST
+          sleep(0.2)
+          retry
+        end
+        begin
+          yield
+        ensure
+          FileUtils.rmdir(lock_file_name)
+        end
+      end
+    
+  end
+  
+end

data/lib/tootsie/queues/sqs_queue.rb

@@ -0,0 +1,93 @@
+require 'json'
+require 'sqs'
+require 'timeout'
+
+module Tootsie
+  
+  class SqsQueueCouldNotFindQueueError < Exception; end
+
+  # A queue which uses Amazon's Simple Queue Service (SQS).
+  class SqsQueue
+    
+    def initialize(queue_name, sqs_service)
+      @logger = Application.get.logger
+      @sqs_service = sqs_service
+      @queue = @sqs_service.queues.find_first(queue_name)
+      unless @queue
+        @sqs_service.queues.create(queue_name)
+        begin
+          timeout(5) do
+            while not @queue
+              sleep(0.5)
+              @queue = @sqs_service.queues.find_first(queue_name)
+            end
+          end
+        rescue Timeout::Error
+          raise SqsQueueCouldNotFindQueueError
+        end
+      end
+      @backoff = 0.5
+    end
+    
+    def count
+      @queue.attributes['ApproximateNumberOfMessages'].to_i
+    end
+    
+    def push(item)
+      retries_left = 5
+      begin
+        return @queue.create_message(item.to_json)
+      rescue Exception => exception
+        check_exception(exception)
+        if retries_left > 0
+          @logger.warn("Writing queue failed with exception (#{exception.message}), will retry")
+          retries_left -= 1
+          sleep(0.5)
+          retry
+        else
+          @logger.error("Writing queue failed with exception #{exception.class}: #{exception.message}")
+          raise exception
+        end
+      end
+    end
+    
+    def pop(options = {})
+      item = nil
+      loop do
+        begin
+          message = @queue.message(5)
+        rescue Exception => exception
+          check_exception(exception)
+          @logger.error("Reading queue failed with exception #{exception.class}: #{exception.message}")
+          break unless options[:wait]
+          sleep(0.5)
+          retry
+        end
+        if message
+          begin
+            item = JSON.parse(message.body)
+          ensure
+            # Always destroy, even if parsing fails
+            message.destroy
+          end
+          @backoff /= 2.0
+          break
+        else
+          @backoff = [@backoff * 0.2, 2.0].min
+        end
+        break unless options[:wait]
+        sleep(@backoff)
+      end
+      item
+    end
+
+    private
+
+      def check_exception(exception)
+        raise exception if exception.is_a?(SystemExit)
+        raise exception if exception.is_a?(SignalException) and not exception.is_a?(Timeout::Error)
+      end
+    
+  end
+  
+end

data/lib/tootsie/s3_utilities.rb

@@ -0,0 +1,24 @@
+module Tootsie
+  
+  module S3Utilities
+    
+    class << self
+      def parse_uri(url)
+        if url =~ /^s3:([^\/]+)\/+(.+?)(?:\?(.*))?$/
+          output = {}.with_indifferent_access
+          output[:bucket], output[:key], option_string = $1, $2, $3
+          unless option_string.blank?
+            option_string.split('&').map { |pair| pair.scan(/^(.*?)=(.*)$/)[0] }.each do |k, v|
+              output[k] = v.to_sym
+            end
+          end
+          output
+        else
+          raise ArgumentError, "Not an S3 URL"
+        end
+      end
+    end
+    
+  end
+  
+end

data/lib/tootsie/spawner.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+
+require 'logger'
+require 'set'
+require 'timeout'
+
+class Spawner
+  
+  def initialize(options = {})
+    @num_children = options[:num_children] || 1
+    @pids = Set.new
+    @logger = options[:logger]
+    @terminating = false
+    @parent = true
+  end
+  
+  def on_spawn(&block)
+    @on_spawn = block
+  end
+  
+  def run(&block)
+    loop do
+      unless @terminating
+        while @pids.length < @num_children
+          pid = Process.fork
+          if pid
+            # In parent process
+            @pids << pid
+            logger.info("Child PID=#{pid} spawned")
+          else
+            # In child process 
+            @parent = false
+            @on_spawn.call
+            exit(0)
+          end
+        end
+      end
+      wait_for_children
+      break if @terminated and @pids.empty?
+    end
+  end
+  
+  def wait_for_children
+    pid = Process.waitpid(-1)
+    if pid
+      status = $?
+      if status.exited? 
+        if status.exitstatus == 0
+          logger.info("Child PID=#{pid} exited normally")
+        else
+          logger.info("Child PID=#{pid} exited unexpectedly with exit code #{status.exitstatus}")
+        end
+      elsif status.stopped?
+        logger.info("Child PID=#{pid} stopped unexpectedly with signal #{status.stopsig}")
+      elsif status.signaled?
+        logger.info("Child PID=#{pid} died unexpectedly by signal #{status.termsig}")
+      else
+        logger.info("Child PID=#{pid} died unexpectedly")
+      end
+      @pids.delete(pid)
+    end
+  end
+  
+  def terminate
+    if @parent
+      logger.info("Parent terminating, will terminate all child PIDs")
+      @terminating = true
+      @pids.each do |pid|
+        logger.info("Terminating child PID=#{pid}")
+        begin
+          Process.kill("TERM", pid)
+        rescue Errno::ESRCH
+          # Ignore
+        end
+      end
+      begin
+        timeout(5) do
+          while @pids.any?
+            sleep(0.5)
+            wait_for_children
+          end
+        end
+      rescue Timeout::Error
+        logger.error("Timed out waiting for children, killing them")
+        @pids.each do |pid|
+          logger.info("Killing child PID=#{pid}")
+          begin
+            Process.kill("KILL", pid)
+          rescue Errno::ESRCH
+            # Ignore
+          end
+        end
+      end
+    end
+  end
+  
+  attr_reader :logger
+  
+end

data/lib/tootsie/task_manager.rb

@@ -0,0 +1,51 @@
+require 'httpclient'
+require 'uri'
+require 'benchmark'
+
+module Tootsie
+  
+  class TaskManager
+    
+    def initialize(queue)
+      @queue = queue
+      @logger = Application.get.logger
+    end
+    
+    def schedule(task)
+      type = task.class.name.gsub(/^(?:[^:]+::)*(.*?)Task$/, '\1').underscore
+      data = task.attributes
+      @logger.info("Scheduling task #{type.inspect}: #{data.inspect}")
+      @queue.push({:task => type, :data => data})
+    end
+    
+    def run!
+      @logger.info "Ready to process tasks"
+      loop do
+        begin
+          task = @queue.pop(:wait => true)
+          if task
+            task = task.with_indifferent_access
+            type, data = task[:task], task[:data]
+            @logger.info("Processing task #{type.inspect}: #{data.inspect}")
+            begin
+              task_class = Tasks.const_get("#{type.camelcase}Task")
+            rescue NameError
+              @logger.error("Invalid task encountered on queue: #{task.inspect}")
+            else
+              task = task_class.new(data)
+              task.execute!
+            end
+          end
+        rescue Interrupt, SignalException, SystemExit
+          raise
+        rescue Exception => exception
+          backtrace = exception.backtrace.map { |s| "  #{s}\n" }.join
+          @logger.error "Task manager exception: #{exception.class}: #{exception}\n#{backtrace}"
+        end
+      end
+      @logger.info "Task manager done"
+    end
+    
+  end
+  
+end

data/lib/tootsie/tasks/job_task.rb

@@ -0,0 +1,111 @@
+module Tootsie
+  module Tasks
+    
+    class JobTask
+    
+      DEFAULT_MAX_RETRIES = 5
+      
+      PROGRESS_NOTIFICATION_INTERVAL = 10.seconds
+    
+      VALID_TYPES = %w(video audio image).freeze
+    
+      def initialize(attributes = {})
+        attributes = attributes.with_indifferent_access
+        @type = attributes[:type].to_s
+        @retries_left = attributes[:retries_left] || DEFAULT_MAX_RETRIES
+        @access_key = attributes[:access_key]
+        @created_at = Time.now
+        @notification_url = attributes[:notification_url]
+        @params = attributes[:params]
+        @logger = Application.get.logger
+        @use_tasks_for_notifications = false  # TODO: Disabled for now, SQS does not preserve order
+      end
+    
+      def valid?
+        return @type && VALID_TYPES.include?(@type)
+      end
+    
+      def execute!
+        @logger.info("Begin processing job: #{attributes.inspect}")
+        notify!(:event => :started)
+        begin
+          result = nil
+          elapsed_time = Benchmark.realtime {
+            next_notify = Time.now + PROGRESS_NOTIFICATION_INTERVAL
+            processor = Processors.const_get("#{@type.camelcase}Processor").new(@params)
+            result = processor.execute! { |progress_data|
+              if Time.now >= next_notify
+                notify!(progress_data.merge(:event => :progress))
+                next_notify = Time.now + PROGRESS_NOTIFICATION_INTERVAL
+              end
+            }
+          }
+          result ||= {}
+          notify!({
+            :event => :completed,
+            :time_taken => elapsed_time
+          }.merge(result))
+        rescue Interrupt
+          @logger.error "Job interrupted"
+          notify!(:event => :failed, :reason => 'Cancelled')
+          raise
+        rescue Exception => exception
+          @logger.error "Job failed with exception #{exception.class}: #{exception}\n" <<
+            "#{exception.backtrace.map { |line| "#{line}\n" }.join}"
+          if @retries_left > 0
+            @retries_left -= 1
+            @logger.info "Pushing job back on queue to retry it"
+            notify!(:event => :failed_will_retry, :reason => exception.message)
+            Application.get.task_manager.schedule(self)
+          else
+            @logger.error "No more retries for job, marking as failed"
+            notify!(:event => :failed, :reason => exception.message)
+          end
+        else
+          @logger.info "Completed job #{attributes.inspect}"
+        end
+      end
+    
+      # Notify the caller of this job with some message.
+      def notify!(message)
+        notification_url = @notification_url
+        if notification_url
+          message = message.merge(:signature => Client.generate_signature(@access_key)) if @access_key
+          message_json = message.stringify_keys.to_json
+          if @use_tasks_for_notifications
+            Application.get.task_manager.schedule(
+              Tasks::NotifyTask.new(:url => notification_url, :message => message_json))
+          else
+            # TODO: Retry on failure
+            @logger.info "Notifying #{notification_url} with message: #{message_json}"
+            begin
+              HTTPClient.new.post(notification_url, message_json,
+                'Content-Type' => 'application/json; charset=utf-8')
+            rescue Exception => exception
+              @logger.error "Notification failed with exception, ignoring it: #{exception.message}"
+            end
+          end
+        end
+      end
+    
+      def attributes
+        return {
+          :type => @type,
+          :notification_url => @notification_url,
+          :retries_left => @retries_left,
+          :access_key => @access_key,
+          :params => @params
+        }
+      end
+    
+      attr_accessor :retries_left
+      attr_accessor :created_at
+      attr_accessor :access_key
+      attr_accessor :notification_url
+      attr_accessor :params
+      attr_accessor :type
+          
+    end
+
+  end  
+end