tootsie 0.9.16 → 0.9.17

data/README.md

@@ -177,6 +177,8 @@ Current limitations
 
 * Transcoding options are very basic.
 * No client access control; anyone can submit jobs.
+* SQS: Due to limitations in the `sqs` gem, only US queues may be used at the moment.
+* AMQP adapter does not support measuring the queue length at the moment.
 
 Requirements
 ============
@@ -244,6 +246,38 @@ Jobs may now be posted to the web service API. For example:
     }
     END
 
+Configuration
+=============
+
+The configuration is a YAML document with the following keys:
+
+    aws_access_key_id: <your Amazon key>
+    aws_secret_access_key: <your Amazon secret>
+    pid_path: <where to write pid file>
+    log_path: <where to write log file>
+    worker_count: <number of workers>
+    queue:
+      <... queue options ...>
+
+The queue options is a hash with a key `adapter` telling Tootsie which queue implementation to use.
+
+The `sqs` adapter takes the following options:
+
+    queue_name: <name of queue, defaults to 'tootsie'>
+    max_backoff: <max seconds to wait when queue is empty, defaults to 2>
+
+For the `amqp` adapter:
+
+    queue_name: <name of queue, defaults to 'tootsie'>
+    host_name: <host name of AMQP server, defaults to localhost>
+    max_backoff: <max seconds to wait when queue is empty, defaults to 2>
+
+For the `file` adapter:
+
+    root: <directory to store files>
+
+Note that when running a large number of workers, you should increase the backoff interval to avoid incurring a lot of queue requests.
+
 License
 =======
 

data/lib/tootsie/application.rb

@@ -26,13 +26,15 @@ module Tootsie
       case adapter
         when 'sqs'
           @queue = Tootsie::SqsQueue.new(
-            queue_options[:queue],
-            @configuration.aws_access_key_id,
-            @configuration.aws_secret_access_key)
+            :queue_name => queue_options[:queue],
+            :access_key_id => @configuration.aws_access_key_id,
+            :secret_access_key => @configuration.aws_secret_access_key,
+            :max_backoff => queue_options[:max_backoff])
         when 'amqp'
           @queue = Tootsie::AmqpQueue.new(
-            queue_options[:host],
-            queue_options[:queue])
+            :host_name => queue_options[:host],
+            :queue_name => queue_options[:queue],
+            :max_backoff => queue_options[:max_backoff])
         when 'file'
           @queue = Tootsie::FileSystemQueue.new(queue_options[:root])
         else

data/lib/tootsie/queues/amqp_queue.rb

@@ -3,10 +3,13 @@ module Tootsie
   # A queue which uses the AMQP protocol.
   class AmqpQueue
     
-    def initialize(host_name, queue_name)
+    def initialize(options = {})
+      options.assert_valid_keys(:host_name, :queue_name, :max_backoff)
       @logger = Application.get.logger
-      @host_name = host_name || 'localhost'
-      @queue_name = queue_name || 'tootsie'
+      @host_name = options[:host_name] || 'localhost'
+      @queue_name = options[:queue_name] || 'tootsie'
+      @max_backoff = (options[:max_backoff] || 2).to_f
+      @backoff = 0.0
       connect!
     end
     
@@ -61,7 +64,7 @@ module Tootsie
             @backoff /= 2.0
             return result
           else
-            @backoff = [@backoff * 1.1, 1.0].min
+            @backoff = [@backoff * 1.1, @max_backoff].min
             @logger.info "Backing off #{@backoff}"
             sleep(@backoff)
           end

data/lib/tootsie/queues/sqs_queue.rb

@@ -9,27 +9,30 @@ module Tootsie
   # A queue which uses Amazon's Simple Queue Service (SQS).
   class SqsQueue
     
-    def initialize(queue_name, access_key_id, secret_access_key)
+    def initialize(options)
+      options.assert_valid_keys(:access_key_id, :secret_access_key, :queue_name, :max_backoff)
       @sqs_service = ::Sqs::Service.new(
-        :access_key_id => access_key_id,
-        :secret_access_key => secret_access_key)
+        :access_key_id => options[:access_key_id],
+        :secret_access_key => options[:secret_access_key])
       @logger = Application.get.logger
-      @queue = @sqs_service.queues.find_first(queue_name)
+      @queue_name = options[:queue_name] || 'tootsie'
+      @queue = @sqs_service.queues.find_first(@queue_name)
       unless @queue
-        @logger.info "Queue #{queue_name} does not exist, creating it"
-        @sqs_service.queues.create(queue_name)
+        @logger.warn "Queue #{@queue_name} does not exist, creating it"
+        @sqs_service.queues.create(@queue_name)
         begin
           timeout(30) do
             while not @queue
               sleep(1)
-              @queue = @sqs_service.queues.find_first(queue_name)
+              @queue = @sqs_service.queues.find_first(@queue_name)
             end
           end
         rescue Timeout::Error
           raise SqsQueueCouldNotFindQueueError
         end
       end
-      @backoff = 0.5
+      @max_backoff = (options[:max_backoff] || 2).to_f
+      @backoff = 0.0
     end
     
     def count
@@ -76,7 +79,7 @@ module Tootsie
           @backoff /= 2.0
           break
         else
-          @backoff = [@backoff * 0.2, 2.0].min
+          @backoff = [@backoff * 0.2, @max_backoff].min
         end
         break unless options[:wait]
         sleep(@backoff)

data/lib/tootsie/version.rb

@@ -1,3 +1,3 @@
 module Tootsie
-  VERSION = '0.9.16'
+  VERSION = '0.9.17'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tootsie
 version: !ruby/object:Gem::Version
-  version: 0.9.16
+  version: 0.9.17
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-01-24 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70283191457260 !ruby/object:Gem::Requirement
+  requirement: &70200327586920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 1.4.6
   type: :runtime
   prerelease: false
-  version_requirements: *70283191457260
+  version_requirements: *70200327586920
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70283191456740 !ruby/object:Gem::Requirement
+  requirement: &70200327586400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70283191456740
+  version_requirements: *70200327586400
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70283191456260 !ruby/object:Gem::Requirement
+  requirement: &70200327585920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 3.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70283191456260
+  version_requirements: *70200327585920
 - !ruby/object:Gem::Dependency
   name: httpclient
-  requirement: &70283191455780 !ruby/object:Gem::Requirement
+  requirement: &70200327585440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 2.2.1
   type: :runtime
   prerelease: false
-  version_requirements: *70283191455780
+  version_requirements: *70200327585440
 - !ruby/object:Gem::Dependency
   name: builder
-  requirement: &70283191455300 !ruby/object:Gem::Requirement
+  requirement: &70200327584960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 2.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *70283191455300
+  version_requirements: *70200327584960
 - !ruby/object:Gem::Dependency
   name: mime-types
-  requirement: &70283191454820 !ruby/object:Gem::Requirement
+  requirement: &70200327584480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: '1.16'
   type: :runtime
   prerelease: false
-  version_requirements: *70283191454820
+  version_requirements: *70200327584480
 - !ruby/object:Gem::Dependency
   name: xml-simple
-  requirement: &70283191454340 !ruby/object:Gem::Requirement
+  requirement: &70200327584000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +87,10 @@ dependencies:
         version: 1.0.12
   type: :runtime
   prerelease: false
-  version_requirements: *70283191454340
+  version_requirements: *70200327584000
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70283191453860 !ruby/object:Gem::Requirement
+  requirement: &70200327583520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +98,10 @@ dependencies:
         version: 1.2.7
   type: :runtime
   prerelease: false
-  version_requirements: *70283191453860
+  version_requirements: *70200327583520
 - !ruby/object:Gem::Dependency
   name: s3
-  requirement: &70283191453380 !ruby/object:Gem::Requirement
+  requirement: &70200327583040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,10 +109,10 @@ dependencies:
         version: 0.3.8
   type: :runtime
   prerelease: false
-  version_requirements: *70283191453380
+  version_requirements: *70200327583040
 - !ruby/object:Gem::Dependency
   name: bunny
-  requirement: &70283191452900 !ruby/object:Gem::Requirement
+  requirement: &70200327582560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -120,10 +120,10 @@ dependencies:
         version: 0.7.8
   type: :runtime
   prerelease: false
-  version_requirements: *70283191452900
+  version_requirements: *70200327582560
 - !ruby/object:Gem::Dependency
   name: sqs
-  requirement: &70283201355500 !ruby/object:Gem::Requirement
+  requirement: &70200327582080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -131,10 +131,10 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *70283201355500
+  version_requirements: *70200327582080
 - !ruby/object:Gem::Dependency
   name: unicorn
-  requirement: &70283201355020 !ruby/object:Gem::Requirement
+  requirement: &70200327581600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -142,10 +142,10 @@ dependencies:
         version: 4.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *70283201355020
+  version_requirements: *70200327581600
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &70283201354540 !ruby/object:Gem::Requirement
+  requirement: &70200327581120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -153,10 +153,10 @@ dependencies:
         version: 0.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *70283201354540
+  version_requirements: *70200327581120
 - !ruby/object:Gem::Dependency
   name: scashin133-syslog_logger
-  requirement: &70283201354060 !ruby/object:Gem::Requirement
+  requirement: &70200327580640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -164,10 +164,10 @@ dependencies:
         version: 1.7.3
   type: :runtime
   prerelease: false
-  version_requirements: *70283201354060
+  version_requirements: *70200327580640
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70283201353680 !ruby/object:Gem::Requirement
+  requirement: &70200327580260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -175,10 +175,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70283201353680
+  version_requirements: *70200327580260
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70283201353220 !ruby/object:Gem::Requirement
+  requirement: &70200327579800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -186,7 +186,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70283201353220
+  version_requirements: *70200327579800
 description: Tootsie is a simple audio/video/image transcoding/modification application.
 email:
 - alex@origo.no