sqewer 5.0.0 → 5.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3fe9d14faeab2b77f5551ae617cf50b24b803698
-  data.tar.gz: 10973f30ea8433ebc3f0cdc2a4b8ae8705a24476
+  metadata.gz: ddea2036ab00b9c68e83ed4bcd64bf7a46ab5114
+  data.tar.gz: 9a0efcf0a664e35d628d1c602f7fb0339490233a
 SHA512:
-  metadata.gz: bc139a5322ae65835d5ebdd0d95568e36f5d54a029340a6b834cd8528b64168c58d80e9b62790707b7eb69bdd9616cb20e781119f4d0e6cc713c82c226749bfe
-  data.tar.gz: 52139badc74df3a2b208415063c5f336b98c294b630abbf95d20165882a01a6908d69139d73ba5e73ecbc52d0b595d1c73de8ded5b2b6ea433f0da87d0ea78d9
+  metadata.gz: 432aa4dca6829cf25a5a402b0d8da73cfc3719aba51479068fb7572bbac0badf15319970f97c015b95dcf3bcce25b1004233a0ecbf5e969ef238c4ce4db2a775
+  data.tar.gz: 2bce53adecc4d6e869b05c0a9d0cdafae7a4cf021b1e73f7ad25cbf061ac7ddd1fb61a2f571bc2a1c61415be35ea6900ae915a2ed8d577df26bb2c29208de0b1

data/.travis.yml

@@ -8,3 +8,4 @@ env:
   - secure: E+uNoJmvjq5+3/PYUs2a1bRh7bCjAbcYEIKJzOVgIiI+gWB4dLpTKwVW/fqJbjCDIGPw9OwiwphN51zLEo0ie5AJIkavcjWRcdazKq8usokAz/A5Ilru5YvGXJ5UQtF8TJNsZfJGOGMJwL5d/MsDSd3/sjy88wrFiSeeNlHA06KDBrGdZXE4/dOb5Z84cSxPaD7nDaGP36V3Uv88X272oezODm9Mp+Fsf4goGoU9I9PBDD36pCXAlpURWpyWMKa2TzEXfgN43TYxta4p+bu6BA871df31j3BfVKwY1K/59eN9HqW8xWF9rgcgLo3NgQuaoqde3ryCE4jqKymjtG1/R5bdbQsn07R7/OagQMPVkbn07Aq+X6Q8GVTkrMl829z87QiMETNnzpsHl/1RD3DVnUo4k+XmqLbwW6NRl452j3eEHsiMvxf1vt8Qn1hu61qm98WGVNCGFWX/eAu/iTpvDcQOkoUuUEfGnXPaGeM8GT7tNdKptT8gsg/H0Dwb4d0qwA8uAEWttmZBmI9145hpnIk+hkDkH22dzhTus8d7CfFDSpu7s3y9434wiCYF5pkugyWJb5dd24d00ssgF5gqhLU/s48WcBwipl8mMGu/H/5ezh/pG9dL2uDIUjKRrUybE/vmDJGy8ZEDnOwJ04NJ701vQjFybgZRXXyU5ESEJQ=
   - secure: hq1T5MsHGmEnfQjOJlu/GzUPJ4WH0+lZOHezb4i0rFmRtWnmOMO+YUA34q9P6VQROk5KcAiRKtqZYmxZwTjyJVk2DTcJkvUAVzWBHD+MZAsH/lql+lnUJTuZRapYI85LChmEZkDMhYxZYfxTqGTbE4GjLtVtGN8zmTRStLxtvqyfiaU0SuGzyw9wfso/QhqHadQAQ7NttTYyiSiO2vHxK7rhmNi99MxAeVL3qZ3H2mcypSEE7CZd+nFa2CQxh/Guce0i+N2PDYTB6dsvA4xA5Xh/e+han3g4x/sRpic2C/9HS2zRYf0iN20+pFyotp5SaW3o5Q5S3iUqQtyZoR2lN1YMd4csyyKp/0G4QOjAPIN3FIef/RTVLqdfZXaord4iAg9GN9eKdtgYuS7JgyDl0VOmTpS2RYuDP+PGR2HRXXVc0gHt2aFl/oy+DTXh6emcOkpZtRZlbHqtVz5WXZZftm8niId+UXQmJuIQU+YgIiFDmviafapwjaYoXHCv8QMAGqNBP5EWDfxvYyPtGhQrj37sRYpfnUCkdJCiZvZLhQgQ0aIvR3KJIFpXtQmWaaT3e7u07B6weBB7Icto6WflfKtYHvVd1YfOX02/J8byhJbzRY+Qjqrq+5aQR1L3AKbmv8AooZdcm/CrEx6MP7xuIM0fjp1oU4cKQe74KX/KuqE=
   - secure: fWzpY63ZAXOtjEJuYoya/mwzJ7fZQJGsN6hhamXorOhUgaeP2vdC7/6gK3PPMIXOJ5NXPiANGbAUWKgLLWJ3F7rK2y9UCUWTzvE2CK/13vUV9de7byQfAjfHo7wjDJuPLved+WmSM7ASVjzGq9wQS8MTeYUymSNVY3xiZcvNTO2XY9AAu32SSWDs1ayXiCGE9DyN5WEvEEPixHqpplZLvSqTtKtEZVdP/a3tPuzogYFvoJ/ZEfG9LRr4nnbwuKCaTljpYaZc/8yFbhhJgCrTuOKb0gB/UFeMjd0WX6MQWofzIj152d0ghCTlgtESR2LQJdSXA2BO0ZzSqYj8NKTAcZ8IsaUj+6kBgfQGHu43TF/dB877v8pY55HS8a3gFPDK/6OZhRT7UcFZgle1qvX422CD196fRHktSBcDTx9jQsG8r7Ray9oJygXQr6tKhrRkjMkkBAiBd/GkIJkJu5RecmlLOZMJ+YeYa95kWDUXwo3HGK9MxtFyuplSRYk4mQ1jbrJXdmufQuTazl2FYL+elOUeUb8ej3iniUcmyDGU6tZx84aZTbBOaovmeQqr3pXIaXYMhvkRGLD3Ky7SNZN9ZzFczRWcKStyQal+BYK1R3RU8cD0m0PEpUxV5ERP50E3GyN/0d0bg22LywK9aWDHOhCzjI0emVvSAn8ZiY78SSo=
+  - AWS_REGION=eu-central-1

data/ACTIVE_JOB.md

@@ -0,0 +1,64 @@
+# Sqewer with ActiveJob
+
+This gem includes a queue adapter for usage with ActiveJob in Rails 4.2+. The functionality
+is well-tested and should function for any well-conforming ActiveJob subclasses.
+
+To run the default `sqewer` worker setup against your Rails application, first set it as the
+executing backend for ActiveJob in your Rails app configuration, set your `SQS_QUEUE_URL`
+in the environment variables, and make sure you can access it using your default (envvar-based
+or machine role based) AWS credentials. Then, set sqewer as the adapter for ActiveJob:
+
+    class Application < Rails::Application
+      ...
+      config.active_job.queue_adapter = :sqewer
+    end
+
+and then run
+
+    $ bundle exec sqewer_rails
+
+in your rails source tree, via a foreman Procfile or similar. If you want to run your own worker binary
+for executing the jobs, be aware that you _have_ to eager-load your Rails application's code explicitly
+before the Sqewer worker is started. The worker is threaded and any kind of autoloading does not generally
+play nice with threading. So do not forget to add this in your worker code:
+
+    Rails.application.eager_load!
+
+For handling error reporting within your Sqewer worker, set up a middleware stack as described in the documentation.
+
+## ActiveJob feature support matrix
+
+Compared to the matrix of features as seen in the
+[official ActiveJob documentation](http://edgeapi.rubyonrails.org/classes/ActiveJob/QueueAdapters.html)
+`sqewer` has the following support for various ActiveJob options, in comparison to the builtin
+ActiveJob adapters:
+
+    |                   | Async | Queues | Delayed    | Priorities | Timeout | Retries |
+    |-------------------|-------|--------|------------|------------|---------|---------|
+    | sqewer            | Yes   | No     | Yes        | No         | No      | Global  |
+    |       //          |  //   |  //    |  //        | //         |  //     | //      |
+    | Active Job Async  | Yes   | Yes    | Yes        | No         | No      | No      |
+    | Active Job Inline | No    | Yes    | N/A        | N/A        | N/A     | N/A     |
+
+Retries are set up globally for the entire SQS queue. There is no specific queue setting per job,
+since all the messages go to the queue available to `Sqewer.submit!`.
+
+There is no timeout handling, if you need it you may want to implement it within your jobs proper.
+Retries are handled on Sqewer level for as many deliveries as your SQS settings permit.
+
+## Delay handling
+
+Delayed execution is handled via a combination
+of the `delay_seconds` SQS parameter and the `_execute_after` job key (see the serializer documentation
+in Sqewer for more). In a nutshell - if you postpone a job by less than 900 seconds, the standard delivery
+delay option will be used - and the job will become visible for workers on the SQS queue only after this period.
+
+If a larger delay is used, the job will receive an additional field called `_execute_after`, which will contain
+a UNIX timestamp in seconds of when it must be executed at the earliest. In addition, the maximum permitted SQS
+delivery delay will be set for it. If the job then gets redelivered, Sqewer will automatically put it back on the
+queue with the same maximum delay, and will continue doing so for as long as necessary.
+
+Note that this will incur extra receives and sends on the queue, and even though it is not substantial,
+it will not be free. We think that this is an acceptable workaround for now, though. If you want a better approach,
+you may be better off using a Rails scheduling system and use a cron job or similar to spin up your enqueue
+for the actual, executable background task.

data/FAQ.md

@@ -3,10 +3,6 @@
 This document tries to answer some questions that may arise when reading or using the library. Hopefully
 this can provide some answers with regards to how things are put together.
 
-## Why no ActiveJob? 
-
-An adapter will be added in the future
-
 ## Why separate `new` and `run` methods instead of just `perform`?
 
 Because the job needs access to the execution context of the worker. It turned out that keeping the context

data/Gemfile

@@ -16,4 +16,8 @@ group :development do
   gem "rdoc", "~> 3.12"
   gem "bundler", "~> 1.0"
   gem "jeweler", "~> 2.0.1"
+  gem 'activerecord', '~> 4.2'
+  gem 'activejob', '~> 4.2.6'
+  gem 'sqlite3'
+  # gem 'pry'
 end

data/README.md

@@ -58,6 +58,10 @@ For more detailed usage information, see [DETAILS.md](./DETAILS.md)
 
 Please see [FAQ.md](./FAQ.md). This might explain some decisions behind the library in greater detail.
 
+## Usage with Rails via ActiveJob
+
+Please see [ACTIVE_JOB.md](./ACTIVE_JOB.md) for the exact description.
+
 ## Contributing to the library
  
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.

data/Rakefile

@@ -22,6 +22,7 @@ Jeweler::Tasks.new do |gem|
   gem.summary = %Q{A full-featured library for all them worker needs}
   gem.email = "me@julik.nl"
   gem.authors = ["Julik Tarkhanov"]
+  gem.executables = ["sqewer", "sqewer_rails"]
   # dependencies defined in Gemfile
 end
 Jeweler::RubygemsDotOrgTasks.new

data/bin/sqewer

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+require_relative '../lib/sqewer'
+
+# Load your application code here:
+# require 'my_application'
+
+Sqewer::CLI.start

data/bin/sqewer_rails

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+require_relative '../lib/sqewer'
+require_relative '../lib/sqewer/extensions/active_job_adapter'
+require_relative '../lib/sqewer/extensions/rails'
+
+# Make sure all the classes are preloaded before the threads are spun up
+Rails.application.eager_load!
+
+# ...and start
+Sqewer::CLI.start(Sqewer::Worker.new(logger: Rails.logger))

data/lib/sqewer.rb

@@ -1,11 +1,28 @@
 # The enclosing module for the library
 module Sqewer
-  Dir.glob(__dir__ + '/**/*.rb').each {|p| require p unless p == __FILE__ }
+  # Eager-load everything except extensions
+  Dir.glob(__dir__ + '/**/*.rb').each do |path|
+    if path != __FILE__ && File.dirname(path) !~ /\/extensions$/
+      require path
+    end
+  end
   
+  # Loads a particular Sqewer extension that is not loaded
+  # automatically during the gem require.
+  #
+  # @param extension_name[String] the name of the extension to load (like `active_job_adapter`)
+  def self.require_extension(extension_name)
+    path = File.join("sqewer", "extensions", extension_name)
+    require_relative path
+  end
+    
   # Shortcut access to Submitter#submit.
   #
   # @see {Sqewer::Submitter#submit!}
   def self.submit!(*jobs, **options)
     Sqewer::Submitter.default.submit!(*jobs, **options)
   end
+  
+  # If we are within Rails, load the railtie
+  require_relative 'sqewer/extensions/railtie' if defined?(Rails)
 end

data/lib/sqewer/atomic_counter.rb

@@ -5,14 +5,14 @@ class Sqewer::AtomicCounter
   def initialize
     @m, @v = Mutex.new, 0
   end
-  
+
   # Returns the current value of the counter
   #
   # @return [Fixnum] the current value of the counter
   def to_i
     @m.synchronize { @v + 0 }
   end
-  
+
   # Increments the counter
   #
   # @return [Fixnum] the current value of the counter

data/lib/sqewer/cli.rb

@@ -20,7 +20,7 @@ module Sqewer::CLI
         # Signal not supported
       end
     end
-    
+
     begin
       worker.start
       # The worker is non-blocking, so in the main CLI process we select() on the signal
@@ -34,7 +34,7 @@ module Sqewer::CLI
       exit 1
     end
   end
-  
+
   def handle_signal(worker, sig)
     case sig
     when 'USR1', 'TERM'
@@ -46,6 +46,6 @@ module Sqewer::CLI
       raise Interrupt
     end
   end
-  
+
   extend self
 end

data/lib/sqewer/connection.rb

@@ -8,18 +8,18 @@
 class Sqewer::Connection
   DEFAULT_TIMEOUT_SECONDS = 5
   BATCH_RECEIVE_SIZE = 10
-  
+
   # A wrapper for most important properties of the received message
   class Message < Struct.new(:receipt_handle, :body)
     def inspect
       body.inspect
     end
-    
+
     def has_body?
       body && !body.empty?
     end
   end
-  
+
   # Returns the default adapter, connected to the queue set via the `SQS_QUEUE_URL`
   # environment variable.
   def self.default
@@ -27,7 +27,7 @@ class Sqewer::Connection
   rescue KeyError => e
     raise "SQS_QUEUE_URL not set in the environment. This is the queue URL that the default that Sqewer uses"
   end
-  
+
   # Initializes a new adapter, with access to the SQS queue at the given URL.
   #
   # @param queue_url[String] the SQS queue URL (the URL can be copied from your AWS console)
@@ -35,7 +35,7 @@ class Sqewer::Connection
     require 'aws-sdk'
     @queue_url = queue_url
   end
-  
+
   # Receive at most 10 messages from the queue, and return the array of Message objects.
   #
   # @return [Array<Message>] an array of Message objects 
@@ -46,7 +46,7 @@ class Sqewer::Connection
       Message.new(message.receipt_handle, message.body)
     end
   end
-  
+
   # Send a message to the backing queue
   #
   # @param message_body[String] the message to send
@@ -56,7 +56,7 @@ class Sqewer::Connection
   def send_message(message_body, **kwargs_for_send)
     send_multiple_messages {|via| via.send_message(message_body, **kwargs_for_send) }
   end
-  
+
   # Stores the messages for the SQS queue (both deletes and sends), and yields them in allowed batch sizes
   class MessageBuffer < Struct.new(:messages)
     MAX_RECORDS = 10
@@ -67,7 +67,7 @@ class Sqewer::Connection
       messages.each_slice(MAX_RECORDS){|batch| yield(batch)}
     end
   end
-  
+
   # Saves the messages to send to the SQS queue
   class SendBuffer < MessageBuffer
     def send_message(message_body, **kwargs_for_send)
@@ -78,7 +78,7 @@ class Sqewer::Connection
       messages << m
     end
   end
-  
+
   # Saves the receipt handles to batch-delete from the SQS queue
   class DeleteBuffer < MessageBuffer
     def delete_message(receipt_handle)
@@ -88,7 +88,7 @@ class Sqewer::Connection
       messages << m
     end
   end
-  
+
   # Send multiple messages. If any messages fail to send, an exception will be raised.
   #
   # @yield [#send_message] the object you can send messages through (will be flushed at method return)
@@ -105,7 +105,7 @@ class Sqewer::Connection
       end
     end
   end
-  
+
   # Deletes a message after it has been succesfully decoded and processed
   #
   # @param message_identifier[String] the ID of the message to delete. For SQS, it is the receipt handle
@@ -113,7 +113,7 @@ class Sqewer::Connection
   def delete_message(message_identifier)
     delete_multiple_messages {|via| via.delete_message(message_identifier) }
   end
-  
+
   # Deletes multiple messages after they all have been succesfully decoded and processed.
   #
   # @yield [#delete_message] an object you can delete an individual message through
@@ -121,7 +121,7 @@ class Sqewer::Connection
   def delete_multiple_messages
     buffer = DeleteBuffer.new
     yield(buffer)
-    
+
     buffer.each_batch do | batch |
       resp = client.delete_message_batch(queue_url: @queue_url, entries: batch)
       failed = resp.failed
@@ -131,9 +131,9 @@ class Sqewer::Connection
       end
     end
   end
-  
+
   private
-  
+
   class RetryWrapper < Struct.new(:sqs_client)
     MAX_RETRIES = 1000
     # Provide retrying wrappers for all the methods of Aws::SQS::Client that we actually use
@@ -153,7 +153,7 @@ class Sqewer::Connection
       end
     end
   end
-  
+
   def client
     @client ||= RetryWrapper.new(Aws::SQS::Client.new)
   end

data/lib/sqewer/connection_messagebox.rb

@@ -23,7 +23,7 @@ class Sqewer::ConnectionMessagebox
     @sends = []
     @mux = Mutex.new
   end
-  
+
   # Saves the given body and the keyword arguments (such as delay_seconds) to be sent into the queue.
   # If there are more sends in the same flush, they will be batched using batched deletes.G
   #
@@ -33,7 +33,7 @@ class Sqewer::ConnectionMessagebox
       @sends << [message_body, kwargs_for_send]
     }
   end
-  
+
   # Saves the given identifier to be deleted from the queue. If there are more
   # deletes in the same flush, they will be batched using batched deletes.
   #
@@ -43,7 +43,7 @@ class Sqewer::ConnectionMessagebox
       @deletes << message_identifier
     }
   end
-  
+
   # Flushes all the accumulated commands to the queue connection.
   # First the message sends are going to be flushed, then the message deletes.
   # All of those will use batching where possible.
@@ -52,7 +52,7 @@ class Sqewer::ConnectionMessagebox
       @connection.send_multiple_messages do | buffer |
         @sends.each { |body, kwargs| buffer.send_message(body, **kwargs) }
       end
-      
+
       @connection.delete_multiple_messages do | buffer |
         @deletes.each { |id| buffer.delete_message(id) }
       end

data/lib/sqewer/execution_context.rb

@@ -14,14 +14,14 @@ class Sqewer::ExecutionContext
     @params = {}
     extra_variables.each_pair{|k, v| self[k]  = v }
   end
-  
+
   # Submits one or more jobs to the queue
   #
   # @see {Sqewer::Submitter#submit!}
   def submit!(job, **execution_options)
     @submitter.submit!(job, **execution_options)
   end
-  
+
   # Sets a key in the execution environment
   #
   # @param key[#to_s] the key to set
@@ -29,14 +29,14 @@ class Sqewer::ExecutionContext
   def []=(key, value)
     @params[key.to_s] = value
   end
-  
+
   # Returns a key of the execution environment by name
   #
   # @param key[#to_s] the key to get
   def [](key)
     @params[key.to_s]
   end
-  
+
   # Returns a key of the execution environment, or executes the given block
   # if the key is not set
   #
@@ -45,7 +45,7 @@ class Sqewer::ExecutionContext
   def fetch(key, &blk)
     @params.fetch(key.to_s, &blk)
   end
-  
+
   # Returns the logger set in the execution environment, or
   # the NullLogger if no logger is set. Can be used to supply
   # a logger prefixed with job parameters per job.

data/lib/sqewer/extensions/active_job_adapter.rb

@@ -0,0 +1,78 @@
+# ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
+# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
+module ActiveJob
+  module QueueAdapters
+
+    # Handle Rails ActiveJob through sqewer.
+    # Set it up like so:
+    #
+    #    Rails.application.config.active_job.queue_adapter = :sqewer
+    class SqewerAdapter
+      # Works as a Job for sqewer, and wraps an ActiveJob Worker which responds to perform()
+      class Performable
+
+        # Creates a new Performable using the passed ActiveJob object. The resulting Performable
+        # can be sent to any Sqewer queue.
+        #
+        # @param active_job_worker[ActiveJob::Job] the job you want to convert
+        def self.from_active_job(active_job_worker)
+          # Try to grab the job class immediately, so that an error is raised in the unserializer
+          # if the class is not available
+          klass = active_job_worker.class.to_s
+          Kernel.const_get(klass)
+          new(job: active_job_worker.serialize)
+        end
+
+        def initialize(job:)
+          @job = job
+        end
+
+        def to_h
+          {job: @job}
+        end
+
+        def inspect
+          '<%s>' % [@job.inspect]
+        end
+
+        # Runs the contained ActiveJob.
+        def run
+          job = ActiveSupport::HashWithIndifferentAccess.new(@job)
+          if active_record_defined_and_connected?
+            with_active_record_connection_from_pool { Base.execute(job) }
+          else
+            Base.execute(job)
+          end
+        end
+        
+        private
+        
+        def with_active_record_connection_from_pool
+          ActiveRecord::Base.connection_pool.with_connection { yield }
+        end
+        
+        def active_record_defined_and_connected?
+          defined?(ActiveRecord) && ActiveRecord::Base.connected?
+        end
+        
+      end
+
+      class << self
+        def enqueue(active_job) #:nodoc:
+          wrapped_job = Performable.from_active_job(active_job)
+
+          Sqewer.submit!(wrapped_job)
+        end
+
+        def enqueue_at(active_job, timestamp) #:nodoc:
+          wrapped_job = Performable.from_active_job(active_job)
+
+          delta_t = (timestamp - Time.now.to_i).to_i
+
+          Sqewer.submit!(wrapped_job, delay_seconds: delta_t)
+        end
+      end
+
+    end
+  end
+end