sqewer 5.0.0 → 5.0.1

data/lib/sqewer/{contrib → extensions}/appsignal_wrapper.rb

@@ -7,19 +7,19 @@ module Sqewer
       # Unserialize the job
       def around_deserialization(serializer, msg_id, msg_payload)
         return yield unless (defined?(Appsignal) && Appsignal.active?)
-    
+
         Appsignal.monitor_transaction('perform_job.demarshal', 
           :class => serializer.class.to_s, :params => {:recepit_handle => msg_id}, :method => 'deserialize') do
           yield
         end
       end
-  
+
       # Run the job with Appsignal monitoring.
       def around_execution(job, context)
         return yield unless (defined?(Appsignal) && Appsignal.active?)
-        
+        job_params = job.respond_to?(:to_h) ? job.to_h : {}
         Appsignal.monitor_transaction('perform_job.sqewer', 
-          :class => job.class.to_s, :params => job.to_h, :method => 'run') do |t|
+          :class => job.class.to_s, :params => job_params, :method => 'run') do |t|
             context['appsignal.transaction'] = t
           yield
         end

data/lib/sqewer/extensions/railtie.rb

@@ -0,0 +1,12 @@
+module Sqewer
+  require 'sqewer/extensions/active_job_adapter' if defined?(::ActiveJob)
+  
+  # Loads the Sqewer components that provide ActiveJob compatibility
+  class Railtie < Rails::Railtie
+    initializer "sqewer.load_active_job_adapter" do |app|
+      if defined?(::ActiveJob)
+        Rails.logger.warn "sqewer set as ActiveJob adapter. Make sure to call 'Rails.application.eager_load!` in your worker process"
+      end
+    end
+  end
+end

data/lib/sqewer/middleware_stack.rb

@@ -1,18 +1,18 @@
 # Allows arbitrary wrapping of the job deserialization and job execution procedures
 class Sqewer::MiddlewareStack
-  
+
   # Returns the default middleware stack, which is empty (an instance of None).
   #
   # @return [MiddlewareStack] the default empty stack
   def self.default
     @instance ||= new
   end
-  
+
   # Creates a new MiddlewareStack. Once created, handlers can be added using `:<<`
   def initialize
     @handlers = []
   end
-  
+
   # Adds a handler. The handler should respond to :around_deserialization and #around_execution.
   #
   # @param handler[#around_deserializarion, #around_execution] The middleware item to insert
@@ -21,13 +21,13 @@ class Sqewer::MiddlewareStack
     @handlers << handler
     # TODO: cache the wrapping proc
   end
-  
+
   def around_execution(job, context, &inner_block)
     return yield if @handlers.empty?
-    
+
     responders = @handlers.select{|e| e.respond_to?(:around_execution) }
     responders.reverse.inject(inner_block) {|outer_block, middleware_object|
-      ->{ 
+      ->{
         middleware_object.public_send(:around_execution, job, context, &outer_block)
       }
     }.call
@@ -35,7 +35,7 @@ class Sqewer::MiddlewareStack
 
   def around_deserialization(serializer, message_id, message_body, &inner_block)
     return yield if @handlers.empty?
-    
+
     responders = @handlers.select{|e| e.respond_to?(:around_deserialization) }
     responders.reverse.inject(inner_block) {|outer_block, middleware_object|
       ->{ middleware_object.public_send(:around_deserialization, serializer, message_id, message_body, &outer_block) }

data/lib/sqewer/null_logger.rb

@@ -4,6 +4,6 @@ module Sqewer::NullLogger
   (Logger.instance_methods- Object.instance_methods).each do | null_method |
     define_method(null_method){|*a| }
   end
-  
+
   extend self
 end

data/lib/sqewer/resubmit.rb

@@ -0,0 +1,17 @@
+module Sqewer
+  class Resubmit
+    attr_reader :job
+    attr_reader :execute_after
+    
+    def initialize(job_to_resubmit, execute_after_timestamp)
+      @job = job_to_resubmit
+      @execute_after = execute_after_timestamp
+    end
+    
+    def run(ctx)
+      # Take the maximum delay period SQS allows
+      required_delay = (@execute_after - Time.now.to_i)
+      ctx.submit!(@job, delay_seconds: required_delay)
+    end
+  end
+end

data/lib/sqewer/serializer.rb

@@ -4,7 +4,7 @@
 # custom job objects from S3 bucket notifications, you might want to override this
 # class and feed the overridden instance to {Sqewer::Worker}.
 class Sqewer::Serializer
-  
+
   # Returns the default Serializer, of which we store one instance
   # (because the serializer is stateless).
   #
@@ -12,10 +12,9 @@ class Sqewer::Serializer
   def self.default
     @instance ||= new
   end
-  
+
   AnonymousJobClass = Class.new(StandardError)
-  ArityMismatch = Class.new(ArgumentError)
-  
+
   # Instantiate a Job object from a message body string. If the
   # returned result is `nil`, the job will be skipped.
   #
@@ -24,48 +23,49 @@ class Sqewer::Serializer
   def unserialize(message_body)
     job_ticket_hash = JSON.parse(message_body, symbolize_names: true)
     raise "Job ticket must unmarshal into a Hash" unless job_ticket_hash.is_a?(Hash)
-    
-    job_ticket_hash = convert_old_ticket_format(job_ticket_hash) if job_ticket_hash[:job_class]
-    
+
     # Use fetch() to raise a descriptive KeyError if none
     job_class_name = job_ticket_hash.delete(:_job_class)
     raise ":_job_class not set in the ticket" unless job_class_name
     job_class = Kernel.const_get(job_class_name)
+
+    # Grab the parameter that is responsible for executing the job later. If it is not set,
+    # use a default that will put us ahead of that execution deadline from the start.
+    t = Time.now.to_i
+    execute_after = job_ticket_hash.fetch(:_execute_after) { t - 5 }
     
     job_params = job_ticket_hash.delete(:_job_params)
-    if job_params.nil? || job_params.empty?
+    job = if job_params.nil? || job_params.empty?
       job_class.new # no args
     else
-      begin
-        job_class.new(**job_params) # The rest of the message are keyword arguments for the job
-      rescue ArgumentError => e
-        raise ArityMismatch, "Could not instantiate #{job_class} because it did not accept the arguments #{job_params.inspect}"
-      end
+      job_class.new(**job_params) # The rest of the message are keyword arguments for the job
     end
+    
+    # If the job is not up for execution now, wrap it with something that will 
+    # re-submit it for later execution when the run() method is called
+    return ::Sqewer::Resubmit.new(job, execute_after) if execute_after > t
+    
+    job
   end
-  
+
   # Converts the given Job into a string, which can be submitted to the queue
   #
   # @param job[#to_h] an object that supports `to_h`
+  # @param execute_after_timestamp[#to_i, nil] the Unix timestamp after which the job may be executed
   # @return [String] serialized string ready to be put into the queue
-  def serialize(job)
+  def serialize(job, execute_after_timestamp = nil)
     job_class_name = job.class.to_s
-    
+
     begin
       Kernel.const_get(job_class_name)
     rescue NameError
       raise AnonymousJobClass, "The class of #{job.inspect} could not be resolved and will not restore to a Job"
     end
-    
+
     job_params = job.respond_to?(:to_h) ? job.to_h : nil
     job_ticket_hash = {_job_class: job_class_name, _job_params: job_params}
+    job_ticket_hash[:_execute_after] = execute_after_timestamp.to_i if execute_after_timestamp
+    
     JSON.dump(job_ticket_hash)
   end
-  
-  private
-  
-  def convert_old_ticket_format(hash_of_properties)
-    job_class = hash_of_properties.delete(:job_class)
-    {_job_class: job_class, _job_params: hash_of_properties}
-  end
-end
+end

data/lib/sqewer/simple_job.rb

@@ -7,19 +7,19 @@
 module Sqewer::SimpleJob
   UnknownJobAttribute = Class.new(StandardError)
   MissingAttribute = Class.new(StandardError)
-  
+
   EQ_END = /(\w+)(\=)$/
-  
+
   # Returns the list of methods on the object that have corresponding accessors.
   # This is then used by #inspect to compose a list of the job parameters, formatted
   # as an inspected Hash.
   #
-  # @return [Array<Symbol>] the array of attributes to show via inspect 
+  # @return [Array<Symbol>] the array of attributes to show via inspect
   def inspectable_attributes
     # All the attributes that have accessors
     methods.grep(EQ_END).map{|e| e.to_s.gsub(EQ_END, '\1')}.map(&:to_sym)
   end
-  
+
   # Returns the inspection string with the job and all of it's instantiation keyword attributes.
   # If `inspectable_attributes` has been overridden, the attributes returned by that method will be the
   # ones returned in the inspection string.
@@ -36,7 +36,7 @@ module Sqewer::SimpleJob
     end
     "<#{self.class}:#{h.inspect}>"
   end
-  
+
   # Initializes a new Job with the given job args. Will check for presence of
   # accessor methods for each of the arguments, and call them with the arguments given.
   #
@@ -49,30 +49,30 @@ module Sqewer::SimpleJob
     @simple_job_args = jobargs.keys
     touched_attributes = Set.new
     jobargs.each do |(k,v)|
-      
+
       accessor = "#{k}="
       touched_attributes << k
       unless respond_to?(accessor)
         raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}" 
       end
-      
+
       send("#{k}=", v)
     end
-    
+
     accessors = methods.grep(EQ_END).map{|method_name| method_name.to_s.gsub(EQ_END, '\1').to_sym }
     settable_attributes = Set.new(accessors)
     missing_attributes = settable_attributes - touched_attributes
-    
+
     missing_attributes.each do | attr |
       raise MissingAttribute, "Missing job attribute #{attr.inspect}"
     end
   end
-  
+
   def to_h
     keys_and_values = @simple_job_args.each_with_object({}) do |k, h|
       h[k] = send(k)
     end
-    
+
     keys_and_values
   end
 end

data/lib/sqewer/state_lock.rb

@@ -13,11 +13,11 @@ class Sqewer::StateLock < SimpleDelegator
     m.permit_transition :starting => :failed # Failed to start
     __setobj__(m)
   end
-  
+
   def in_state?(some_state)
     @m.synchronize { __getobj__.in_state?(some_state) }
   end
-    
+
   def transition!(to_state)
     @m.synchronize { __getobj__.transition!(to_state) }
   end

data/lib/sqewer/submitter.rb

@@ -3,14 +3,28 @@
 # and the serializer (something that responds to `#serialize`) to
 # convert the job into the string that will be put in the queue.
 class Sqewer::Submitter < Struct.new(:connection, :serializer)
-  
+
   # Returns a default Submitter, configured with the default connection
   # and the default serializer.
   def self.default
     new(Sqewer::Connection.default, Sqewer::Serializer.default)
   end
-  
+
   def submit!(job, **kwargs_for_send)
-    connection.send_message(serializer.serialize(job), **kwargs_for_send)
+    message_body = if delay_by_seconds = kwargs_for_send[:delay_seconds]
+      clamped_delay = clamp_delay(delay_by_seconds)
+      kwargs_for_send[:delay_seconds] = clamped_delay
+      # Pass the actual delay value to the serializer, to be stored in executed_at
+      serializer.serialize(job, Time.now.to_i + delay_by_seconds)
+    else
+      serializer.serialize(job)
+    end
+    connection.send_message(message_body, **kwargs_for_send)
+  end
+  
+  private
+  
+  def clamp_delay(delay)
+    [1, 899, delay].sort[1]
   end
 end

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '5.0.0'
+  VERSION = '5.0.1'
 end

data/lib/sqewer/worker.rb

@@ -8,38 +8,38 @@ class Sqewer::Worker
   DEFAULT_NUM_THREADS = 4
   SLEEP_SECONDS_ON_EMPTY_QUEUE = 1
   THROTTLE_FACTOR = 2
-  
+
   # @return [Logger] The logger used for job execution
   attr_reader :logger
-  
+
   # @return [Sqewer::Connection] The connection for sending and receiving messages
   attr_reader :connection
-  
+
   # @return [Sqewer::Serializer] The serializer for unmarshalling and marshalling
   attr_reader :serializer
-  
+
   # @return [Sqewer::MiddlewareStack] The stack used when executing the job
   attr_reader :middleware_stack
-  
+
   # @return [Class] The class to use when instantiating the execution context
   attr_reader :execution_context_class
-  
+
   # @return [Class] The class used to create the Submitter used by jobs to spawn other jobs
   attr_reader :submitter_class
-  
+
   # @return [Array<Thread>] all the currently running threads of the Worker
   attr_reader :threads
-  
+
   # @return [Fixnum] the number of worker threads set up for this Worker
   attr_reader :num_threads
-  
-  # Returns the default Worker instance, configured based on the default components
+
+  # Returns a Worker instance, configured based on the default components
   #
   # @return [Sqewer::Worker]
   def self.default
-    @default ||= new
+    new
   end
-  
+
   # Creates a new Worker. The Worker, unlike it is in the Rails tradition, is only responsible for
   # the actual processing of jobs, and not for the job arguments.
   #
@@ -58,7 +58,7 @@ class Sqewer::Worker
       middleware_stack: Sqewer::MiddlewareStack.default,
       logger: Logger.new($stderr),
       num_threads: DEFAULT_NUM_THREADS)
-    
+
     @logger = logger
     @connection = connection
     @serializer = serializer
@@ -66,38 +66,38 @@ class Sqewer::Worker
     @execution_context_class = execution_context_class
     @submitter_class = submitter_class
     @num_threads = num_threads
-    
+
     @threads = []
-    
+
     raise ArgumentError, "num_threads must be > 0" unless num_threads > 0
-    
+
     @execution_counter = Sqewer::AtomicCounter.new
-    
+
     @state = Sqewer::StateLock.new
   end
-  
+
   # Start listening on the queue, spin up a number of consumer threads that will execute the jobs.
   #
   # @param num_threads[Fixnum] the number of consumer/executor threads to spin up
   # @return [void]
   def start
     @state.transition! :starting
-    
+
     @logger.info { '[worker] Starting with %d consumer threads' % @num_threads }
     @execution_queue = Queue.new
-    
+
     consumers = (1..@num_threads).map do
       Thread.new do
         catch(:goodbye) { loop {take_and_execute} }
       end
     end
-    
+
     # Create the provider thread. When the execution queue is exhausted,
     # grab new messages and place them on the local queue.
     provider = Thread.new do
       loop do
         break if stopping?
-        
+
         if queue_has_capacity?
           messages = @connection.receive_messages
           if messages.any?
@@ -110,12 +110,12 @@ class Sqewer::Worker
         else
           @logger.debug { "[worker] Cache is full (%d items), postponing receive" % @execution_queue.length }
           sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
-        end 
+        end
       end
     end
-    
+
     @threads = consumers + [provider]
-    
+
     # If any of our threads are already dead, it means there is some misconfiguration and startup failed
     if @threads.any?{|t| !t.alive? }
       @threads.map(&:kill)
@@ -126,7 +126,7 @@ class Sqewer::Worker
       @logger.info { '[worker] Started, %d consumer threads' % consumers.length }
     end
   end
-  
+
   # Attempts to softly stop the running consumers and the producer. Once the call is made,
   # all the threads will stop after the local cache of messages is emptied. This is to ensure that
   # message drops do not happen just because the worker is about to be terminated.
@@ -140,20 +140,20 @@ class Sqewer::Worker
     loop do
       n_live = @threads.select(&:alive?).length
       break if n_live.zero?
-      
+
       n_dead = @threads.length - n_live
       @logger.info { '[worker] Staged shutdown, %d threads alive, %d have quit, %d jobs in local cache' %
         [n_live, n_dead, @execution_queue.length] }
-      
+
       sleep 2
     end
-    
+
     @threads.map(&:join)
     @logger.info { '[worker] Stopped'}
     @state.transition! :stopped
     true
   end
-  
+
   # Peforms a hard shutdown by killing all the threads
   def kill
     @state.transition! :stopping
@@ -162,7 +162,7 @@ class Sqewer::Worker
     @logger.info { '[worker] Stopped'}
     @state.transition! :stopped
   end
-  
+
   # Prints the status and the backtraces of all controlled threads to the logger
   def debug_thread_information!
     @threads.each do | t |
@@ -170,48 +170,48 @@ class Sqewer::Worker
       @logger.debug { t.backtrace }
     end
   end
-  
+
   private
-  
+
   def stopping?
     @state.in_state?(:stopping)
   end
-  
+
   def queue_has_capacity?
     @execution_queue.length < (@num_threads * THROTTLE_FACTOR)
   end
-  
+
   def handle_message(message)
     return unless message.receipt_handle
-    
+
     # Create a messagebox that buffers all the calls to Connection, so that
     # we can send out those commands in one go (without interfering with senders
     # on other threads, as it seems the Aws::SQS::Client is not entirely
     # thread-safe - or at least not it's HTTP client part).
     box = Sqewer::ConnectionMessagebox.new(connection)
     return box.delete_message(message.receipt_handle) unless message.has_body?
-  
+
     job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body) do
       serializer.unserialize(message.body)
     end
     return unless job
-  
+
     submitter = submitter_class.new(box, serializer)
     context = execution_context_class.new(submitter, {'logger' => logger})
-  
+
     t = Time.now
     middleware_stack.around_execution(job, context) do
       job.method(:run).arity.zero? ? job.run : job.run(context)
     end
     box.delete_message(message.receipt_handle)
-  
+
     delta = Time.now - t
     logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
   ensure
     n_flushed = box.flush!
     logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
   end
-  
+
   def take_and_execute
     message = @execution_queue.pop(nonblock=true)
     handle_message(message)
@@ -222,31 +222,31 @@ class Sqewer::Worker
     @logger.error { '[worker] Failed "%s..." with %s: %s' % [message.inspect[0..32], e.class, e.message] }
     e.backtrace.each { |s| @logger.error{"\t#{s}"} }
   end
-  
+
   def perform(message)
     # Create a messagebox that buffers all the calls to Connection, so that
     # we can send out those commands in one go (without interfering with senders
     # on other threads, as it seems the Aws::SQS::Client is not entirely
     # thread-safe - or at least not it's HTTP client part).
     box = Sqewer::ConnectionMessagebox.new(connection)
-    
+
     job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body) do
       serializer.unserialize(message.body)
     end
     return unless job
-    
+
     submitter = submitter_class.new(box, serializer)
     context = execution_context_class.new(submitter, {'logger' => logger})
-    
+
     t = Time.now
     middleware_stack.around_execution(job, context) do
       job.method(:run).arity.zero? ? job.run : job.run(context)
     end
-    
+
     # Perform two flushes, one for any possible jobs the job has spawned,
     # and one for the job delete afterwards
     box.delete_message(message.receipt_handle)
-    
+
     delta = Time.now - t
     logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
   ensure