sqewer 1.0.0

data/lib/sqewer/connection.rb

@@ -0,0 +1,66 @@
+# Adapter that handles communication with a specific queue. In the future this
+# could be switched to a Google PubSub queue, or to AMQP, or to any other queue
+# with guaranteed re-delivery without ACK. The required queue semantics are
+# very simple:
+#
+# * no message should be deleted if the receiving client has not deleted it explicitly
+# * any execution that ends with an exception should cause the message to be re-enqueued
+class Sqewer::Connection
+  DEFAULT_TIMEOUT_SECONDS = 5
+  BATCH_RECEIVE_SIZE = 10
+  
+  # Returns the default adapter, connected to the queue set via the `SQS_QUEUE_URL`
+  # environment variable.
+  def self.default
+    new(ENV.fetch('SQS_QUEUE_URL'))
+  rescue KeyError => e
+    raise "SQS_QUEUE_URL not set in the environment. This is the queue URL that the default that Sqewer uses"
+  end
+  
+  # Initializes a new adapter, with access to the SQS queue at the given URL.
+  #
+  # @param queue_url[String] the SQS queue URL (the URL can be copied from your AWS console)
+  def initialize(queue_url)
+    require 'aws-sdk'
+    @queue_url = queue_url
+  end
+  
+  # Poll for messages, and return if no records are received within the given period.
+  #
+  # @param timeout[Fixnum] the number of seconds to wait before returning if no messages appear on the queue
+  # @yield [String, String] the receipt identifier and contents of the message body
+  # @return [void]
+  def poll(timeout = DEFAULT_TIMEOUT_SECONDS)
+    poller = ::Aws::SQS::QueuePoller.new(@queue_url)
+    # SDK v2 automatically deletes messages if the block returns normally, but we want it to happen manually
+    # from the caller.
+    poller.poll(max_number_of_messages: BATCH_RECEIVE_SIZE, skip_delete: true, 
+      idle_timeout: timeout.to_i, wait_time_seconds: timeout.to_i) do | sqs_messages |
+      
+      sqs_messages.each do | sqs_message |
+        yield [sqs_message.receipt_handle, sqs_message.body]
+      end
+    
+    end
+  end
+  
+  # Send a message to the backing queue
+  #
+  # @param message_body[String] the message to send
+  # @param kwargs_for_send[Hash] additional arguments for the submit (such as `delay_seconds`).
+  # Passes the arguments to the AWS SDK. 
+  # @return [void]
+  def send_message(message_body, **kwargs_for_send)
+    client = ::Aws::SQS::Client.new
+    client.send_message(queue_url: @queue_url, message_body: message_body, **kwargs_for_send)
+  end
+  
+  # Deletes a message after it has been succesfully decoded and processed
+  #
+  # @param message_identifier[String] the ID of the message to delete. For SQS, it is the receipt handle
+  # @return [void]
+  def delete_message(message_identifier)
+    client = ::Aws::SQS::Client.new
+    client.delete_message(queue_url: @queue_url, receipt_handle: message_identifier)
+  end
+end

data/lib/sqewer/contrib/appsignal_wrapper.rb

@@ -0,0 +1,29 @@
+module Sqewer
+  module Contrib
+    # Can be used as a wrapper middleware in an ExecutionContext to log exceptions
+    # to Appsignal and to monitor performance. Will only activate
+    # if the Appsignal gem is loaded within the current process and active.
+    class AppsignalWrapper
+      # Unserialize the job
+      def around_deserialization(serializer, msg_id, msg_payload)
+        return yield unless (defined?(Appsignal) && Appsignal.active?)
+    
+        Appsignal.monitor_transaction('perform_job.demarshal', 
+          :class => serializer.class.to_s, :params => {:recepit_handle => msg_id}, :method => 'deserialize') do
+          yield
+        end
+      end
+  
+      # Run the job with Appsignal monitoring.
+      def around_execution(job, context)
+        return yield unless (defined?(Appsignal) && Appsignal.active?)
+        
+        Appsignal.monitor_transaction('perform_job.sqewer', 
+          :class => job.class.to_s, :params => job.to_h, :method => 'run') do |t|
+            context['appsignal.transaction'] = t
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/sqewer/contrib/performable.rb

@@ -0,0 +1,23 @@
+module Sqewer
+  module Contrib
+    # A job class that can be used to adapt Jobs from ActiveJob and friends. They use
+    # the `perform` method which gets the arguments.
+    class Performable
+      def initialize(performable_class:, perform_arguments:)
+        @class, @args = performable_class, perform_arguments
+      end
+      
+      def to_h
+        {performable_class: @class, perform_arguments: @args}
+      end
+      
+      def inspect
+        '<%s{%s}>' % [@class, @args.inspect]
+      end
+      
+      def run(context)
+        Kernel.const_get(@class).perform(*@args)
+      end
+    end
+  end
+end

data/lib/sqewer/execution_context.rb

@@ -0,0 +1,55 @@
+require 'logger'
+
+# Is passed to each Job when executing (is the argument for the `#run` method
+# of the job). The job can use this object to submit extra jobs, or to get
+# at the things specific for the execution context (database/key-value store
+# connections, error handling transaction and so on).
+class Sqewer::ExecutionContext
+  # Create a new ExecutionContext with an environment hash.
+  #
+  # @param submitter[Sqewer::Submitter] the object to submit new jobs through. Used when jobs want to submit jobs
+  # @param extra_variables[Hash] any extra data to pass around to each Job
+  def initialize(submitter, extra_variables={})
+    @submitter = submitter
+    @params = {}
+    extra_variables.each_pair{|k, v| self[k]  = v }
+  end
+  
+  # Submits one or more jobs to the queue
+  def submit!(*jobs, **execution_options)
+    @submitter.submit!(*jobs, **execution_options)
+  end
+  
+  # Sets a key in the execution environment
+  #
+  # @param key[#to_s] the key to set
+  # @param value the value to set
+  def []=(key, value)
+    @params[key.to_s] = value
+  end
+  
+  # Returns a key of the execution environment by name
+  #
+  # @param key[#to_s] the key to get
+  def [](key)
+    @params[key.to_s]
+  end
+  
+  # Returns a key of the execution environment, or executes the given block
+  # if the key is not set
+  #
+  # @param key[#to_s] the key to get
+  # @param blk the block to execute if no such key is present
+  def fetch(key, &blk)
+    @params.fetch(key.to_s, &blk)
+  end
+  
+  # Returns the logger set in the execution environment, or
+  # the NullLogger if no logger is set. Can be used to supply
+  # a logger prefixed with job parameters per job.
+  #
+  # @return [Logger] the logger to send messages to.
+  def logger
+    @params.fetch('logger') { Sqewer::NullLogger }
+  end
+end

data/lib/sqewer/isolator.rb

@@ -0,0 +1,33 @@
+# Used to isolate the execution environment of the jobs. You can use it to run each
+# job in a separate process (a-la Resque) or stick to the default of running those jobs
+# in threads (a-la Sidekiq).
+class Sqewer::Isolator
+  # Used for running each job in a separate process.
+  class PerProcess
+    # The method called to isolate a particular job flow (both instantiation and execution)
+    def isolate
+      require 'exceptional_fork'
+      ExceptionalFork.fork_and_wait { yield }
+    end
+  end
+    
+  # Returns the Isolator that runs each job unserialization and execution
+  # as a separate process, and then ensures that that process quits cleanly.
+  #
+  # @return [Sqewer::Isolator::PerProcess] the isolator
+  def self.process
+    @per_process ||= PerProcess.new
+  end
+  
+  # Returns the default Isolator that just wraps the instantiation/execution block
+  #
+  # @return [Sqewer::Isolator] the isolator
+  def self.default
+    @default ||= new
+  end
+  
+  # The method called to isolate a particular job flow (both instantiation and execution)
+  def isolate
+    yield
+  end
+end

data/lib/sqewer/middleware_stack.rb

@@ -0,0 +1,44 @@
+# Allows arbitrary wrapping of the job deserialization and job execution procedures
+class Sqewer::MiddlewareStack
+  
+  # Returns the default middleware stack, which is empty (an instance of None).
+  #
+  # @return [MiddlewareStack] the default empty stack
+  def self.default
+    @instance ||= new
+  end
+  
+  # Creates a new MiddlewareStack. Once created, handlers can be added using `:<<`
+  def initialize
+    @handlers = []
+  end
+  
+  # Adds a handler. The handler should respond to :around_deserialization and #around_execution.
+  #
+  # @param handler[#around_deserializarion, #around_execution] The middleware item to insert
+  # @return [void]
+  def <<(handler)
+    @handlers << handler
+    # TODO: cache the wrapping proc
+  end
+  
+  def around_execution(job, context, &inner_block)
+    return yield if @handlers.empty?
+    
+    responders = @handlers.select{|e| e.respond_to?(:around_execution) }
+    responders.reverse.inject(inner_block) {|outer_block, middleware_object|
+      ->{ 
+        middleware_object.public_send(:around_execution, job, context, &outer_block)
+      }
+    }.call
+  end
+
+  def around_deserialization(serializer, message_id, message_body, &inner_block)
+    return yield if @handlers.empty?
+    
+    responders = @handlers.select{|e| e.respond_to?(:around_deserialization) }
+    responders.reverse.inject(inner_block) {|outer_block, middleware_object|
+      ->{ middleware_object.public_send(:around_deserialization, serializer, message_id, message_body, &outer_block) }
+    }.call
+  end
+end

data/lib/sqewer/null_logger.rb

@@ -0,0 +1,9 @@
+require 'logger'
+
+module Sqewer::NullLogger
+  (Logger.instance_methods- Object.instance_methods).each do | null_method |
+    define_method(null_method){|*a| }
+  end
+  
+  extend self
+end

data/lib/sqewer/serializer.rb

@@ -0,0 +1,71 @@
+# Converts jobs into strings that can be sent to the job queue, and
+# restores jobs from those strings. If you want to use, say, Marshal
+# to store your jobs instead of the default, or if you want to generate
+# custom job objects from S3 bucket notifications, you might want to override this
+# class and feed the overridden instance to {Sqewer::Worker}.
+class Sqewer::Serializer
+  
+  # Returns the default Serializer, of which we store one instance
+  # (because the serializer is stateless).
+  #
+  # @return [Serializer] the instance of the default JSON serializer
+  def self.default
+    @instance ||= new
+  end
+  
+  AnonymousJobClass = Class.new(StandardError)
+  ArityMismatch = Class.new(ArgumentError)
+  
+  # Instantiate a Job object from a message body string. If the
+  # returned result is `nil`, the job will be skipped.
+  #
+  # @param message_body[String] a string in JSON containing the job parameters
+  # @return [#run, NilClass] an object that responds to `run()` or nil.
+  def unserialize(message_body)
+    job_ticket_hash = JSON.parse(message_body, symbolize_names: true)
+    raise "Job ticket must unmarshal into a Hash" unless job_ticket_hash.is_a?(Hash)
+    
+    job_ticket_hash = convert_old_ticket_format(job_ticket_hash) if job_ticket_hash[:job_class]
+    
+    # Use fetch() to raise a descriptive KeyError if none
+    job_class_name = job_ticket_hash.delete(:_job_class)
+    raise ":_job_class not set in the ticket" unless job_class_name
+    job_class = Kernel.const_get(job_class_name)
+    
+    job_params = job_ticket_hash.delete(:_job_params)
+    if job_params.nil? || job_params.empty?
+      job_class.new # no args
+    else
+      begin
+        job_class.new(**job_params) # The rest of the message are keyword arguments for the job
+      rescue ArgumentError => e
+        raise ArityMismatch, "Could not instantiate #{job_class} because it did not accept the arguments #{job_params.inspect}"
+      end
+    end
+  end
+  
+  # Converts the given Job into a string, which can be submitted to the queue
+  #
+  # @param job[#to_h] an object that supports `to_h`
+  # @return [String] serialized string ready to be put into the queue
+  def serialize(job)
+    job_class_name = job.class.to_s
+    
+    begin
+      Kernel.const_get(job_class_name)
+    rescue NameError
+      raise AnonymousJobClass, "The class of #{job.inspect} could not be resolved and will not restore to a Job"
+    end
+    
+    job_params = job.respond_to?(:to_h) ? job.to_h : nil
+    job_ticket_hash = {_job_class: job_class_name, _job_params: job_params}
+    JSON.dump(job_ticket_hash)
+  end
+  
+  private
+  
+  def convert_old_ticket_format(hash_of_properties)
+    job_class = hash_of_properties.delete(:job_class)
+    {_job_class: job_class, _job_params: hash_of_properties}
+  end
+end

data/lib/sqewer/simple_job.rb

@@ -0,0 +1,78 @@
+# A module that you can include into your Job class.
+# It adds the following features:
+#
+# * initialize() will have keyword access to all accessors, and will ensure you have called each one of them
+# * to_h() will produce a symbolized Hash with all the properties defined using attr_accessor, and the job_class_name
+# * inspect() will provide a sensible default string representation for logging
+module Sqewer::SimpleJob
+  UnknownJobAttribute = Class.new(StandardError)
+  MissingAttribute = Class.new(StandardError)
+  
+  EQ_END = /(\w+)(\=)$/
+  
+  # Returns the list of methods on the object that have corresponding accessors.
+  # This is then used by #inspect to compose a list of the job parameters, formatted
+  # as an inspected Hash.
+  #
+  # @return [Array<Symbol>] the array of attributes to show via inspect 
+  def inspectable_attributes
+    # All the attributes that have accessors
+    methods.grep(EQ_END).map{|e| e.to_s.gsub(EQ_END, '\1')}.map(&:to_sym)
+  end
+  
+  # Returns the inspection string with the job and all of it's instantiation keyword attributes.
+  # If `inspectable_attributes` has been overridden, the attributes returned by that method will be the
+  # ones returned in the inspection string.
+  #
+  #     j = SomeJob.new(retries: 4, param: 'a')
+  #     j.inspect #=> "<SomeJob:{retries: 4, param: \"a\"}>"
+  #
+  # @return [String] the object inspect string
+  def inspect
+    key_attrs = inspectable_attributes
+    hash_repr = to_h
+    h = key_attrs.each_with_object({}) do |k, o|
+      o[k] = hash_repr[k]
+    end
+    "<#{self.class}:#{h.inspect}>"
+  end
+  
+  # Initializes a new Job with the given job args. Will check for presence of
+  # accessor methods for each of the arguments, and call them with the arguments given.
+  #
+  # If one of the accessors was not triggered during the call, an exception will be raised
+  # (because you most likely forgot a parameter for a job, or the job class changed whereas
+  # the queue still contains jobs in old formats).
+  #
+  # @param jobargs[Hash] the keyword arguments, mapping 1 to 1 to the accessors of the job
+  def initialize(**jobargs)
+    @simple_job_args = jobargs.keys
+    touched_attributes = Set.new
+    jobargs.each do |(k,v)|
+      
+      accessor = "#{k}="
+      touched_attributes << k
+      unless respond_to?(accessor)
+        raise UnknownJobAttribute, "Unknown attribute #{k.inspect} for #{self.class}" 
+      end
+      
+      send("#{k}=", v)
+    end
+    
+    accessors = methods.grep(EQ_END).map{|method_name| method_name.to_s.gsub(EQ_END, '\1').to_sym }
+    settable_attributes = Set.new(accessors)
+    missing_attributes = settable_attributes - touched_attributes
+    
+    missing_attributes.each do | attr |
+      raise MissingAttribute, "Missing job attribute #{attr.inspect}"
+    end
+  end
+  
+  def to_h
+    keys_and_values = @simple_job_args.each_with_object({}) do |k, h|
+      h[k] = send(k)
+    end
+    
+    keys_and_values
+  end
+end

data/lib/sqewer/submitter.rb

@@ -0,0 +1,18 @@
+# A shim for submitting jobs to the queue. Accepts a connection
+# (something that responds to `#send_message`)
+# and the serializer (something that responds to `#serialize`) to
+# convert the job into the string that will be put in the queue.
+class Sqewer::Submitter < Struct.new(:connection, :serializer)
+  
+  # Returns a default Submitter, configured with the default connection
+  # and the default serializer.
+  def self.default
+    new(Sqewer::Connection.default, Sqewer::Serializer.default)
+  end
+  
+  def submit!(*jobs, **kwargs_for_send)
+    jobs.each do | job |
+      connection.send_message(serializer.serialize(job), **kwargs_for_send)
+    end
+  end
+end

data/lib/sqewer/version.rb

@@ -0,0 +1,3 @@
+module Sqewer
+  VERSION = '1.0.0'
+end

data/lib/sqewer/worker.rb

@@ -0,0 +1,200 @@
+require 'logger'
+require 'thread'
+require 'very_tiny_state_machine'
+require 'fiber'
+
+# A massively threaded worker engine
+class Sqewer::Worker
+  DEFAULT_NUM_THREADS = 4
+  SLEEP_SECONDS_ON_EMPTY_QUEUE = 1
+  THROTTLE_FACTOR = 2
+  
+  # Returns the default Worker instance, configured based on the default components
+  #
+  # @return [Sqewer::Worker]
+  def self.default
+    @default ||= new
+  end
+  
+  # Creates a new Worker. The Worker, unlike it is in the Rails tradition, is only responsible for
+  # the actual processing of jobs, and not for the job arguments.
+  #
+  # @param connection[Sqewer::Connection] the object that handles polling and submitting
+  # @param serializer[#serialize, #unserialize] the serializer/unserializer for the jobs
+  # @param execution_context_class[Class] the class for the execution context (will be instantiated by 
+  # the worker for each job execution)
+  # @param submitter_class[Class] the class used for submitting jobs (will be instantiated by the worker for each job execution)
+  # @param middleware_stack[Sqewer::MiddlewareStack] the middleware stack that is going to be used
+  # @param logger[Logger] the logger to log execution to and to pass to the jobs
+  # @param isolator[Sqewer::Isolator] the isolator to encapsulate job instantiation and execution, if desired
+  # @param num_threads[Fixnum] how many worker threads to spawn
+  def initialize(connection: Sqewer::Connection.default,
+      serializer: Sqewer::Serializer.default,
+      execution_context_class: Sqewer::ExecutionContext,
+      submitter_class: Sqewer::Submitter,
+      middleware_stack: Sqewer::MiddlewareStack.default,
+      logger: Logger.new($stderr),
+      isolator: Sqewer::Isolator.default,
+      num_threads: DEFAULT_NUM_THREADS)
+    
+    @logger = logger
+    @connection = connection
+    @serializer = serializer
+    @middleware_stack = middleware_stack
+    @execution_context_class = execution_context_class
+    @submitter_class = submitter_class
+    @isolator = isolator
+    @num_threads = num_threads
+    
+    raise ArgumentError, "num_threads must be > 0" unless num_threads > 0
+    
+    @execution_counter = Sqewer::AtomicCounter.new
+    
+    @state = VeryTinyStateMachine.new(:stopped)
+    @state.permit_state :starting, :running, :stopping, :stopped, :failed
+    @state.permit_transition :stopped => :starting, :starting => :running, :running => :stopping, :stopping => :stopped
+    @state.permit_transition :starting => :failed # Failed to start
+  end
+  
+  # Start listening on the queue, spin up a number of consumer threads that will execute the jobs.
+  #
+  # @param num_threads[Fixnum] the number of consumer/executor threads to spin up
+  # @return [void]
+  def start
+    @state.transition! :starting
+    
+    Thread.abort_on_exception = true
+
+    @logger.info { '[worker] Starting with %d consumer threads' % @num_threads }
+    @execution_queue = Queue.new
+    
+    consumers = (1..@num_threads).map do
+      Thread.new do
+        loop { 
+          break if @state.in_state?(:stopping)
+          take_and_execute
+        }
+      end
+    end
+    
+    # Create a fiber-based provider thread. When the execution queue is exhausted, use
+    # the fiber to take a new job and place it on the queue. We use a fiber to have a way
+    # to "suspend" the polling loop in the SQS client when the local buffer queue fills up.
+    provider = Thread.new do
+      feeder_fiber = Fiber.new do
+        loop do
+          break if @state.in_state?(:stopping)
+          @connection.poll do |message_id, message_body| 
+            break if @state.in_state?(:stopping)
+            Fiber.yield([message_id, message_body])
+          end
+        end
+      end
+      
+      loop do
+        break if !feeder_fiber.alive?
+        break if stopping?
+        
+        if @execution_queue.length < (@num_threads * THROTTLE_FACTOR)
+          @execution_queue << feeder_fiber.resume
+        else
+          @logger.debug "Suspending poller (%d items buffered)" % @execution_queue.length
+          sleep 0.2
+          Thread.pass
+        end 
+      end
+    end
+    
+    # It makes sense to have one GC caller per process, since a GC cuts across threads.
+    # We will perform a full GC cycle after the same number of jobs as our consumer thread
+    # count - so not on every job, but still as often as we can to keep the memory use in check.
+    gc = Thread.new do
+      loop do
+        break if stopping?
+        GC.start if (@execution_counter.to_i % @num_threads).zero?
+        sleep 0.5
+      end
+    end
+    
+    @threads = [provider, gc] + consumers
+    
+    # If any of our threads are already dead, it means there is some misconfiguration and startup failed
+    if @threads.any?{|t| !t.alive? }
+      @threads.map(&:kill)
+      @state.transition! :failed
+      @logger.fatal { '[worker] Failed to start (one or more threads died on startup)' }
+    else
+      @state.transition! :running
+      @logger.info { '[worker] Started, %d consumer threads' % consumers.length }
+    end
+  end
+  
+  # Attempts to softly stop the running consumers and the producer. Once the call is made,
+  # all the threads will stop at their next loop iteration.
+  def stop
+    @state.transition! :stopping
+    @logger.info { '[worker] Stopping (clean shutdown), will wait for threads to terminate'}
+    @threads.map(&:join)
+    @logger.info { '[worker] Stopped'}
+    @state.transition! :stopped
+  end
+  
+  # Peforms a hard shutdown by killing all the threads
+  def kill
+    @state.transition! :stopping
+    @logger.info { '[worker] Killing (unclean shutdown), will kill all threads'}
+    @threads.map(&:kill)
+    @logger.info { '[worker] Stopped'}
+    @state.transition! :stopped
+  end
+  
+  private
+  
+  def stopping?
+    @state.in_state?(:stopping)
+  end
+  
+  
+  def take_and_execute
+    message_id, message_body = @execution_queue.pop(nonblock=true)
+    return unless message_id
+    return @connection.delete_message(message_id) unless message_body && !message_body.empty?
+    
+    @isolator.isolate do
+      job = @middleware_stack.around_deserialization(@serializer, message_id, message_body) do
+        @serializer.unserialize(message_body)
+      end
+      
+      if job # if the serializer returns a nil or false
+        t = Time.now
+        submitter = @submitter_class.new(@connection, @serializer)
+        context = @execution_context_class.new(submitter, {STR_logger => @logger})
+        
+        begin
+          @middleware_stack.around_execution(job, context) do
+            job.method(:run).arity.zero? ? job.run : job.run(context)
+          end
+          @logger.info { "[worker] Finished #{job.inspect} in %0.2fs" % (Time.now - t) }
+        rescue => e
+          @logger.error { "[worker] Failed #{job.inspect} with a #{e}" }
+          raise e
+        end
+      end
+    end
+    
+    @connection.delete_message(message_id)
+  rescue ThreadError # Queue is empty
+    sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
+    Thread.pass
+  rescue SystemExit, SignalException, Interrupt => e # Time to quit
+    @logger.error { "[worker] Signaled, will quit the consumer" }
+    return
+  rescue => e # anything else, at or below StandardError that does not need us to quit
+    @logger.fatal { "[worker] Failed #{message_id} with #{e}" }
+    @logger.fatal(e.class)
+    @logger.fatal(e.message)
+    e.backtrace.each { |s| @logger.fatal{"\t#{s}"} }
+  end
+  
+  STR_logger = 'logger'
+end