sqewer 4.2.0 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d580e81c48aeb4f31bf0069c603c3cee0413c51d
-  data.tar.gz: 87d546de3454ff297750d70d355a71c45340cc0a
+  metadata.gz: 3fe9d14faeab2b77f5551ae617cf50b24b803698
+  data.tar.gz: 10973f30ea8433ebc3f0cdc2a4b8ae8705a24476
 SHA512:
-  metadata.gz: 4be05bef37befcb1e3f49021be38af632bbb4af52bfac17a4bc66fa9f81bca65406416538a59e6c94faac584f6d60884e43a46547cc5c96a9e900cd8eef7532e
-  data.tar.gz: 9bcf12c6ff462a7ca49659077193c1b95568872c4dd38f8444899000434cf8ce83d79e555bfd7e6226c4b18f9b2fefb72cc3feec28687b65c3047fe0b7858965
+  metadata.gz: bc139a5322ae65835d5ebdd0d95568e36f5d54a029340a6b834cd8528b64168c58d80e9b62790707b7eb69bdd9616cb20e781119f4d0e6cc713c82c226749bfe
+  data.tar.gz: 52139badc74df3a2b208415063c5f336b98c294b630abbf95d20165882a01a6908d69139d73ba5e73ecbc52d0b595d1c73de8ded5b2b6ea433f0da87d0ea78d9

data/DETAILS.md

@@ -171,13 +171,26 @@ started with your custom Worker of choice:
 
 ## Threads versus processes
 
-You can elect to use process isolation per job. Each job will then run in a forked process, on which
-the main worker process will `wait()`. To do so, pass `Sqewer::Isolator.process` as the `isolator:`
-option to the Worker
+sqewer uses threads. If you need to run your job from a forked subprocess (primarily for memory
+management reasons) you can do so from the `run` method. Note that you might need to apply extra gymnastics
+to submit extra jobs in this case, as it is the job of the controlling worker thread to submit the messages
+you generate. For example, you could use a pipe. But in a more general case something like this can be used:
 
-    proc_worker = Sqewer::Worker.new(isolator: Sqewer::Isolator.process)
-
-By default the system is working with threads only, as processes obviously have some overhead.
+    class MyJob
+      def run
+        pid = fork do
+          SomeRemoteService.reconnect # you are in the child process now
+          ActiveRAMGobbler.fetch_stupendously_many_things.each do |...|
+          end
+        end
+        
+        _, status = Process.wait2(pid)
+        
+        # Raise an error in the parent process to signal Sqewer that the job failed
+        # if the child exited with a non-0 status
+        raise "Child process crashed" unless status.exitstatus && status.exitstatus.zero?
+      end
+    end
 
 ## Execution and serialization wrappers (middleware)
 

data/lib/sqewer/version.rb

@@ -1,3 +1,3 @@
 module Sqewer
-  VERSION = '4.2.0'
+  VERSION = '5.0.0'
 end

data/lib/sqewer/worker.rb

@@ -27,9 +27,6 @@ class Sqewer::Worker
   # @return [Class] The class used to create the Submitter used by jobs to spawn other jobs
   attr_reader :submitter_class
   
-  # @return [#perform] The isolator to use when executing each job
-  attr_reader :isolator
-  
   # @return [Array<Thread>] all the currently running threads of the Worker
   attr_reader :threads
   
@@ -53,7 +50,6 @@ class Sqewer::Worker
   # @param submitter_class[Class] the class used for submitting jobs (will be instantiated by the worker for each job execution)
   # @param middleware_stack[Sqewer::MiddlewareStack] the middleware stack that is going to be used
   # @param logger[Logger] the logger to log execution to and to pass to the jobs
-  # @param isolator[Sqewer::Isolator] the isolator to encapsulate job instantiation and execution, if desired
   # @param num_threads[Fixnum] how many worker threads to spawn
   def initialize(connection: Sqewer::Connection.default,
       serializer: Sqewer::Serializer.default,
@@ -61,7 +57,6 @@ class Sqewer::Worker
       submitter_class: Sqewer::Submitter,
       middleware_stack: Sqewer::MiddlewareStack.default,
       logger: Logger.new($stderr),
-      isolator: Sqewer::Isolator.default,
       num_threads: DEFAULT_NUM_THREADS)
     
     @logger = logger
@@ -70,7 +65,6 @@ class Sqewer::Worker
     @middleware_stack = middleware_stack
     @execution_context_class = execution_context_class
     @submitter_class = submitter_class
-    @isolator = isolator
     @num_threads = num_threads
     
     @threads = []
@@ -189,9 +183,33 @@ class Sqewer::Worker
   
   def handle_message(message)
     return unless message.receipt_handle
-    return @connection.delete_message(message.receipt_handle) unless message.has_body?
-    @isolator.perform(self, message)
-    # The message delete happens within the Isolator
+    
+    # Create a messagebox that buffers all the calls to Connection, so that
+    # we can send out those commands in one go (without interfering with senders
+    # on other threads, as it seems the Aws::SQS::Client is not entirely
+    # thread-safe - or at least not it's HTTP client part).
+    box = Sqewer::ConnectionMessagebox.new(connection)
+    return box.delete_message(message.receipt_handle) unless message.has_body?
+  
+    job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body) do
+      serializer.unserialize(message.body)
+    end
+    return unless job
+  
+    submitter = submitter_class.new(box, serializer)
+    context = execution_context_class.new(submitter, {'logger' => logger})
+  
+    t = Time.now
+    middleware_stack.around_execution(job, context) do
+      job.method(:run).arity.zero? ? job.run : job.run(context)
+    end
+    box.delete_message(message.receipt_handle)
+  
+    delta = Time.now - t
+    logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
+  ensure
+    n_flushed = box.flush!
+    logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
   end
   
   def take_and_execute
@@ -204,4 +222,35 @@ class Sqewer::Worker
     @logger.error { '[worker] Failed "%s..." with %s: %s' % [message.inspect[0..32], e.class, e.message] }
     e.backtrace.each { |s| @logger.error{"\t#{s}"} }
   end
+  
+  def perform(message)
+    # Create a messagebox that buffers all the calls to Connection, so that
+    # we can send out those commands in one go (without interfering with senders
+    # on other threads, as it seems the Aws::SQS::Client is not entirely
+    # thread-safe - or at least not it's HTTP client part).
+    box = Sqewer::ConnectionMessagebox.new(connection)
+    
+    job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body) do
+      serializer.unserialize(message.body)
+    end
+    return unless job
+    
+    submitter = submitter_class.new(box, serializer)
+    context = execution_context_class.new(submitter, {'logger' => logger})
+    
+    t = Time.now
+    middleware_stack.around_execution(job, context) do
+      job.method(:run).arity.zero? ? job.run : job.run(context)
+    end
+    
+    # Perform two flushes, one for any possible jobs the job has spawned,
+    # and one for the job delete afterwards
+    box.delete_message(message.receipt_handle)
+    
+    delta = Time.now - t
+    logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
+  ensure
+    n_flushed = box.flush!
+    logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
+  end
 end

data/spec/sqewer/worker_spec.rb

@@ -8,7 +8,7 @@ describe Sqewer::Worker, :sqs => true do
   
   it 'has all the necessary attributes' do
     attrs = [:logger, :connection, :serializer, :middleware_stack, 
-      :execution_context_class, :submitter_class, :isolator, :num_threads]
+      :execution_context_class, :submitter_class, :num_threads]
     default_worker = described_class.default
     attrs.each do | attr_name |
       expect(default_worker).to respond_to(attr_name)
@@ -106,38 +106,5 @@ describe Sqewer::Worker, :sqs => true do
         worker.stop
       end
     end
-    
-    it 'sets up the processing pipeline so that jobs can execute in sequence (with processes)' do
-      class SecondaryJob
-        def run
-          File.open('secondary-job-run','w') {}
-        end
-      end
-      
-      class InitialJob
-        def run(executor)
-          File.open('initial-job-run','w') {}
-          executor.submit!(SecondaryJob.new)
-        end
-      end
-      
-      payload = JSON.dump({job_class: 'InitialJob'})
-      client = Aws::SQS::Client.new
-      client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
-      
-      worker = described_class.new(logger: test_logger, num_threads: 8, isolator: Sqewer::Isolator.process)
-      
-      worker.start
-      
-      begin
-        wait_for { File.exist?('initial-job-run') }.to eq(true)
-        wait_for { File.exist?('secondary-job-run') }.to eq(true)
-        
-        File.unlink('initial-job-run')
-        File.unlink('secondary-job-run')
-      ensure
-        worker.stop
-      end
-    end
   end
 end

data/sqewer.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: sqewer 4.2.0 ruby lib
+# stub: sqewer 5.0.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "sqewer"
-  s.version = "4.2.0"
+  s.version = "5.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Julik Tarkhanov"]
-  s.date = "2016-02-12"
+  s.date = "2016-02-13"
   s.description = "Process jobs from SQS"
   s.email = "me@julik.nl"
   s.extra_rdoc_files = [
@@ -35,7 +35,6 @@ Gem::Specification.new do |s|
     "lib/sqewer/contrib/appsignal_wrapper.rb",
     "lib/sqewer/contrib/performable.rb",
     "lib/sqewer/execution_context.rb",
-    "lib/sqewer/isolator.rb",
     "lib/sqewer/middleware_stack.rb",
     "lib/sqewer/null_logger.rb",
     "lib/sqewer/serializer.rb",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqewer
 version: !ruby/object:Gem::Version
-  version: 4.2.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-12 00:00:00.000000000 Z
+date: 2016-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -216,7 +216,6 @@ files:
 - lib/sqewer/contrib/appsignal_wrapper.rb
 - lib/sqewer/contrib/performable.rb
 - lib/sqewer/execution_context.rb
-- lib/sqewer/isolator.rb
 - lib/sqewer/middleware_stack.rb
 - lib/sqewer/null_logger.rb
 - lib/sqewer/serializer.rb

data/lib/sqewer/isolator.rb

@@ -1,72 +0,0 @@
-# Used to isolate the execution environment of the jobs. You can use it to run each
-# job in a separate process (a-la Resque) or stick to the default of running those jobs
-# in threads (a-la Sidekiq).
-class Sqewer::Isolator
-  # Used for running each job in a separate process.
-  class PerProcess < self
-    # The method called to isolate a particular job flow (both instantiation and execution)
-    #
-    # @see {Isolator#perform}
-    def perform(*)
-      require 'exceptional_fork' unless defined?(ExceptionalFork)
-      ExceptionalFork.fork_and_wait { super }
-    end
-  end
-    
-  # Returns the Isolator that runs each job unserialization and execution
-  # as a separate process, and then ensures that that process quits cleanly.
-  #
-  # @return [Sqewer::Isolator::PerProcess] the isolator
-  def self.process
-    @per_process ||= PerProcess.new
-  end
-  
-  # Returns the default Isolator that just wraps the instantiation/execution block
-  #
-  # @return [Sqewer::Isolator] the isolator
-  def self.default
-    @default ||= new
-  end
-  
-  # The method called to isolate a particular job flow (both instantiation and execution)
-  #
-  # @param worker[Sqewer::Worker] the worker that is running the jobs
-  # @param message[Sqewer::Connection::Message] the message that is being processed
-  def perform(worker, message)
-    submitter_class = worker.submitter_class
-    execution_context_class = worker.execution_context_class
-    middleware_stack = worker.middleware_stack
-    connection = worker.connection
-    logger = worker.logger
-    serializer = worker.serializer
-    
-    # Create a messagebox that buffers all the calls to Connection, so that
-    # we can send out those commands in one go (without interfering with senders
-    # on other threads, as it seems the Aws::SQS::Client is not entirely
-    # thread-safe - or at least not it's HTTP client part).
-    box = Sqewer::ConnectionMessagebox.new(connection)
-    
-    job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body) do
-      serializer.unserialize(message.body)
-    end
-    return unless job
-    
-    submitter = submitter_class.new(box, serializer)
-    context = execution_context_class.new(submitter, {'logger' => logger})
-    
-    t = Time.now
-    middleware_stack.around_execution(job, context) do
-      job.method(:run).arity.zero? ? job.run : job.run(context)
-    end
-    
-    # Perform two flushes, one for any possible jobs the job has spawned,
-    # and one for the job delete afterwards
-    box.delete_message(message.receipt_handle)
-    
-    delta = Time.now - t
-    logger.info { "[worker] Finished %s in %0.2fs" % [job.inspect, delta] }
-  ensure
-    n_flushed = box.flush!
-    logger.debug { "[worker] Flushed %d connection commands" % n_flushed } if n_flushed.nonzero?
-  end
-end