sucker_punch 1.6.0 → 2.0.0.beta1

data/lib/sucker_punch/job.rb

@@ -1,25 +1,68 @@
 module SuckerPunch
+  # Include this module in your job class
+  # to create asynchronous jobs:
+  #
+  # class LogJob
+  #   include SuckerPunch::Job
+  #   workers 4
+  #
+  #   def perform(*args)
+  #     # log the things
+  #   end
+  # end
+  #
+  # To trigger asynchronous job:
+  #
+  #   LogJob.perform_async(1, 2, 3)
+  #   LogJob.perform_in(60, 1, 2, 3) # `perform` will be excuted 60 sec. later
+  #
+  # Note that perform_async is a class method, perform is an instance method.
   module Job
     def self.included(base)
-      base.send(:include, ::Celluloid)
       base.extend(ClassMethods)
+      base.class_attribute :num_workers
 
-      base.class_eval do
-        def self.new
-          define_celluloid_pool(self, @workers)
-        end
-      end
+      base.num_workers = 2
+    end
+
+    def logger
+      SuckerPunch.logger
     end
 
     module ClassMethods
+      def perform_async(*args)
+        return unless SuckerPunch::RUNNING.true?
+        queue = SuckerPunch::Queue.find_or_create(self.to_s, num_workers)
+        queue.post(args) { |args| __run_perform(*args) }
+      end
+
+      def perform_in(interval, *args)
+        return unless SuckerPunch::RUNNING.true?
+        queue = SuckerPunch::Queue.find_or_create(self.to_s, num_workers)
+        job = Concurrent::ScheduledTask.execute(interval.to_f, args: args, executor: queue) do
+          __run_perform(*args)
+        end
+        job.pending?
+      end
+
       def workers(num)
-        @workers = num
+        self.num_workers = num
       end
 
-      def define_celluloid_pool(klass, num_workers)
-        SuckerPunch::Queue.new(klass).register(num_workers)
+      def __run_perform(*args)
+        # break if shutdown began while I was waiting in the queue
+        return unless SuckerPunch::RUNNING.true?
+
+        SuckerPunch::Counter::Busy.new(self.to_s).increment
+        result = self.new.perform(*args)
+        SuckerPunch::Counter::Processed.new(self.to_s).increment
+        result
+      rescue => ex
+        SuckerPunch::Counter::Failed.new(self.to_s).increment
+        SuckerPunch.exception_handler.call(ex, self, args)
+      ensure
+        SuckerPunch::Counter::Busy.new(self.to_s).decrement
       end
     end
-
   end
 end

data/lib/sucker_punch/queue.rb

@@ -1,57 +1,182 @@
-require 'thread'
+require 'forwardable'
 
 module SuckerPunch
-  class Queue
-    attr_reader :klass
+  class Queue < Concurrent::Synchronization::LockableObject
+    extend Forwardable
+    include Concurrent::ExecutorService
 
-    DEFAULT_OPTIONS = { workers: 2 }
-    PREFIX = "sucker_punch"
-    class MaxWorkersExceeded < StandardError; end
-    class NotEnoughWorkers < StandardError; end
+    DEFAULT_EXECUTOR_OPTIONS = {
+      min_threads:     2,
+      max_threads:     2,
+      idletime:        60, # 1 minute
+      max_queue:       0, # unlimited
+      auto_terminate:  false # Let shutdown modes handle thread termination
+    }.freeze
 
-    def self.find(klass)
-      queue = self.new(klass)
-      Celluloid::Actor[queue.name]
+    QUEUES = Concurrent::Map.new
+
+    def self.find_or_create(name, num_workers = 2)
+      pool = QUEUES.fetch_or_store(name) do
+        options = DEFAULT_EXECUTOR_OPTIONS.merge({
+          min_threads: num_workers,
+          max_threads: num_workers
+        })
+        Concurrent::ThreadPoolExecutor.new(options)
+      end
+
+      new(name, pool)
+    end
+
+    def self.all
+      queues = Concurrent::Array.new
+      QUEUES.each_pair do |name, pool|
+        queues.push new(name, pool)
+      end
+      queues
     end
 
-    def initialize(klass)
-      @klass = klass
-      @mutex = Mutex.new
+    def self.clear
+      # susceptible to race conditions--only use in testing
+      old = all
+      QUEUES.clear
+      SuckerPunch::Counter::Busy.clear
+      SuckerPunch::Counter::Processed.clear
+      SuckerPunch::Counter::Failed.clear
+      old.each { |queue| queue.kill }
+    end
+
+    def self.stats
+      queues = {}
+
+      all.each do |queue|
+        queues[queue.name] = {
+          "workers" => {
+            "total" => queue.total_workers,
+            "busy" => queue.busy_workers,
+            "idle" => queue.idle_workers,
+          },
+          "jobs" => {
+            "processed" => queue.processed_jobs,
+            "failed" => queue.failed_jobs,
+            "enqueued" => queue.enqueued_jobs,
+          }
+        }
+      end
+
+      queues
     end
 
-    def register(num_workers = DEFAULT_OPTIONS[:workers])
-      num_workers ||= DEFAULT_OPTIONS[:workers]
-      raise MaxWorkersExceeded if num_workers > 200
-      raise NotEnoughWorkers if num_workers < 1
+    def self.shutdown_all
+      if SuckerPunch::RUNNING.make_false
+        SuckerPunch.logger.info("Shutdown triggered...executing remaining in-process jobs")
 
-      @mutex.synchronize {
-        unless registered?
-          initialize_celluloid_pool(num_workers)
+        queues = all
+        latch = Concurrent::CountDownLatch.new(queues.length)
+
+        queues.each do |queue|
+          queue.post(latch) { |l| l.count_down }
+          queue.shutdown
+        end
+
+        if latch.wait(SuckerPunch.shutdown_timeout)
+          SuckerPunch.logger.info("Remaining jobs have finished")
+        else
+          queues.each { |queue| queue.kill }
+          SuckerPunch.logger.info("Remaining jobs didn't finish in time...killing remaining jobs")
         end
-      }
-      self.class.find(klass)
+      end
+    end
+
+    attr_reader :name
+
+    def_delegators :@pool,
+      :max_length,
+      :min_length,
+      :length,
+      :queue_length#,
+      #:idletime,
+      #:max_queue,
+      #:largest_length,
+      #:scheduled_task_count,
+      #:completed_task_count,
+      #:can_overflow?,
+      #:remaining_capacity,
+      #:running?,
+      #:shuttingdown?
+
+    alias_method :total_workers, :length
+    alias_method :enqueued_jobs, :queue_length
+
+    def initialize(name, pool)
+      super()
+      @running = true
+      @name, @pool = name, pool
+    end
+
+    def running?
+      synchronize { @running }
     end
 
-    def registered?
-      Celluloid::Actor.registered.include?(name.to_sym)
+    def ==(other)
+      pool == other.pool
     end
 
-    def name
-      klass_name = klass.to_s.underscore
-      "#{PREFIX}_#{klass_name}".to_sym
+    def busy_workers
+      SuckerPunch::Counter::Busy.new(name).value
+    end
+
+    def idle_workers
+      total_workers - busy_workers
+    end
+
+    def processed_jobs
+      SuckerPunch::Counter::Processed.new(name).value
+    end
+
+    def failed_jobs
+      SuckerPunch::Counter::Failed.new(name).value
+    end
+
+    def post(*args, &block)
+      synchronize do
+        if @running
+          @pool.post(*args, &block)
+        else
+          false
+        end
+      end
+    end
+
+    def kill
+      if can_initiate_shutdown?
+        @pool.kill
+      end
+    end
+
+    def shutdown
+      if can_initiate_shutdown?
+        @pool.shutdown
+      end
+    end
+
+    protected
+
+    def pool
+      @pool
     end
 
     private
 
-    def initialize_celluloid_pool(num_workers)
-      pool_class = klass
-      pool_name = name
-      pool = Class.new(Celluloid::Supervision::Container) do
-        pool pool_class, as: pool_name, size: num_workers
+    def can_initiate_shutdown?
+      synchronize do
+        if @running
+          @running = false
+          true
+        else
+          false
+        end
       end
-      pool.run!
     end
   end
 end
 
-

data/lib/sucker_punch/testing/inline.rb

@@ -1,11 +1,37 @@
 require 'sucker_punch'
-require 'celluloid/proxy/abstract'
-require 'celluloid/proxy/sync'
-require 'celluloid/proxy/actor'
 
-class Celluloid::Proxy::Cell < Celluloid::Proxy::Sync
-  def async(method_name = nil, *args, &block)
-    self
+# Include this in your tests to simulate
+# immediate execution of your asynchronous jobs
+#
+# class LogJob
+#   include SuckerPunch::Job
+#
+#   def perform(*args)
+#     # log the things
+#   end
+# end
+#
+# To trigger asynchronous job:
+#
+#   LogJob.perform_async(1, 2, 3)
+#
+# Include inline testing lib:
+#
+# require 'sucker_punch/testing/inline"
+#
+# LogJob.perform_async(1, 2, 3) is now synchronous
+# LogJob.perform_in(1, 2, 3) is now synchronous
+#
+module SuckerPunch
+  module Job
+    module ClassMethods
+      def perform_async(*args)
+        self.new.perform(*args)
+      end
+
+      def perform_in(_, *args)
+        self.new.perform(*args)
+      end
+    end
   end
 end
-

data/lib/sucker_punch/version.rb

@@ -1,3 +1,3 @@
 module SuckerPunch
-  VERSION = "1.6.0"
+  VERSION = "2.0.0.beta1"
 end

data/lib/sucker_punch.rb

@@ -1,21 +1,58 @@
-require 'celluloid/current'
+require 'concurrent'
 require 'sucker_punch/core_ext'
+require 'sucker_punch/counter'
 require 'sucker_punch/job'
 require 'sucker_punch/queue'
 require 'sucker_punch/version'
+require 'logger'
 
 module SuckerPunch
-  def self.logger
-    Celluloid.logger
-  end
+  RUNNING = Concurrent::AtomicBoolean.new(true)
 
-  def self.logger=(logger)
-    Celluloid.logger = logger
-  end
+  class << self
+    def exception_handler=(handler)
+      @exception_handler = handler
+    end
+
+    def exception_handler
+      @exception_handler || method(:default_exception_handler)
+    end
+
+    def default_exception_handler(ex, klass, args)
+      msg = "Sucker Punch job error for class: '#{klass}' args: #{args}\n"
+      msg += "#{ex.class} #{ex}\n"
+      msg += "#{ex.backtrace.nil? ? '' : ex.backtrace.join("\n")}"
+      logger.error msg
+    end
+
+    def logger
+      @logger || default_logger
+    end
 
-  def self.exception_handler(&block)
-    Celluloid.exception_handler(&block)
+    def logger=(log)
+      @logger = (log ? log : Logger.new('/dev/null'))
+    end
+
+    def default_logger
+      l = Logger.new(STDOUT)
+      l.level = Logger::INFO
+      l
+    end
+
+    def shutdown_timeout
+      # 10 seconds on heroku, minus a grace period
+      @shutdown_timeout || 8
+    end
+
+    def shutdown_timeout=(timeout)
+      @shutdown_timeout = timeout
+    end
   end
 end
 
+at_exit do
+  SuckerPunch::Queue.shutdown_all
+  SuckerPunch.logger.info("All is quiet...byebye")
+end
+
 require 'sucker_punch/railtie' if defined?(::Rails)

data/sucker_punch.gemspec

@@ -18,9 +18,11 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_development_dependency "rspec"
-  gem.add_development_dependency "rake"
+  gem.post_install_message = "Sucker Punch Version 2.0 introduces backwards-incompatible changes. Please see https://github.com/brandonhilkert/sucker_punch/blob/master/CHANGES.md#20 for details."
+
+  gem.add_development_dependency "rake", "~> 10.0"
+  gem.add_development_dependency "minitest"
   gem.add_development_dependency "pry"
 
-  gem.add_dependency "celluloid", "~> 0.17.2"
+  gem.add_dependency "concurrent-ruby", "~> 1.0.0"
 end

data/test/sucker_punch/async_syntax_test.rb

@@ -0,0 +1,33 @@
+require 'test_helper'
+
+module SuckerPunch
+  class AsyncSyntaxTest < Minitest::Test
+    def setup
+      require 'sucker_punch/async_syntax'
+      SuckerPunch::Queue.clear
+    end
+
+    def teardown
+      SuckerPunch::Queue.clear
+    end
+
+    def test_perform_async_runs_job_asynchronously
+      arr = Concurrent::Array.new
+      latch = Concurrent::CountDownLatch.new
+      FakeLatchJob.new.async.perform(arr, latch)
+      latch.wait(0.2)
+      assert_equal 1, arr.size
+    end
+
+    private
+
+    class FakeLatchJob
+      include SuckerPunch::Job
+
+      def perform(arr, latch)
+        arr.push true
+        latch.count_down
+      end
+    end
+  end
+end

data/test/sucker_punch/counter_test.rb

@@ -0,0 +1,83 @@
+require 'test_helper'
+
+module SuckerPunch
+  class CounterTest < Minitest::Test
+    def setup
+      @queue = "fake"
+      SuckerPunch::Queue.clear
+    end
+
+    def teardown
+      SuckerPunch::Queue.clear
+      SuckerPunch::Counter::Busy.clear
+      SuckerPunch::Counter::Failed.clear
+      SuckerPunch::Counter::Processed.clear
+    end
+
+    def test_busy_counter_can_be_cleared
+      SuckerPunch::Counter::Busy.new(@queue).increment
+      SuckerPunch::Counter::Busy.clear
+      assert_equal 0, SuckerPunch::Counter::Busy.new(@queue).value
+    end
+
+    def test_same_busy_counter_is_returned
+      c = SuckerPunch::Counter::Busy.new(@queue)
+      assert_equal c.counter, SuckerPunch::Counter::Busy::COUNTER[@queue]
+    end
+
+    def test_busy_counter_default_is_0
+      c = SuckerPunch::Counter::Busy.new(@queue)
+      assert_equal 0, c.value
+    end
+
+    def test_busy_counter_supports_incrementing_and_decrementing
+      c = SuckerPunch::Counter::Busy.new(@queue)
+      assert_equal 1, c.increment
+      assert_equal 0, c.decrement
+    end
+
+    def test_processed_counter_can_be_cleared
+      SuckerPunch::Counter::Processed.new(@queue).increment
+      SuckerPunch::Counter::Processed.clear
+      assert_equal 0, SuckerPunch::Counter::Processed.new(@queue).value
+    end
+
+    def test_same_counter_is_returned_for_processed
+      c = SuckerPunch::Counter::Processed.new(@queue)
+      assert_equal c.counter, SuckerPunch::Counter::Processed::COUNTER[@queue]
+    end
+
+    def test_processed_counter_default_is_0
+      c = SuckerPunch::Counter::Processed.new(@queue)
+      assert_equal 0, c.value
+    end
+
+    def test_processed_counter_supports_incrementing_and_decrementing
+      c = SuckerPunch::Counter::Processed.new(@queue)
+      assert_equal 1, c.increment
+      assert_equal 0, c.decrement
+    end
+
+    def test_failed_counter_can_be_cleared
+      SuckerPunch::Counter::Failed.new(@queue).increment
+      SuckerPunch::Counter::Failed.clear
+      assert_equal 0, SuckerPunch::Counter::Failed.new(@queue).value
+    end
+
+    def test_same_counter_is_returned_for_failed
+      c = SuckerPunch::Counter::Failed.new(@queue)
+      assert_equal c.counter, SuckerPunch::Counter::Failed::COUNTER[@queue]
+    end
+
+    def test_failed_counter_default_is_0
+      c = SuckerPunch::Counter::Failed.new(@queue)
+      assert_equal 0, c.value
+    end
+
+    def test_failed_counter_supports_incrementing_and_decrementing
+      c = SuckerPunch::Counter::Failed.new(@queue)
+      assert_equal 1, c.increment
+      assert_equal 0, c.decrement
+    end
+  end
+end

data/test/sucker_punch/job_test.rb

@@ -0,0 +1,157 @@
+require 'test_helper'
+
+module SuckerPunch
+  class JobTest < Minitest::Test
+    def setup
+      SuckerPunch::Queue.clear
+    end
+
+    def teardown
+      SuckerPunch::Queue.clear
+      SuckerPunch::RUNNING.make_true
+    end
+
+    def test_perform_async_runs_job_asynchronously
+      arr = Concurrent::Array.new
+      latch = Concurrent::CountDownLatch.new
+      FakeLatchJob.perform_async(arr, latch)
+      latch.wait(0.2)
+      assert_equal 1, arr.size
+    end
+
+    def test_job_isnt_run_with_perform_async_if_sucker_punch_is_shutdown
+      SuckerPunch::RUNNING.make_false
+      arr = Concurrent::Array.new
+      latch = Concurrent::CountDownLatch.new
+      FakeLatchJob.perform_async(arr, latch)
+      latch.wait(0.2)
+      assert_equal 0, arr.size
+    end
+
+    def test_perform_in_runs_job_in_future
+      arr = Concurrent::Array.new
+      latch = Concurrent::CountDownLatch.new
+      FakeLatchJob.perform_in(0.1, arr, latch)
+      latch.wait(0.2)
+      assert_equal 1, arr.size
+    end
+
+    def test_job_isnt_run_with_perform_in_if_sucker_punch_is_shutdown
+      SuckerPunch::RUNNING.make_false
+      arr = Concurrent::Array.new
+      latch = Concurrent::CountDownLatch.new
+      FakeLatchJob.perform_in(0.1, arr, latch)
+      latch.wait(0.2)
+      assert_equal 0, arr.size
+    end
+
+    def test_default_workers_is_2
+      assert_equal 2, FakeLogJob.num_workers
+    end
+
+    def test_can_set_workers_count
+      FakeLogJob.workers(4)
+      assert_equal 4, FakeLogJob.num_workers
+      FakeLogJob.workers(2)
+    end
+
+    def test_logger_is_accessible_from_instance
+      SuckerPunch.logger = SuckerPunch.default_logger
+      assert_equal SuckerPunch.logger, FakeLogJob.new.logger
+      SuckerPunch.logger = nil
+    end
+
+    def test_num_workers_can_be_set_by_worker_method
+      assert_equal 4, FakeWorkerJob.num_workers
+    end
+
+    def test_num_workers_is_set_when_enqueueing_job_immediately
+      FakeWorkerJob.perform_async
+      pool = SuckerPunch::Queue::QUEUES[FakeWorkerJob.to_s]
+      assert_equal 4, pool.max_length
+      assert_equal 4, pool.min_length
+    end
+
+    def test_num_workers_is_set_when_enqueueing_job_in_future
+      FakeWorkerJob.perform_in(30)
+      pool = SuckerPunch::Queue::QUEUES[FakeWorkerJob.to_s]
+      assert_equal 4, pool.max_length
+      assert_equal 4, pool.min_length
+    end
+
+    def test_run_perform_delegates_to_instance_perform
+      assert_equal "fake", FakeLogJob.__run_perform
+    end
+
+    def test_busy_workers_is_incremented_during_job_execution
+      FakeSlowJob.perform_async
+      sleep 0.1
+      assert SuckerPunch::Counter::Busy.new(FakeSlowJob.to_s).value > 0
+    end
+
+    def test_processed_jobs_is_incremented_on_successful_completion
+      latch = Concurrent::CountDownLatch.new
+      2.times{ FakeLogJob.perform_async }
+      queue = SuckerPunch::Queue.find_or_create(FakeLogJob.to_s)
+      queue.post { latch.count_down }
+      latch.wait(0.2)
+      assert SuckerPunch::Counter::Processed.new(FakeLogJob.to_s).value > 0
+    end
+
+    def test_processed_jobs_is_incremented_when_enqueued_with_perform_in
+      latch = Concurrent::CountDownLatch.new
+      FakeLatchJob.perform_in(0.1, [], latch)
+      latch.wait(0.2)
+      assert SuckerPunch::Counter::Processed.new(FakeLatchJob.to_s).value > 0
+    end
+
+    def test_failed_jobs_is_incremented_when_job_raises
+      latch = Concurrent::CountDownLatch.new
+      2.times{ FakeErrorJob.perform_async }
+      queue = SuckerPunch::Queue.find_or_create(FakeErrorJob.to_s)
+      queue.post { latch.count_down }
+      latch.wait(0.2)
+      assert SuckerPunch::Counter::Failed.new(FakeErrorJob.to_s).value > 0
+    end
+
+    private
+
+    class FakeLatchJob
+      include SuckerPunch::Job
+      def perform(arr, latch)
+        arr.push true
+        latch.count_down
+      end
+    end
+
+    class FakeSlowJob
+      include SuckerPunch::Job
+      def perform
+        sleep 0.3
+      end
+    end
+
+    class FakeLogJob
+      include SuckerPunch::Job
+      def perform
+        "fake"
+      end
+    end
+
+    class FakeWorkerJob
+      include SuckerPunch::Job
+      workers 4
+      def perform
+        "fake"
+      end
+    end
+
+    class FakeErrorJob
+      include SuckerPunch::Job
+      def perform
+        raise "error"
+      end
+    end
+  end
+end
+