fleiss 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e9eb58758de53588746446aedfbd877e9b84d361de7650185265b9973aeff0b3
-  data.tar.gz: d5f1ac77ce8f6dbef855c6a4e0f2eecdad56a7f8a5fc30f5851d95934eb92453
+  metadata.gz: 92e4e656c34af27e17ac918eaa0d21f5d01f31f3d5f04f4c08290ce2ebb49035
+  data.tar.gz: d80b83c7dcb419ff05bcee827b8e7ba314be13c7845d658a86e5142d13c34a84
 SHA512:
-  metadata.gz: c1b97a6658b29170a51a85df91611ff562f319d03f29626bbcf3069858e19cf7cc5aed4cfec7b0508fc90a1899cf325b84fef67e9c270842609acb62cbe0fad7
-  data.tar.gz: 28dccc2dced0062b9355f75da37989f2c008bd3b6d3be0d8cac4c8c3ea637eb41f4b94a747e10abd19ef6f85e51d45f01db62c581f1dee0b98377f6affb6cb3f
+  metadata.gz: 1bd0fc7a843d98020b0491fb5a1c457315f5589823b87d43ffebcce07933d548421ee9f3c705d9090d81ccc20319d363aba789559122a5fb54e65751a773c84a
+  data.tar.gz: 44fea18168d2e79bf9d0eec495a010f85edf313c543a1a271b5090132806c2c03a7cc653c77949a4ffe26cf46d88383192847d8d90565e37b3294a6e751494f1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fleiss (0.1.1)
+    fleiss (0.1.2)
       activejob (>= 5.0)
       activerecord (>= 5.0)
       concurrent-ruby
@@ -40,7 +40,7 @@ GEM
     pg (1.1.3)
     powerpack (0.1.2)
     rainbow (3.0.0)
-    rake (12.3.1)
+    rake (12.3.2)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)

data/fleiss.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'fleiss'
-  s.version       = '0.1.1'
+  s.version       = '0.1.2'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Minimialist background jobs backed by ActiveJob and ActiveRecord.)

data/lib/fleiss/backend/active_record/concern.rb

@@ -6,6 +6,7 @@ module Fleiss
 
         included do
           scope :in_queue, ->(qs) { where(queue_name: Array.wrap(qs)) }
+          scope :finished, -> { where.not(finished_at: nil) }
           scope :not_finished, -> { where(finished_at: nil) }
           scope :not_expired,  ->(now=Time.zone.now) { where(arel_table[:expires_at].eq(nil).or(arel_table[:expires_at].gt(now))) }
           scope :started,      -> { where(arel_table[:started_at].not_eq(nil)) }
@@ -29,15 +30,6 @@ module Fleiss
             started.not_finished.where(owner: owner)
           end
 
-          # Reschedules jobs to run again.
-          def reschedule_all(at=Time.zone.now)
-            update_all(
-              started_at: nil,
-              owner: nil,
-              scheduled_at: at,
-            )
-          end
-
           # @param [ActiveJob::Base] job the job instance
           # @option [Time] :scheduled_at schedule job at
           def enqueue(job, scheduled_at: nil)
@@ -68,23 +60,47 @@ module Fleiss
         # @param [String] owner
         # @return [Boolean] true if job was started.
         def start(owner, now: Time.zone.now)
-          with_lock do
+          with_isolation do
             self.class.pending(now)
                 .where(id: id)
-                .update_all(started_at: now, owner: owner) == 1
-          end
+                .update_all(started_at: now, owner: owner)
+          end == 1
+        rescue ::ActiveRecord::SerializationFailure
+          false
         end
 
         # Marks a job as finished.
         # @param [String] owner
         # @return [Boolean] true if successful.
         def finish(owner, now: Time.zone.now)
-          with_lock do
+          with_isolation do
             self.class
                 .in_progress(owner)
                 .where(id: id)
-                .update_all(finished_at: now) == 1
-          end
+                .update_all(finished_at: now)
+          end == 1
+        rescue ::ActiveRecord::SerializationFailure
+          false
+        end
+
+        # Reschedules the job to run again.
+        def reschedule(owner, now: Time.zone.now)
+          with_isolation do
+            self.class
+                .in_progress(owner)
+                .where(id: id)
+                .update_all(started_at: nil, owner: nil, scheduled_at: now)
+          end == 1
+        rescue ::ActiveRecord::SerializationFailure
+          false
+        end
+
+        private
+
+        def with_isolation(&block)
+          return yield unless self.class.connection.supports_transaction_isolation?
+
+          self.class.transaction(isolation: :repeatable_read, &block)
         end
       end
     end

data/lib/fleiss/cli.rb

@@ -42,26 +42,15 @@ module Fleiss
     end
 
     def run!
-      return if @worker
-
       $LOAD_PATH.concat opts[:include]
       opts[:require].each {|n| require n }
       require 'fleiss/worker'
 
-      Signal.trap('TERM') { shutdown }
-      Signal.trap('INT')  { shutdown }
-
-      @worker = Fleiss::Worker.new \
+      Fleiss::Worker.run \
         queues: opts[:queues],
         concurrency: opts[:concurrency],
         wait_time: opts[:wait_time],
         logger: Logger.new(opts[:logfile])
-      @worker.run
-      @worker.wait
-    end
-
-    def shutdown
-      @worker.shutdown if @worker
     end
 
     def parser # rubocop:disable Metrics/MethodLength

data/lib/fleiss/worker.rb

@@ -7,6 +7,11 @@ require 'securerandom'
 class Fleiss::Worker
   attr_reader :queues, :uuid, :wait_time, :logger
 
+  # Shortcut for new(*args).run
+  def self.run(*args)
+    new(*args).run
+  end
+
   # Init a new worker instance
   # @param [ConnectionPool] disque client connection pool
   # @param [Hash] options
@@ -27,36 +32,20 @@ class Fleiss::Worker
     logger.info "Worker #{uuid} starting - queues: #{queues.inspect}, concurrency: #{@pool.max_length}"
     loop do
       run_cycle
-      break if @stopped
-
       sleep @wait_time
     end
-    logger.info "Worker #{uuid} shutting down"
-  end
-
-  # Blocks until worker until it's stopped.
-  def wait
+  rescue SignalException => e
+    logger.info "Worker #{uuid} received #{e.message}. Shutting down..."
+  ensure
     @pool.shutdown
-    @pool.wait_for_termination(1)
-
-    Fleiss.backend
-          .in_queue(queues)
-          .in_progress(uuid)
-          .reschedule_all(10.seconds.from_now)
     @pool.wait_for_termination
-    logger.info "Worker #{uuid} shutdown complete"
-  rescue StandardError => e
-    handle_exception e, 'shutdown'
-  end
-
-  # Initiates the shutdown process
-  def shutdown
-    @stopped = true
   end
 
   private
 
   def run_cycle
+    return if @pool.shuttingdown?
+
     capacity = @pool.max_length - @pool.scheduled_task_count + @pool.completed_task_count
     return unless capacity.positive?
 
@@ -67,8 +56,6 @@ class Fleiss::Worker
                   .to_a
 
     batch.each do |job|
-      break if @stopped
-
       @pool.post { perform(job) }
     end
   rescue StandardError => e
@@ -76,16 +63,23 @@ class Fleiss::Worker
   end
 
   def perform(job)
-    return unless job.start(uuid)
-
-    thread_id = Thread.current.object_id.to_s(36)
-    logger.info { "Worker #{uuid} execute job ##{job.id} by thread #{thread_id}" }
-
-    ActiveJob::Base.execute job.job_data
+    thread_id = Thread.current.object_id.to_s(16).reverse
+    owner     = "#{uuid}/#{thread_id}"
+    return unless job.start(owner)
+
+    logger.info { "Worker #{uuid} execute job ##{job.id} (by thread #{thread_id})" }
+    finished = false
+    begin
+      ActiveJob::Base.execute job.job_data
+      finished = true
+    rescue StandardError
+      finished = true
+      raise
+    ensure
+      finished ? job.finish(owner) : job.reschedule(owner)
+    end
   rescue StandardError => e
     handle_exception e, "processing job ##{job.id} (by thread #{thread_id})"
-  ensure
-    job.finish(uuid)
   end
 
   def handle_exception(err, intro)

data/spec/fleiss/backend/active_record_spec.rb

@@ -46,13 +46,6 @@ RSpec.describe Fleiss::Backend::ActiveRecord do
     expect(rec.job_id.size).to eq(36)
   end
 
-  it 'should reschedule' do
-    job = TestJob.perform_later
-    described_class.reschedule_all(1.hour.from_now)
-    rec = retrieve(job)
-    expect(rec.scheduled_at).to be_within(2.seconds).of(1.hour.from_now)
-  end
-
   it 'should scope pending' do
     j1 = TestJob.perform_later
     expect(retrieve(j1).start('owner')).to be_truthy
@@ -118,4 +111,16 @@ RSpec.describe Fleiss::Backend::ActiveRecord do
     expect(rec.started_at).to be_within(2.seconds).of(Time.zone.now)
     expect(rec.finished_at).to be_within(2.seconds).of(Time.zone.now)
   end
+
+  it 'should reschedule' do
+    job = TestJob.perform_later
+    rec = retrieve(job)
+    expect(rec.reschedule('owner')).to be_falsey
+    expect(rec.start('owner')).to be_truthy
+    expect(rec.reschedule('other')).to be_falsey
+    expect(rec.reschedule('owner')).to be_truthy
+    expect(rec.reload.owner).to be_nil
+    expect(rec.started_at).to be_nil
+    expect(rec.scheduled_at).to be_within(2.seconds).of(Time.zone.now)
+  end
 end

data/spec/fleiss/worker_spec.rb

@@ -13,12 +13,11 @@ RSpec.describe Fleiss::Worker do
   end
 
   after do
-    subject.shutdown
-    subject.wait
+    runner.kill
   end
 
   def wait_for
-    200.times do
+    100.times do
       break if yield
 
       sleep(0.1)
@@ -26,22 +25,23 @@ RSpec.describe Fleiss::Worker do
     expect(yield).to be_truthy
   end
 
-  it 'should run/process/shutdown' do
-    # seed 50 jobs
-    50.times {|n| TestJob.perform_later(n) }
-    wait_for { Fleiss.backend.not_finished.count > 0 }
+  it 'should run' do
+    # seed 24 jobs
+    24.times {|n| TestJob.perform_later(n) }
+    wait_for { Fleiss.backend.not_finished.count.positive? }
 
     # ensure runner processes them all
     wait_for { Fleiss.backend.not_finished.count.zero? }
 
     # check what's been performed
-    expect(TestJob.performed.size).to eq(50)
-    expect(TestJob.performed).to match_array(0..49)
+    expect(TestJob.performed.size).to eq(24)
+    expect(Fleiss.backend.finished.count).to eq(24)
+    expect(TestJob.performed).to match_array(0..23)
   end
 
   it 'should handle failing jobs' do
     TestJob.perform_later('raise')
     wait_for { Fleiss.backend.not_finished.count.zero? }
-    expect(Fleiss.backend.count).to eq(1)
+    expect(Fleiss.backend.finished.count).to eq(1)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fleiss
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-06 00:00:00.000000000 Z
+date: 2018-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -201,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Minimialist background jobs backed by ActiveJob and ActiveRecord.