skiplock 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9731224db727e3ebe4b32b94b4edff6bf7a53faca14be5a6f5d7cc20a02836e9
-  data.tar.gz: 9c3cb7ccf0d98d7e8ec4bcd21e33269504d166804f0679fe7d286290527a8457
+  metadata.gz: c53cdb42608339ffd7ba7532393e933ce9128757ee35679a665fd2f1e1d0fba3
+  data.tar.gz: e7dfcc14eed82fef12747d9c0172de006cdb8823fdcaa1d7e944b8d0bde71be1
 SHA512:
-  metadata.gz: 560b30ea96a8f255811b1c8a0beb7e161ba1edd2dde238e52cd31625e1a4e4d729ff24f3d4d37e4a29fbb2c3e2543e86b9cb1720553cacab346ac30db2970aea
-  data.tar.gz: '093b4ee02119973dbd1c8c905d2ccbcc1d6d9efad440ba23ba8fc05a8a2a3580ec7a7d3c7c47d5f6077b789a440e4c46e455f81a5c10fd032b01ffefae906a0e'
+  metadata.gz: b32ac8c7b3cf90be24c86310d4e2b2fe250a5ec7b244aad267144f49e09f14d9a46e64214f45afde8093d4a867f1c93245c5e216ac214cacca555f3965770f7c
+  data.tar.gz: 0a61bf8d14f3162fe981420d43278642ac98a4ab5ac6705bc9a9f65ddd3330e26b6ec1d3ce9a196155e2316ed972954cd657f4d36a48bda83554ea97eb205237

data/README.md

@@ -1,6 +1,6 @@
 # Skiplock
 
-Skiplock is a background job queuing system that improves the performance and reliability of the job executions while providing the same ACID guarantees as the rest of your data.  It is designed for Active Jobs with Ruby on Rails using PostgreSQL database adapter, but it can be modified to work with other frameworks easily.
+`Skiplock` is a background job queuing system that improves the performance and reliability of the job executions while providing the same ACID guarantees as the rest of your data.  It is designed for Active Jobs with Ruby on Rails using PostgreSQL database adapter, but it can be modified to work with other frameworks easily.
 
 It only uses the `LISTEN/NOTIFY/SKIP LOCKED` features provided natively on PostgreSQL 9.5+ to efficiently and reliably dispatch jobs to worker processes and threads ensuring that each job can be completed successfully **only once**.  No other polling or timer is needed.
 
@@ -14,7 +14,7 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
 
 ## Installation
 
-1. Add `skiplock` to your application's Gemfile:
+1. Add `Skiplock` to your application's Gemfile:
 
     ```ruby
     gem 'skiplock'
@@ -26,7 +26,7 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     $ bundle install
     ```
 
-3. Run the Skiplock install generator. This will generate a configuration file and database migration to store the job records:
+3. Run the `Skiplock` install generator. This will generate a configuration file and database migration to store the job records:
 
     ```bash
     $ rails g skiplock:install
@@ -46,7 +46,7 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     # config/application.rb
     config.active_job.queue_adapter = :skiplock
     ```
-2. Skiplock configuration
+2. `Skiplock` configuration
     ```yaml
     # config/skiplock.yml
     ---
@@ -104,7 +104,7 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     INSERT INTO skiplock.jobs(job_class,priority,scheduled_at,data) VALUES ('MyJob',10,NOW()+INTERVAL '5 min','{"arguments":[1,2,3]}');
     ```
 ## Cron system
-Skiplock provides the capability to setup cron jobs for running tasks periodically.  It fully supports the cron syntax to specify the frequency of the jobs.  To setup a cron job, simply assign a valid cron schedule to the constant `CRON` for the Job Class.
+`Skiplock` provides the capability to setup cron jobs for running tasks periodically.  It fully supports the cron syntax to specify the frequency of the jobs.  To setup a cron job, simply assign a valid cron schedule to the constant `CRON` for the Job Class.
 - setup `MyJob` to run as cron job every hour at 30 minutes past
 
     ```ruby
@@ -123,7 +123,7 @@ Skiplock provides the capability to setup cron jobs for running tasks periodical
 - to remove the cron schedule from the job, simply comment out the constant definition or delete the line then re-deploy the application. At startup, the cron jobs that were undefined will be removed automatically
 
 ## Retry system
-Skiplock fully supports ActiveJob built-in retry system.  It also has its own retry system for fallback.  To use ActiveJob retry system, define the rescue blocks per ActiveJob's documentation.
+`Skiplock` fully supports ActiveJob built-in retry system.  It also has its own retry system for fallback.  To use ActiveJob retry system, define the rescue blocks per ActiveJob's documentation.
 - configures `MyJob` to retry at maximum 20 attempts on StandardError with fixed delay of 5 seconds
     ```ruby
     class MyJob < ActiveJob::Base

data/lib/generators/skiplock/templates/migration.rb.erb

@@ -2,6 +2,7 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
   def up
     execute 'CREATE SCHEMA skiplock'
     create_table 'skiplock.jobs', id: :uuid do |t|
+      t.uuid      :worker_id, index: true
       t.string    :job_class, null: false
       t.string    :cron
       t.string    :queue_name
@@ -11,12 +12,20 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
       t.integer   :executions
       t.jsonb     :exception_executions
       t.jsonb     :data
-      t.boolean   :running, null: false, default: false
+      t.boolean   :running, null: false, default: false, index: true
       t.timestamp :expired_at
       t.timestamp :finished_at
       t.timestamp :scheduled_at
       t.timestamps null: false, default: -> { 'now()' }
     end
+    create_table 'skiplob.workers', id: :uuid do |t|
+      t.integer   :pid, null: false, index: true
+      t.integer   :ppid, index: true
+      t.integer   :remaining_capacity, null: false
+      t.string    :hostname, null: false, index: true
+      t.jsonb     :data
+      t.timestamps null: false, index: true, default: -> { 'now()' }
+    end    
     execute %(CREATE OR REPLACE FUNCTION skiplock.notify() RETURNS TRIGGER AS $$
       BEGIN
         IF (NEW.finished_at IS NULL AND NEW.expired_at IS NULL) THEN
@@ -27,8 +36,8 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
       $$ LANGUAGE plpgsql
     )
     execute "CREATE TRIGGER notify_job AFTER INSERT OR UPDATE ON skiplock.jobs FOR EACH ROW EXECUTE PROCEDURE skiplock.notify()"
-    execute "CREATE INDEX jobs_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE running = 'f' AND expired_at IS NULL AND finished_at IS NULL"
-    execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE running = 'f' AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
+    execute "CREATE INDEX jobs_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE expired_at IS NULL AND finished_at IS NULL"
+    execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
     execute "CREATE INDEX jobs_cron_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE cron IS NOT NULL AND finished_at IS NULL"
     execute "CREATE UNIQUE INDEX jobs_unique_cron_index ON skiplock.jobs (job_class) WHERE cron IS NOT NULL"
   end

data/lib/skiplock.rb

@@ -6,6 +6,7 @@ require 'skiplock/dispatcher'
 require 'skiplock/manager'
 require 'skiplock/notification'
 require 'skiplock/job'
+require 'skiplock/worker'
 require 'skiplock/version'
 
 module Skiplock

data/lib/skiplock/dispatcher.rb

@@ -1,8 +1,13 @@
 module Skiplock
   class Dispatcher
-    def initialize(master: true)
+    def initialize(master: true, worker_num: nil, worker_pids: [])
       @executor = Concurrent::ThreadPoolExecutor.new(min_threads: Settings['min_threads'], max_threads: Settings['max_threads'], max_queue: Settings['max_threads'], idletime: 60, auto_terminate: true, fallback_policy: :discard)
       @master = master
+      if @master
+        @worker_pids = worker_pids + [ Process.pid ]
+      else
+        @worker_num = worker_num
+      end
       @next_schedule_at = Time.now.to_f
       @running = true
     end
@@ -11,9 +16,19 @@ module Skiplock
       Thread.new do
         Rails.application.reloader.wrap do
           sleep(0.1) while @running && !Rails.application.initialized?
+          Process.setproctitle("skiplock-#{@master ? 'master' : 'worker[' + @worker_num.to_s + ']'}") if Settings['workers'] > 0
           ActiveRecord::Base.connection_pool.with_connection do |connection|
-            connection.exec_query('LISTEN skiplock')
+            connection.exec_query('LISTEN "skiplock::jobs"')
+            hostname = `hostname`.strip
+            @worker = Worker.create!(pid: Process.pid, ppid: (@master ? nil : Process.ppid), capacity: Settings['max_threads'], hostname: hostname)
             if @master
+              # get worker ids that were not shutdown properly on the host
+              dead_worker_ids = Worker.where(hostname: hostname).where.not(pid: @worker_pids).ids
+              if dead_worker_ids.count > 0
+                # reset orphaned jobs of the dead worker ids for retry
+                Job.where(running: true, worker_id: dead_worker_ids).update_all(running: false, worker_id: nil)
+                Worker.where(id: dead_worker_ids).delete_all
+              end
               # reset retries schedules on startup
               Job.where('scheduled_at > NOW() AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL').update_all(scheduled_at: nil, updated_at: Time.now)
               Cron.setup
@@ -25,7 +40,7 @@ module Skiplock
                   unless connection.active?
                     connection.reconnect!
                     sleep(0.5)
-                    connection.exec_query('LISTEN skiplock')
+                    connection.exec_query('LISTEN "skiplock::jobs"')
                     @next_schedule_at = Time.now
                   end
                   error = false
@@ -42,10 +57,11 @@ module Skiplock
                     notifications << payload[:extra]
                   end
                   notifications.each do |n|
-                    op, id, queue, priority, time = n.split(',')
-                    if time.to_f <= Time.now.to_f
+                    op, id, worker_id, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
+                    next if op == 'DELETE' || running == 'true' || expired_at || finished_at
+                    if scheduled_at.to_f <= Time.now.to_f
                       @next_schedule_at = Time.now.to_f
-                    elsif time.to_f < @next_schedule_at
+                    elsif scheduled_at.to_f < @next_schedule_at
                       @next_schedule_at = time.to_f
                     end
                   end
@@ -53,10 +69,10 @@ module Skiplock
               rescue Exception => ex
                 # TODO: Report exception
                 error = true
-                timestamp = Time.now
+                t = Time.now
                 while @running
                   sleep(0.5)
-                  break if Time.now - timestamp > 10
+                  break if Time.now - t > 5
                 end              
               end
               sleep(0.1)
@@ -71,22 +87,21 @@ module Skiplock
       @running = false
       @executor.shutdown
       @executor.wait_for_termination if wait
+      @worker.delete if @worker
     end
 
     private
 
     def do_work
-      connection = ActiveRecord::Base.connection_pool.checkout
       while @running
-        result = Job.dispatch(connection: connection)
+        result = Job.dispatch(worker_id: @worker.id)
         next if result.is_a?(Hash)
         @next_schedule_at = result if result.is_a?(Float)
         break
       end
     rescue Exception => e
+      puts "Exception => #{e.message}   #{e.backtrace}"
       # TODO: Report exception
-    ensure
-      ActiveRecord::Base.connection_pool.checkin(connection)
     end
   end
 end

data/lib/skiplock/job.rb

@@ -2,70 +2,70 @@ module Skiplock
   class Job < ActiveRecord::Base
     self.table_name = 'skiplock.jobs'
 
-    # Accept: An active ActiveRecord database connection (eg. ActiveRecord::Base.connection)
-    #         The connection should be checked out using ActiveRecord::Base.connection_pool.checkout, and be checked
-    #         in using ActiveRecord::Base.conection_pool.checkin once all of the job dispatches have been completed.
-    #         *** IMPORTANT: This connection cannot be shared with the job's execution
-    #
     # Return: Attributes hash of the Job if it was executed; otherwise returns the next Job's schedule time in FLOAT
-    def self.dispatch(connection: ActiveRecord::Base.connection)
-      connection.exec_query('BEGIN')
-      job = connection.exec_query("SELECT * FROM #{self.table_name} WHERE running = 'f' AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
-      if job && job['scheduled_at'].to_f <= Time.now.to_f  # found job ready to perform
-        # update the job to mark it in progress in case database server goes down during job execution
-        connection.exec_query("UPDATE #{self.table_name} SET running = 't' WHERE id = '#{job['id']}'")
-        connection.exec_query('END') # close the transaction commit the state of job in progress
-        executions = (job['executions'] || 0) + 1
-        exceptions = job['exception_executions'] ? JSON.parse(job['exception_executions']) : {}
-        data = job['data'] ? JSON.parse(job['data']) : {}
-        job_data = job.slice('job_class', 'queue_name', 'locale', 'timezone', 'priority', 'executions').merge('job_id' => job['id'], 'exception_executions' => exceptions, 'enqueued_at' => job['updated_at']).merge(data)
-        Thread.current[:skiplock_dispatch_data] = job_data
-        begin
-          ActiveJob::Base.execute(job_data)
-        rescue Exception => ex
+    def self.dispatch(worker_id: nil)
+      self.connection.exec_query('BEGIN')
+      job = self.find_by_sql("SELECT id, scheduled_at FROM #{self.table_name} WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
+      if job.nil? || job.scheduled_at.to_f > Time.now.to_f
+        self.connection.exec_query('END')
+        return (job ? job.scheduled_at.to_f : Float::INFINITY)
+      end
+      job = Skiplock::Job.find_by_sql("UPDATE #{self.table_name} SET running = TRUE, worker_id = #{self.connection.quote(worker_id)} WHERE id = '#{job.id}' RETURNING *").first
+      self.connection.exec_query('END')
+      job.data ||= {}
+      job.exception_executions ||= {}
+      job_data = job.attributes.slice('job_class', 'queue_name', 'locale', 'timezone', 'priority', 'executions', 'exception_executions').merge('job_id' => job.id, 'enqueued_at' => job.updated_at, 'arguments' => (job.data['arguments'] || []))
+      job.executions = (job.executions || 0) + 1
+      Thread.current[:skiplock_dispatch_job] = job
+      begin
+        ActiveJob::Base.execute(job_data)
+      rescue Exception => ex
+      end
+      job.running = false
+      if ex
+        # TODO: report exception
+        job.exception_executions["[#{ex.class.name}]"] = (job.exception_executions["[#{ex.class.name}]"] || 0) + 1 unless job.exception_executions.key?('activejob_retry')
+        if job.executions >= Settings['max_retries'] || job.exception_executions.key?('activejob_retry')
+          job.expired_at = Time.now
+          job.save!
+        else
+          job.scheduled_at = Time.now + (5 * 2**job.executions)
+          job.save!
         end
-        if ex
-          # TODO: report exception
-          exceptions["[#{ex.class.name}]"] = (exceptions["[#{ex.class.name}]"] || 0) + 1 unless exceptions.key?('activejob_retry')
-          if executions >= Settings['max_retries'] || exceptions.key?('activejob_retry')
-            connection.exec_query("UPDATE #{self.table_name} SET running = 'f', executions = #{executions}, exception_executions = '#{connection.quote_string(exceptions.to_json.to_s)}', expired_at = NOW(), updated_at = NOW() WHERE id = '#{job['id']}' RETURNING *").first
-          else
-            timestamp = Time.now + (5 * 2**executions)
-            connection.exec_query("UPDATE #{self.table_name} SET running = 'f', executions = #{executions}, exception_executions = '#{connection.quote_string(exceptions.to_json.to_s)}', scheduled_at = TO_TIMESTAMP(#{timestamp.to_f}), updated_at = NOW() WHERE id = '#{job['id']}' RETURNING *").first
-          end
-        elsif exceptions.key?('activejob_retry')
-          connection.exec_query("UPDATE #{self.table_name} SET running = 'f', executions = #{job_data['executions']}, exception_executions = '#{connection.quote_string(job_data['exception_executions'].to_json.to_s)}', scheduled_at = TO_TIMESTAMP(#{job_data['scheduled_at'].to_f}), updated_at = NOW() WHERE id = '#{job['id']}' RETURNING *").first
-        elsif job['cron']
-          data['last_cron_run'] = Time.now.utc.to_s
-          next_cron_at = Cron.next_schedule_at(job['cron'])
-          if next_cron_at
-            connection.exec_query("UPDATE #{self.table_name} SET running = 'f', scheduled_at = TO_TIMESTAMP(#{next_cron_at}), executions = 1, exception_executions = NULL, data = '#{connection.quote_string(data.to_json.to_s)}', updated_at = NOW() WHERE id = '#{job['id']}' RETURNING *").first
-          else
-            connection.exec_query("DELETE FROM #{self.table_name} WHERE id = '#{job['id']}' RETURNING *").first
-          end
-        elsif Settings['purge_completion']
-          connection.exec_query("DELETE FROM #{self.table_name} WHERE id = '#{job['id']}' RETURNING *").first
+      elsif job.exception_executions.key?('activejob_retry')
+        job.save!
+      elsif job['cron']
+        job.data['last_cron_run'] = Time.now.utc.to_s
+        next_cron_at = Cron.next_schedule_at(job['cron'])
+        if next_cron_at
+          job.executions = 1
+          job.exception_executions = nil
+          job.scheduled_at = Time.at(next_cron_at)
+          job.save!
         else
-          connection.exec_query("UPDATE #{self.table_name} SET running = 'f', executions = #{executions}, exception_executions = NULL, finished_at = NOW(), updated_at = NOW() WHERE id = '#{job['id']}' RETURNING *").first
+          job.delete
         end
+      elsif Settings['purge_completion']
+        job.delete
       else
-        connection.exec_query('END')
-        job ? job['scheduled_at'].to_f : Float::INFINITY
+        job.finished_at = Time.now
+        job.exception_executions = nil
+        job.save!
       end
+      job
     ensure
       Thread.current[:skiplock_dispatch_data] = nil
     end
 
-    def self.enqueue_at(job, timestamp)
-      if Thread.current[:skiplock_dispatch_data]
-        job.exception_executions['activejob_retry'] = true
-        Thread.current[:skiplock_dispatch_data]['executions'] = job.executions
-        Thread.current[:skiplock_dispatch_data]['exception_executions'] = job.exception_executions
-        Thread.current[:skiplock_dispatch_data]['scheduled_at'] = Time.at(timestamp)
-        self.new(Thread.current[:skiplock_dispatch_data].slice(*self.column_names).merge(id: job.job_id))
+    def self.enqueue_at(activejob, timestamp)
+      timestamp = Time.at(timestamp) if timestamp
+      if Thread.current[:skiplock_dispatch_job].try(:id) == activejob.job_id
+        Thread.current[:skiplock_dispatch_job].exception_executions = activejob.exception_executions.merge('activejob_retry' => true)
+        Thread.current[:skiplock_dispatch_job].executions = activejob.executions
+        Thread.current[:skiplock_dispatch_job].scheduled_at = timestamp
+        Thread.current[:skiplock_dispatch_job]
       else
-        timestamp = Time.at(timestamp) if timestamp
-        Job.create!(id: job.job_id, job_class: job.class.name, queue_name: job.queue_name, locale: job.locale, timezone: job.timezone, priority: job.priority, executions: job.executions, data: { 'arguments' => job.arguments }, scheduled_at: timestamp)
+        Job.create!(id: activejob.job_id, job_class: activejob.class.name, queue_name: activejob.queue_name, locale: activejob.locale, timezone: activejob.timezone, priority: activejob.priority, executions: activejob.executions, data: { 'arguments' => activejob.serialize['arguments'] }, scheduled_at: timestamp)
       end
     end
   end

data/lib/skiplock/manager.rb

@@ -65,10 +65,10 @@ module Skiplock
       shutdown = false
       Signal.trap("INT") { shutdown = true }
       Signal.trap("TERM") { shutdown = true }
-      Settings['workers'].times do |w|
-        fork do
-          Process.setproctitle("skiplock-worker[#{w+1}]")
-          dispatcher = Dispatcher.new(master: false)
+      worker_pids = []
+      Settings['workers'].times do |n|
+        worker_pids << fork do
+          dispatcher = Dispatcher.new(master: false, worker_num: n)
           thread = dispatcher.run
           loop do
             sleep 0.5
@@ -80,8 +80,7 @@ module Skiplock
         end
       end
       sleep 0.1
-      Process.setproctitle("skiplock-master")
-      dispatcher = Dispatcher.new
+      dispatcher = Dispatcher.new(worker_pids: worker_pids)
       thread = dispatcher.run
       loop do
         sleep 0.5

data/lib/skiplock/version.rb

@@ -1,4 +1,4 @@
 module Skiplock
-  VERSION = Version = '1.0.3'
+  VERSION = Version = '1.0.4'
 end
 

data/lib/skiplock/worker.rb

@@ -0,0 +1,5 @@
+module Skiplock
+  class Worker < ActiveRecord::Base
+    self.table_name = 'skiplock.workers'
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skiplock
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Tin Vo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-17 00:00:00.000000000 Z
+date: 2021-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -88,6 +88,7 @@ files:
 - lib/skiplock/manager.rb
 - lib/skiplock/notification.rb
 - lib/skiplock/version.rb
+- lib/skiplock/worker.rb
 homepage: https://github.com/vtt/skiplock
 licenses:
 - MIT
@@ -107,7 +108,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: ActiveJob Queue Adapter for PostgreSQL