skiplock 1.0.7 → 1.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b5f33c5c4f2f2223b68f6d025989cc3eb6568fb52a7e39ef3bf4c48a1807ec3
-  data.tar.gz: 3839f01f8771b1ec06a0ab33c32555d4aa7ad0a48e142d0224fe46868692bfa5
+  metadata.gz: 432874dc801864f01a8c4de896f330ec62cecdb43d71ededc622ece7d9a17402
+  data.tar.gz: 6034dcb3cfa194b186465a5a84227899c8f00fdf4d62d5b20e0370c9c122cde5
 SHA512:
-  metadata.gz: 7bf28c36dca7f41518c6d5be2651ff7b40e0f62c526248a478db4a88545b7663260225de5b2c6eacf89e17e34cf24ac2fd38c0bbf95c4cac54a8f57f68fd772a
-  data.tar.gz: 3998b4377bbe2dd0b993ac9abcf848c6164fcb4378387cc8c7187d507c6fc6df1676fdba30ce98e26ac41730d0569a64197317c52447b1e8d6a06c93fb313fa7
+  metadata.gz: 89f0ef53c0740cf5522e20aa1473ed9020bf5d116054acbb77f451472c1d9af2b123b47cdc593efb4545383829609537a4e2395b4c23c7482f04cff04b11ceab
+  data.tar.gz: 640fa077844855e312b6018d2858cb495915fba0f167c711540ff3b713c66dfc5aa76ca283d9dffaf3d778f37c7025a1f3f7ee3728d4120d87bc25079717bc60

data/README.md

@@ -48,72 +48,74 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     ```
 2. `Skiplock` configuration
     ```yaml
-    # config/skiplock.yml
+    # config/skiplock.yml (default settings)
     ---
-    logging: timestamp
     min_threads: 1
     max_threads: 5
     max_retries: 20
-    notification: auto
+    logfile: log/skiplock.log
+    notification: custom
+    extensions: false
     purge_completion: true
     queues:
       default: 200
-      mailers: 100
+      mailers: 999
     workers: 0
     ```
     Available configuration options are:
-    - **logging** (*enumeration*): sets the logging capability to **true** or **false**; setting to **timestamp** will enable logging with timestamps. The log files are: log/skiplock.log and log/skiplock.error.log
     - **min_threads** (*integer*): sets minimum number of threads staying idle
     - **max_threads** (*integer*): sets the maximum number of threads allowed to run jobs
-    - **max_retries** (*integer*): sets the maximum attempt a job will be retrying before it is marked expired (see Retry System for more details)
-    - **notification** (*enumeration*): sets the library to be used for notifying errors and exceptions (`auto, airbrake, bugsnag, exception_notification, false`)
+    - **max_retries** (*integer*): sets the maximum attempt a job will be retrying before it is marked expired.  See `Retry system` for more details
+    - **logfile** (*string*): path filename for skiplock logs; empty logfile will disable logging
+    - **notification** (*string*): sets the library to be used for notifying errors and exceptions (`auto, airbrake, bugsnag, exception_notification, custom`); using `auto` will detect library if available.  See `Notification system` for more details
+    - **extensions** (*boolean*): enable or disable the class method extension.  See `ClassMethod extension` for more details
     - **purge_completion** (*boolean*): when set to **true** will delete jobs after they were completed successfully; if set to **false** then the completed jobs should be purged periodically to maximize performance (eg. clean up old jobs after 3 months)
-    - **queues** (*hash*): defines the set of queues with priorites; lower priority takes precedence
+    - **queues** (*hash*): defines the set of queues with priorities; lower priority takes precedence
     - **workers** (*integer*) sets the maximum number of processes when running in standalone mode using the `skiplock` executable; setting this to **0** will enable **async mode**
-    
-    #### Async mode
-    When **workers** is set to **0** then the jobs will be performed in the web server process using separate threads.  If using multi-worker cluster web server like Puma, then it should be configured as below:
-    ```ruby
-    # config/puma.rb
-    before_fork do
-      # ...
-      Skiplock::Manager.shutdown
-    end
 
-    on_worker_boot do
-      # ...
-      Skiplock::Manager.start
-    end
+    #### **Async mode**
+    When **workers** is set to **0** then the jobs will be performed in the web server process using separate threads.  If using multi-worker cluster mode web server like Puma, then all the Puma workers will also be able to perform `Skiplock` jobs.
 
-    on_worker_shutdown do
-      # ...
-      Skiplock::Manager.shutdown
-    end
+    #### **Standalone mode**
+    `Skiplock` standalone mode can be launched by using the `skiplock` executable; command line options can be provided to override the `Skiplock` configuration file.  
+    ```
+    $ bundle exec skiplock -h
+    Usage: skiplock [options]
+      -e, --environment STRING         Rails environment
+      -l, --logfile STRING             Full path to logfile
+      -s, --graceful-shutdown NUM      Number of seconds to wait for graceful shutdown
+      -r, --max-retries NUM            Number of maxixum retries
+      -t, --max-threads NUM            Number of maximum threads
+      -T, --min-threads NUM            Number of minimum threads
+      -w, --workers NUM                Number of workers
+      -h, --help                       Show this message
     ```
 
 ## Usage
-
-- Inside the Rails application, queue your job:
+Inside the Rails application:
+- queue your job
     ```ruby
     MyJob.perform_later
     ```
-- Skiplock supports all ActiveJob features:
+- Skiplock supports all ActiveJob features
     ```ruby
     MyJob.set(queue: 'my_queue', wait: 5.minutes, priority: 10).perform_later(1,2,3)
     ```
-- Outside of Rails application, queue the jobs by inserting the job records directly to the database table eg:
+Outside the Rails application:
+- queue the jobs by inserting the job records directly to the database table
     ```sql
     INSERT INTO skiplock.jobs(job_class) VALUES ('MyJob');
     ```
-- Or with scheduling, priority, queue and arguments:
+- with scheduling, priority, queue and arguments
     ```sql
-    INSERT INTO skiplock.jobs(job_class,queue_name,priority,scheduled_at,data) VALUES ('MyJob','my_queue',10,NOW()+INTERVAL '5 min','{"arguments":[1,2,3]}');
+    INSERT INTO skiplock.jobs(job_class, queue_name, priority, scheduled_at, data)
+      VALUES ('MyJob', 'my_queue', 10, NOW() + INTERVAL '5 min', '{"arguments":[1,2,3]}');
     ```
-## Queues priority vs Job priority
-*Why do queues have priorities when jobs already have priorities?*
+## Queue priority vs Job priority
+*Why do queues use priorities when jobs already have priorities?*
 - Jobs are only prioritized with other jobs from the same queue
 - Queues, on the other hand, are prioritized with other queues
-- Rails has built-in queues that dispatches jobs without priorities (eg. Mail Delivery will queue as **mailers** with no priority)
+- Rails has built-in queues that dispatch jobs without priorities (eg. Mail Delivery will queue as **mailers** with no priority)
 
 ## Cron system
 `Skiplock` provides the capability to setup cron jobs for running tasks periodically.  It fully supports the cron syntax to specify the frequency of the jobs.  To setup a cron job, simply assign a valid cron schedule to the constant `CRON` for the Job Class.
@@ -135,7 +137,7 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
 - to remove the cron schedule from the job, simply comment out the constant definition or delete the line then re-deploy the application. At startup, the cron jobs that were undefined will be removed automatically
 
 ## Retry system
-`Skiplock` fully supports ActiveJob built-in retry system.  It also has its own retry system for fallback.  To use ActiveJob retry system, define the rescue blocks per ActiveJob's documentation.
+`Skiplock` fully supports ActiveJob built-in retry system.  It also has its own retry system for fallback.  To use ActiveJob retry system, define the `retry_on` block per ActiveJob's documentation.
 - configures `MyJob` to retry at maximum 20 attempts on StandardError with fixed delay of 5 seconds
     ```ruby
     class MyJob < ActiveJob::Base
@@ -151,21 +153,37 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     # ...
     end
     ```
-Once the retry attempt limit configured in ActiveJob has been reached, the control will be passed back to `skiplock` to be marked as an expired job.
+If the retry attempt limit configured in ActiveJob has been reached, then the control will be passed back to `skiplock` to be marked as an expired job.
 
-If the rescue blocks are not defined, then the built-in retry system of `skiplock` will kick in automatically.  The retrying schedule is using an exponential formula (5 + 2**attempt).  The `skiplock` configuration `max_retries` determines the the limit of attempts before the failing job is marked as expired.  The maximum retry limit can be set as high as 20; this allows up to 12 days of retrying before the job is marked as expired.
+If the `retry_on` block is not defined, then the built-in retry system of `skiplock` will kick in automatically.  The retrying schedule is using an exponential formula (5 + 2**attempt).  The `skiplock` configuration `max_retries` determines the the limit of attempts before the failing job is marked as expired.  The maximum retry limit can be set as high as 20; this allows up to 12 days of retrying before the job is marked as expired.
 
 ## Notification system
-`Skiplock` can use existing exception notification library to notify errors and exceptions.  It supports `airbrake`, `bugsnag`, and `exception_notification`.  A customized function can also be called whenever an exception occurs; this can be configured in an initializer as below:
+`Skiplock` can use existing exception notification library to notify errors and exceptions.  It supports `airbrake`, `bugsnag`, and `exception_notification`.  Custom notification can also be called whenever an exception occurs; it can be configured in an initializer like below:
 ```ruby
   # config/initializers/skiplock.rb
-  Skiplock.on_error = -> (ex, previous = nil) do
+  Skiplock.on_error do |ex, previous|
     if ex.backtrace != previous.try(:backtrace)
       # sends custom email on new exceptions only
       # the same repeated exceptions will only be sent once to avoid SPAM
+      # NOTE: exceptions generated from Job executions will not provide 'previous' exceptions
     end
   end
+  # supports multiple 'on_error' event callbacks
 ```
+## ClassMethod extension
+`Skiplock` can add extension to allow all class methods to be performed as a background job; it is disabled in the default configuration.  To enable, edit the `config/skiplock.yml` configuration file and change `extensions` to `true`.
+- Queue class method `generate_thumbnails` of class `Image` as background job to run as soon as possible
+  ```ruby
+  Image.skiplock.generate_thumbnails(height: 100, ratio: true)
+  ```
+- Queue class method `cleanup` of class `Session` as background job on queue `maintenance` to run after 5 minutes
+  ```ruby
+  Session.skiplock(wait: 5.minutes, queue: 'maintenance').cleanup
+  ```
+- Queue class method `charge` of class `Subscription` as background job to run tomorrow at noon
+  ```ruby
+  Subscription.skiplock(wait_until: Date.tomorrow.noon).charge(amount: 100)
+  ```
 
 ## Contributing
 

data/bin/skiplock

@@ -1,3 +1,28 @@
 #!/usr/bin/env ruby
+require 'optparse'
+options = {}
+begin
+  op = OptionParser.new do |opts|
+    opts.banner = "Usage: #{File.basename($0)} [options]"
+    opts.on('-e', '--environment STRING', String, 'Rails environment')
+    opts.on('-l', '--logfile STRING', String, 'Full path to logfile')
+    opts.on('-s', '--graceful-shutdown NUM', Integer, 'Number of seconds to wait for graceful shutdown')
+    opts.on('-r', '--max-retries NUM', Integer, 'Number of maxixum retries')
+    opts.on('-t', '--max-threads NUM', Integer, 'Number of maximum threads')
+    opts.on('-T', '--min-threads NUM', Integer, 'Number of minimum threads')
+    opts.on('-w', '--workers NUM', Integer, 'Number of workers')
+    opts.on_tail('-h', '--help', 'Show this message') do
+      exit
+    end
+  end
+  op.parse!(into: options)
+rescue Exception => e
+  puts "\n#{e.message}\n\n" unless e.is_a?(SystemExit)
+  puts op
+  exit
+end
+options.transform_keys! { |k| k.to_s.gsub('-', '_').to_sym }
+env = options.delete(:environment)
+ENV['RAILS_ENV'] = env if env
 require File.expand_path("config/environment.rb")
-Skiplock::Manager.start(standalone: true)
+Skiplock::Manager.new(**options.merge(standalone: true))

data/lib/active_job/queue_adapters/skiplock_adapter.rb

@@ -2,11 +2,11 @@ module ActiveJob
   module QueueAdapters
     class SkiplockAdapter
       def initialize
-        Skiplock::Manager.start
+        Rails.application.config.after_initialize { Skiplock::Manager.new }
       end
 
       def enqueue(job)
-        enqueue_at(job, nil)
+        Skiplock::Job.enqueue(job)
       end
 
       def enqueue_at(job, timestamp)

data/lib/generators/skiplock/install_generator.rb

@@ -12,7 +12,7 @@ module Skiplock
     end
 
     def create_config_file
-      create_file 'config/skiplock.yml', Skiplock::Settings.to_yaml
+      create_file 'config/skiplock.yml', Skiplock::DEFAULT_CONFIG.to_yaml
     end
 
     def create_migration_file

data/lib/generators/skiplock/templates/migration.rb.erb

@@ -1,6 +1,14 @@
 class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]" %>
   def up
     execute 'CREATE SCHEMA skiplock'
+    create_table 'skiplock.counters', id: :uuid do |t|
+      t.integer   :completions, null: false, default: 0
+      t.integer   :dispatches, null: false, default: 0
+      t.integer   :expiries, null: false, default: 0
+      t.integer   :failures, null: false, default: 0
+      t.integer   :retries, null: false, default: 0
+      t.date      :day, null: false, index: { unique: true }
+    end
     create_table 'skiplock.jobs', id: :uuid do |t|
       t.uuid      :worker_id, index: true
       t.string    :job_class, null: false
@@ -19,10 +27,11 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
       t.timestamps null: false, default: -> { 'now()' }
     end
     create_table 'skiplock.workers', id: :uuid do |t|
-      t.integer   :pid, null: false, index: true
-      t.integer   :ppid, index: true
+      t.integer   :pid, null: false
+      t.integer   :sid, null: false
       t.integer   :capacity, null: false
       t.string    :hostname, null: false, index: true
+      t.boolean   :master, null: false, default: false, index: true
       t.jsonb     :data
       t.timestamps null: false, index: true, default: -> { 'now()' }
     end    
@@ -31,12 +40,27 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
         DECLARE
           record RECORD;
         BEGIN
+          record = NEW;
           IF (TG_OP = 'DELETE') THEN
             record = OLD;
-          ELSE
-            record = NEW;
+            IF (record.finished_at IS NOT NULL OR record.expired_at IS NOT NULL) THEN
+              RETURN NULL;
+            END IF;
+            INSERT INTO skiplock.counters (day,completions) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET completions = skiplock.counters.completions + 1;
+          ELSIF (record.running = TRUE) THEN
+            IF (record.executions IS NULL) THEN
+              INSERT INTO skiplock.counters (day,dispatches) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET dispatches = skiplock.counters.dispatches + 1;
+            ELSE
+              INSERT INTO skiplock.counters (day,retries) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET retries = skiplock.counters.retries + 1;
+            END IF;
+          ELSIF (record.finished_at IS NOT NULL) THEN
+            INSERT INTO skiplock.counters (day,completions) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET completions = skiplock.counters.completions + 1;
+          ELSIF (record.expired_at IS NOT NULL) THEN
+            INSERT INTO skiplock.counters (day,expiries) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET expiries = skiplock.counters.expiries + 1;
+          ELSIF (record.executions IS NOT NULL AND record.scheduled_at IS NOT NULL) THEN
+            INSERT INTO skiplock.counters (day,failures) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET failures = skiplock.counters.failures + 1;
           END IF;
-          PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.scheduled_at) AS FLOAT)::TEXT));
+          PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM CASE WHEN record.scheduled_at IS NULL THEN record.updated_at ELSE record.scheduled_at END) AS FLOAT)::TEXT));
           RETURN NULL;
         END;
       $$ LANGUAGE plpgsql
@@ -51,7 +75,7 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
           ELSE
             record = NEW;
           END IF;
-          PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.hostname,',',record.capacity,',',record.pid,',',record.ppid));
+          PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.hostname,',',record.master::TEXT,',',record.capacity,',',record.pid,',',record.sid,',',CAST(EXTRACT(EPOCH FROM record.created_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.updated_at) AS FLOAT)::TEXT));
           RETURN NULL;
         END;
       $$ LANGUAGE plpgsql;
@@ -62,6 +86,7 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
     execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE running = FALSE AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
     execute "CREATE INDEX jobs_cron_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE cron IS NOT NULL AND finished_at IS NULL"
     execute "CREATE UNIQUE INDEX jobs_unique_cron_index ON skiplock.jobs (job_class) WHERE cron IS NOT NULL"
+    execute "CREATE UNIQUE INDEX workers_unique_master_index ON skiplock.workers(hostname) WHERE master = 't'"
   end
 
   def down

data/lib/skiplock/counter.rb

@@ -0,0 +1,4 @@
+module Skiplock
+  class Counter < ActiveRecord::Base
+  end
+end

data/lib/skiplock/cron.rb

@@ -2,17 +2,18 @@ require 'cron_parser'
 module Skiplock
   class Cron
     def self.setup
+      Rails.application.eager_load! if Rails.env.development?
       cronjobs = []
       ActiveJob::Base.descendants.each do |j|
         next unless j.const_defined?('CRON')
         cron = j.const_get('CRON')
-        job = Job.find_by('job_class = ? AND cron IS NOT NULL', j.name) || Job.new(job_class: j.name, cron: cron)
+        job = Job.find_by('job_class = ? AND cron IS NOT NULL', j.name) || Job.new(job_class: j.name, cron: cron, locale: I18n.locale, timezone: Time.zone.name)
         time = self.next_schedule_at(cron)
         if time
           job.cron = cron
           job.running = false
           job.scheduled_at = Time.at(time)
-          job.save!
+          job.save
           cronjobs << j.name
         end
       end

data/lib/skiplock/dispatcher.rb

@@ -1,132 +1,115 @@
 module Skiplock
   class Dispatcher
-    def initialize(master: true, worker_num: nil, worker_pids: [])
-      @queues_order_query = Settings['queues'].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if Settings['queues'].is_a?(Hash) && Settings['queues'].count > 0
-      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: Settings['min_threads'], max_threads: Settings['max_threads'], max_queue: Settings['max_threads'], idletime: 60, auto_terminate: true, fallback_policy: :discard)
-      @master = master
-      if @master
-        @worker_pids = worker_pids + [ Process.pid ]
-      else
-        @worker_num = worker_num
-      end
+    def initialize(worker:, worker_num: nil, **config)
+      @config = config
+      @worker = worker
+      @queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
+      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads], max_threads: @config[:max_threads], max_queue: @config[:max_threads], idletime: 60, auto_terminate: true, fallback_policy: :discard)
+      @last_dispatch_at = 0
       @next_schedule_at = Time.now.to_f
-      @running = true
+      Process.setproctitle("skiplock-#{@worker.master ? 'master[0]' : 'worker[' + worker_num.to_s + ']'}") if @config[:standalone]
     end
     
     def run
+      @running = true
       Thread.new do
-        Rails.application.reloader.wrap do
-          sleep(0.1) while @running && !Rails.application.initialized?
-          Process.setproctitle("skiplock-#{@master ? 'master[0]' : 'worker[' + @worker_num.to_s + ']'}") if Settings['workers'] > 0
-          ActiveRecord::Base.connection_pool.with_connection do |connection|
-            connection.exec_query('LISTEN "skiplock::jobs"')
-            hostname = `hostname -f`.strip
-            @worker = Worker.create!(pid: Process.pid, ppid: (@master ? nil : Process.ppid), capacity: Settings['max_threads'], hostname: hostname)
-            if @master
-              if File.exists?('tmp/cache/skiplock')
-                # get performed jobs that could not sync with database
-                job_ids = File.read('tmp/cache/skiplock').split("\n")
-                if Settings['purge_completion']
-                  Job.where(id: job_ids, running: true).delete_all
-                else
-                  Job.where(id: job_ids, running: true).update_all(running: false, finished_at: File.mtime('tmp/cache/skiplock'))
+        ActiveRecord::Base.connection_pool.with_connection do |connection|
+          connection.exec_query('LISTEN "skiplock::jobs"')
+          if @worker.master
+            Dir.mkdir('tmp/skiplock') unless Dir.exist?('tmp/skiplock')
+            check_sync_errors
+            Cron.setup
+          end
+          error = false
+          timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          while @running
+            begin
+              if error
+                unless connection.active?
+                  connection.reconnect!
+                  sleep(0.5)
+                  connection.exec_query('LISTEN "skiplock::jobs"')
+                  @next_schedule_at = Time.now.to_f
                 end
-                File.delete('tmp/cache/skiplock')
+                check_sync_errors
+                error = false
               end
-              # get current worker ids
-              worker_ids = Worker.where(hostname: hostname, pid: @worker_pids).ids
-              # reset orphaned jobs of the dead worker ids for retry
-              Job.where(running: true).where.not(worker_id: worker_ids).update_all(running: false, worker_id: nil)
-              # remove workers that were not shutdown properly on the host
-              Worker.where(hostname: hostname).where.not(pid: @worker_pids).delete_all
-              # reset retries schedules on startup
-              Job.where('scheduled_at > NOW() AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL').update_all(scheduled_at: nil, updated_at: Time.now)
-              Cron.setup
-            end
-            error = false
-            while @running
-              begin
-                if error
-                  unless connection.active?
-                    connection.reconnect!
-                    sleep(0.5)
-                    connection.exec_query('LISTEN "skiplock::jobs"')
-                    @next_schedule_at = Time.now
-                  end
-                  error = false
+              job_notifications = []
+              connection.raw_connection.wait_for_notify(0.1) do |channel, pid, payload|
+                job_notifications << payload if payload
+                loop do
+                  payload = connection.raw_connection.notifies
+                  break unless @running && payload
+                  job_notifications << payload[:extra]
                 end
-                if Job::Errors.keys.count > 0
-                  completed_ids = Job::Errors.keys.map { |k| k if Job::Errors[k] }.compact
-                  if Settings['purge_completion'] && completed_ids.count > 0
-                    Job.where(id: completed_ids, running: true).delete_all
-                  elsif completed_ids.count > 0
-                    Job.where(id: completed_ids, running: true).update_all(running: false, finished_at: Time.now)
+                job_notifications.each do |n|
+                  op, id, worker_id, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
+                  next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0  || scheduled_at.to_f < @last_dispatch_at
+                  if scheduled_at.to_f <= Time.now.to_f
+                    @next_schedule_at = Time.now.to_f
+                  elsif scheduled_at.to_f < @next_schedule_at
+                    @next_schedule_at = scheduled_at.to_f
                   end
-                  orphaned_ids = Job::Errors.keys.map { |k| k unless Job::Errors[k] }.compact
-                  Job.where(id: orphaned_ids, running: true).update_all(running: false, worker_id: nil, scheduled_at: (Time.now + 10)) if orphaned_ids.count > 0
-                  Job::Errors.clear
-                end
-                if Time.now.to_f >= @next_schedule_at && @executor.remaining_capacity > 0
-                  @executor.post { do_work }
                 end
-                notifications = []
-                connection.raw_connection.wait_for_notify(0.1) do |channel, pid, payload|
-                  notifications << payload if payload
-                  loop do
-                    payload = connection.raw_connection.notifies
-                    break unless @running && payload
-                    notifications << payload[:extra]
-                  end
-                  notifications.each do |n|
-                    op, id, worker_id, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
-                    next if op == 'DELETE' || running == 'true' || expired_at.to_s.length > 0 || finished_at.to_s.length > 0
-                    if scheduled_at.to_f <= Time.now.to_f
-                      @next_schedule_at = Time.now.to_f
-                    elsif scheduled_at.to_f < @next_schedule_at
-                      @next_schedule_at = scheduled_at.to_f
-                    end
-                  end
-                end
-              rescue Exception => ex
-                STDERR.puts ex.message
-                STDERR.puts ex.backtrace
-                Skiplock.on_error.call(ex, @last_exception) if Skiplock.on_error.is_a?(Proc)
-                error = true
-                t = Time.now
-                while @running
-                  sleep(0.5)
-                  break if Time.now - t > 5
-                end              
-                @last_exception = ex
               end
-              sleep(0.2)
+              if Time.now.to_f >= @next_schedule_at && @executor.remaining_capacity > 0
+                @executor.post { do_work }
+              end
+              if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
+                @worker.touch
+                timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              end
+            rescue Exception => ex
+              # most likely error with database connection
+              Skiplock.logger.error(ex)
+              Skiplock.on_errors.each { |p| p.call(ex, @last_exception) }
+              error = true
+              t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              while @running
+                sleep(0.5)
+                break if Process.clock_gettime(Process::CLOCK_MONOTONIC) - t > 5
+              end
+              @last_exception = ex
             end
-            connection.exec_query('UNLISTEN *')
+            sleep(0.2)
           end
+          connection.exec_query('UNLISTEN *')
+          @executor.shutdown
+          @executor.kill unless @executor.wait_for_termination(@config[:graceful_shutdown])
         end
       end
     end
 
-    def shutdown(wait: true)
+    def shutdown
       @running = false
-      @executor.shutdown
-      @executor.wait_for_termination if wait
-      @worker.delete if @worker
     end
 
     private
 
+    def check_sync_errors
+      # get performed jobs that could not sync with database
+      Dir.glob('tmp/skiplock/*').each do |f|
+        job_from_db = Job.find_by(id: File.basename(f), running: true)
+        disposed = true
+        if job_from_db
+          job, ex = YAML.load_file(f) rescue nil
+          disposed = job.dispose(ex, purge_completion: @config[:purge_completion], max_retries: @config[:max_retries])
+        end
+        File.delete(f) if disposed
+      end
+    end
+
     def do_work
       while @running
-        result = Job.dispatch(queues_order_query: @queues_order_query, worker_id: @worker.id)
-        next if result.is_a?(Job)
+        @last_dispatch_at = Time.now.to_f - 1  # 1 second allowance for time drift
+        result = Job.dispatch(queues_order_query: @queues_order_query, worker_id: @worker.id, purge_completion: @config[:purge_completion], max_retries: @config[:max_retries])
+        next if result.is_a?(Job) && Time.now.to_f >= @next_schedule_at
         @next_schedule_at = result if result.is_a?(Float)
         break
       end
     rescue Exception => ex
-      STDERR.puts ex.message
-      STDERR.puts ex.backtrace
-      Skiplock.on_error.call(ex, @last_exception) if Skiplock.on_error.is_a?(Proc)
+      Skiplock.logger.error(ex)
+      Skiplock.on_errors.each { |p| p.call(ex, @last_exception) }
       @last_exception = ex
     end
   end

data/lib/skiplock/extension.rb

@@ -0,0 +1,25 @@
+module Skiplock
+  module Extension
+    class Proxy < BasicObject
+      def initialize(target, options = {})
+        @target = target
+        @options = options
+      end
+
+      def method_missing(name, *args)
+        ProxyJob.set(@options).perform_later(::YAML.dump([ @target, name, args ]))
+      end
+    end
+
+    class ProxyJob < ActiveJob::Base
+      def perform(yml)
+        target, method_name, args = ::YAML.load(yml)
+        target.__send__(method_name, *args)
+      end
+    end
+
+    def skiplock(options = {})
+      Proxy.new(self, options)
+    end
+  end
+end

data/lib/skiplock/job.rb

@@ -1,73 +1,44 @@
 module Skiplock
   class Job < ActiveRecord::Base
-    self.table_name = 'skiplock.jobs'
-    Errors = Concurrent::Map.new
+    self.implicit_order_column = 'created_at'
 
-    # Return: Skiplock::Job if it was executed; otherwise returns the next Job's schedule time in FLOAT
-    def self.dispatch(queues_order_query: nil, worker_id: nil)
-      performed = false
-      self.connection.exec_query('BEGIN')
-      job = self.find_by_sql("SELECT id, scheduled_at FROM #{self.table_name} WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{queues_order_query ? ' CASE ' + queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
-      if job.nil? || job.scheduled_at.to_f > Time.now.to_f
-        self.connection.exec_query('END')
-        return (job ? job.scheduled_at.to_f : Float::INFINITY)
+    def self.dispatch(queues_order_query: nil, worker_id: nil, purge_completion: true, max_retries: 20)
+      job = nil
+      self.transaction do
+        job = self.find_by_sql("SELECT id, scheduled_at FROM #{self.table_name} WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{queues_order_query ? ' CASE ' + queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
+        return (job ? job.scheduled_at.to_f : Float::INFINITY) if job.nil? || job.scheduled_at.to_f > Time.now.to_f
+        job = Skiplock::Job.find_by_sql("UPDATE #{self.table_name} SET running = TRUE, worker_id = #{self.connection.quote(worker_id)}, updated_at = NOW() WHERE id = '#{job.id}' RETURNING *").first
       end
-      job = Skiplock::Job.find_by_sql("UPDATE #{self.table_name} SET running = TRUE, worker_id = #{self.connection.quote(worker_id)} WHERE id = '#{job.id}' RETURNING *").first
-      self.connection.exec_query('END')
       job.data ||= {}
       job.exception_executions ||= {}
       job_data = job.attributes.slice('job_class', 'queue_name', 'locale', 'timezone', 'priority', 'executions', 'exception_executions').merge('job_id' => job.id, 'enqueued_at' => job.updated_at, 'arguments' => (job.data['arguments'] || []))
       job.executions = (job.executions || 0) + 1
+      Skiplock.logger.info "[Skiplock] Performing #{job.job_class} (#{job.id}) from queue '#{job.queue_name || 'default'}'..."
       Thread.current[:skiplock_dispatch_job] = job
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       begin
         ActiveJob::Base.execute(job_data)
       rescue Exception => ex
+        Skiplock.logger.error(ex)
       end
-      performed = true
-      job.running = false
-      if ex
-        job.exception_executions["[#{ex.class.name}]"] = (job.exception_executions["[#{ex.class.name}]"] || 0) + 1 unless job.exception_executions.key?('activejob_retry')
-        if job.executions >= Settings['max_retries'] || job.exception_executions.key?('activejob_retry')
-          job.expired_at = Time.now
-          job.save!
-        else
-          job.scheduled_at = Time.now + (5 * 2**job.executions)
-          job.save!
-        end
-        Skiplock.on_error.call(ex) if Skiplock.on_error.is_a?(Proc) && (job.executions % 3 == 1)
-      elsif job.exception_executions.key?('activejob_retry')
-        job.save!
-      elsif job.cron
-        job.data['last_cron_run'] = Time.now.utc.to_s
-        next_cron_at = Cron.next_schedule_at(job.cron)
-        if next_cron_at
-          job.executions = 1
-          job.exception_executions = nil
-          job.scheduled_at = Time.at(next_cron_at)
-          job.save!
-        else
-          job.delete
+      unless ex
+        end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        job_name = job.job_class
+        if job.job_class == 'Skiplock::Extension::ProxyJob'
+          target, method_name = ::YAML.load(job.data['arguments'].first)
+          job_name = "'#{target.name}.#{method_name}'"
         end
-      elsif Settings['purge_completion']
-        job.delete
-      else
-        job.finished_at = Time.now
-        job.exception_executions = nil
-        job.save!
+        Skiplock.logger.info "[Skiplock] Performed #{job_name} (#{job.id}) from queue '#{job.queue_name || 'default'}' in #{end_time - start_time} seconds"
       end
-      job
-    rescue Exception => ex
-      if performed
-        Errors[job.id] = true
-        File.write('tmp/cache/skiplock', job.id + "\n", mode: 'a')
-      else
-        Errors[job.id] = false
-      end
-      raise ex
+      job.dispose(ex, purge_completion: purge_completion, max_retries: max_retries)
     ensure
       Thread.current[:skiplock_dispatch_job] = nil
     end
 
+    def self.enqueue(activejob)
+      self.enqueue_at(activejob, nil)
+    end
+
     def self.enqueue_at(activejob, timestamp)
       timestamp = Time.at(timestamp) if timestamp
       if Thread.current[:skiplock_dispatch_job].try(:id) == activejob.job_id
@@ -76,8 +47,57 @@ module Skiplock
         Thread.current[:skiplock_dispatch_job].scheduled_at = timestamp
         Thread.current[:skiplock_dispatch_job]
       else
-        Job.create!(id: activejob.job_id, job_class: activejob.class.name, queue_name: activejob.queue_name, locale: activejob.locale, timezone: activejob.timezone, priority: activejob.priority, executions: activejob.executions, data: { 'arguments' => activejob.serialize['arguments'] }, scheduled_at: timestamp)
+        serialize = activejob.serialize
+        Job.create!(serialize.slice(*self.column_names).merge('id' => serialize['job_id'], 'data' => { 'arguments' => serialize['arguments'] }, 'scheduled_at' => timestamp))
+      end
+    end
+
+    def self.reset_retry_schedules
+      self.where('scheduled_at > NOW() AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL').update_all(scheduled_at: nil, updated_at: Time.now)
+    end
+
+    def dispose(ex, purge_completion: true, max_retries: 20)
+      dup = self.dup
+      self.running = false
+      self.worker_id = nil
+      self.updated_at = (Time.now > self.updated_at ? Time.now : self.updated_at + 1)
+      if ex
+        self.exception_executions["[#{ex.class.name}]"] = (self.exception_executions["[#{ex.class.name}]"] || 0) + 1 unless self.exception_executions.key?('activejob_retry')
+        if self.executions >= max_retries || self.exception_executions.key?('activejob_retry')
+          self.expired_at = Time.now
+        else
+          self.scheduled_at = Time.now + (5 * 2**self.executions)
+        end
+        self.save!
+        Skiplock.on_errors.each { |p| p.call(ex) }
+      elsif self.exception_executions.try(:key?, 'activejob_retry')
+        self.save!
+      elsif self.cron
+        self.data ||= {}
+        self.data['crons'] = (self.data['crons'] || 0) + 1
+        self.data['last_cron_at'] = Time.now.utc.to_s
+        next_cron_at = Cron.next_schedule_at(self.cron)
+        if next_cron_at
+          self.executions = nil
+          self.exception_executions = nil
+          self.scheduled_at = Time.at(next_cron_at)
+          self.save!
+        else
+          Skiplock.logger.error "[Skiplock] ERROR: Invalid CRON '#{self.cron}' for Job #{self.job_class}"
+          self.delete
+        end
+      elsif purge_completion
+        self.delete
+      else
+        self.finished_at = Time.now
+        self.exception_executions = nil
+        self.save!
       end
+      self
+    rescue Exception => e
+      Skiplock.logger.error(e)
+      File.write("tmp/skiplock/#{self.id}", [dup, ex].to_yaml)
+      nil
     end
   end
 end

data/lib/skiplock/manager.rb

@@ -1,152 +1,163 @@
 module Skiplock
   class Manager
-    def self.start(standalone: false, workers: nil, max_retries: nil, max_threads: nil, min_threads: nil, logging: nil)
-      unless Settings.frozen?
-        load_settings
-        Settings['logging'] = logging if logging
-        Settings['max_retries'] = max_retries if max_retries
-        Settings['max_threads'] = max_threads if max_threads
-        Settings['min_threads'] = min_threads if min_threads
-        Settings['workers'] = workers if workers
-        Settings['max_retries'] = 20 if Settings['max_retries'] > 20
-        Settings['max_retries'] = 0 if Settings['max_retries'] < 0
-        Settings['max_threads'] = 1 if Settings['max_threads'] < 1
-        Settings['max_threads'] = 20 if Settings['max_threads'] > 20
-        Settings['min_threads'] = 0 if Settings['min_threads'] < 0
-        Settings['workers'] = 0 if Settings['workers'] < 0
-        Settings.freeze
-      end
-      return unless standalone || (caller.any?{|l| l =~ %r{/rack/}} && (Settings['workers'] == 0 || Rails.env.development?))
-      if standalone
-        self.standalone
+    def initialize(**config)
+      @config = Skiplock::DEFAULT_CONFIG.dup
+      @config.merge!(YAML.load_file('config/skiplock.yml')) rescue nil
+      @config.symbolize_keys!
+      @config.transform_values! {|v| v.is_a?(String) ? v.downcase : v}
+      @config.merge!(config)
+      Module.__send__(:include, Skiplock::Extension) if @config[:extensions] == true
+      return unless @config[:standalone] || (caller.any?{ |l| l =~ %r{/rack/} } && (@config[:workers] == 0 || Rails.env.development?))
+      @config[:hostname] = `hostname -f`.strip
+      do_config
+      banner if @config[:standalone]
+      cleanup_workers
+      create_worker
+      ActiveJob::Base.logger = nil
+      if @config[:standalone]
+        standalone
       else
-        @dispatcher = Dispatcher.new
-        @thread = @dispatcher.run
-        at_exit { self.shutdown }
+        dispatcher = Dispatcher.new(worker: @worker, **@config)
+        thread = dispatcher.run
+        at_exit do
+          dispatcher.shutdown
+          thread.join(@config[:graceful_shutdown])
+          @worker.delete
+        end
       end
+    rescue Exception => ex
+      @logger.error(ex)
+    end
+
+  private
+
+    def banner
+      title = "Skiplock #{Skiplock::VERSION} (Rails #{Rails::VERSION::STRING} | Ruby #{RUBY_VERSION}-p#{RUBY_PATCHLEVEL})"
+      @logger.info "-"*(title.length)
+      @logger.info title
+      @logger.info "-"*(title.length)
+      @logger.info "ClassMethod extensions: #{@config[:extensions]}"
+      @logger.info "      Purge completion: #{@config[:purge_completion]}"
+      @logger.info "          Notification: #{@config[:notification]}"
+      @logger.info "           Max retries: #{@config[:max_retries]}"
+      @logger.info "           Min threads: #{@config[:min_threads]}"
+      @logger.info "           Max threads: #{@config[:max_threads]}"
+      @logger.info "           Environment: #{Rails.env}"
+      @logger.info "               Logfile: #{@config[:logfile] || '(disabled)'}"
+      @logger.info "               Workers: #{@config[:workers]}"
+      @logger.info "                Queues: #{@config[:queues].map {|k,v| k + '(' + v.to_s + ')'}.join(', ')}" if @config[:queues].is_a?(Hash)
+      @logger.info "                   PID: #{Process.pid}"
+      @logger.info "-"*(title.length)
+      @logger.warn "[Skiplock] Custom notification has no registered 'on_error' callback" if Skiplock.on_errors.count == 0
     end
-    
-    def self.shutdown(wait: true)
-      if @dispatcher && @thread
-        @dispatcher.shutdown(wait: wait)
-        @thread.join
+
+    def cleanup_workers
+      delete_ids = []
+      Worker.where(hostname: @config[:hostname]).each do |worker|
+        sid = Process.getsid(worker.pid) rescue nil
+        delete_ids << worker.id if worker.sid != sid || worker.updated_at < 30.minutes.ago
+      end
+      if delete_ids.count > 0
+        Job.where(running: true, worker_id: delete_ids).update_all(running: false, worker_id: nil)
+        Worker.where(id: delete_ids).delete_all
       end
     end
 
-  private
+    def create_worker(pid: Process.pid, sid: Process.getsid(), master: true)
+      @worker = Worker.create!(pid: pid, sid: sid, master: master, hostname: @config[:hostname], capacity: @config[:max_threads])
+    rescue
+      @worker = Worker.create!(pid: pid, sid: sid, master: false, hostname: @config[:hostname], capacity: @config[:max_threads])
+    end
 
-    def self.load_settings
-      config = YAML.load_file('config/skiplock.yml') rescue {}
-      Settings.merge!(config)
-      Settings['queues'].values.each { |v| raise 'Queue value must be an integer' unless v.is_a?(Integer) } if Settings['queues'].is_a?(Hash)
-      case Settings['notification'].to_s.downcase
-      when 'auto'
+    def do_config
+      @config[:graceful_shutdown] = 300 if @config[:graceful_shutdown] > 300
+      @config[:graceful_shutdown] = nil if @config[:graceful_shutdown] < 0
+      @config[:max_retries] = 20 if @config[:max_retries] > 20
+      @config[:max_retries] = 0 if @config[:max_retries] < 0
+      @config[:max_threads] = 1 if @config[:max_threads] < 1
+      @config[:max_threads] = 20 if @config[:max_threads] > 20
+      @config[:min_threads] = 0 if @config[:min_threads] < 0
+      @config[:workers] = 0 if @config[:workers] < 0
+      @config[:workers] = 1 if @config[:standalone] && @config[:workers] <= 0
+      @logger = ActiveSupport::Logger.new(STDOUT)
+      @logger.level = Rails.logger.level
+      Skiplock.logger = @logger
+      raise "Cannot create logfile '#{@config[:logfile]}'" if @config[:logfile] && !File.writable?(File.dirname(@config[:logfile]))
+      @config[:logfile] = nil if @config[:logfile].to_s.length == 0
+      if @config[:logfile]
+        @logger.extend(ActiveSupport::Logger.broadcast(::Logger.new(@config[:logfile])))
+        if @config[:standalone]
+          Rails.logger.reopen('/dev/null')
+          Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
+        end
+      end
+      @config[:queues].values.each { |v| raise 'Queue value must be an integer' unless v.is_a?(Integer) } if @config[:queues].is_a?(Hash)
+      if @config[:notification] == 'auto'
         if defined?(Airbrake)
-          Skiplock.on_error = -> (ex, previous = nil) { Airbrake.notify_sync(ex) unless ex.backtrace == previous.try(:backtrace) }
+          @config[:notification] = 'airbrake'
         elsif defined?(Bugsnag)
-          Skiplock.on_error = -> (ex, previous = nil) { Bugsnag.notify(ex) unless ex.backtrace == previous.try(:backtrace) }
+          @config[:notification] = 'bugsnag'
         elsif defined?(ExceptionNotifier)
-          Skiplock.on_error = -> (ex, previous = nil) { ExceptionNotifier.notify_exception(ex) unless ex.backtrace == previous.try(:backtrace) }
+          @config[:notification] = 'exception_notification'
         else
-          puts "Unable to detect any known exception notification gem. Please define custom 'on_error' function and disable notification in 'config/skiplock.yml'"
-          exit
+          raise "Unable to detect any known exception notification library. Please define custom 'on_error' event callbacks and change to 'custom' notification in 'config/skiplock.yml'"
         end
+      end
+      case @config[:notification]
       when 'airbrake'
         raise 'airbrake gem not found' unless defined?(Airbrake)
-        Skiplock.on_error = -> (ex, previous = nil) { Airbrake.notify_sync(ex) unless ex.backtrace == previous.try(:backtrace) }
+        Skiplock.on_error do |ex, previous|
+          Airbrake.notify_sync(ex) unless ex.backtrace == previous.try(:backtrace)
+        end
       when 'bugsnag'
         raise 'bugsnag gem not found' unless defined?(Bugsnag)
-        Skiplock.on_error = -> (ex, previous = nil) { Bugsnag.notify(ex) unless ex.backtrace == previous.try(:backtrace) }
+        Skiplock.on_error do |ex, previous|
+          Bugsnag.notify(ex) unless ex.backtrace == previous.try(:backtrace)
+        end
       when 'exception_notification'
         raise 'exception_notification gem not found' unless defined?(ExceptionNotifier)
-        Skiplock.on_error = -> (ex, previous = nil) { ExceptionNotifier.notify_exception(ex) unless ex.backtrace == previous.try(:backtrace) }
+        Skiplock.on_error do |ex, previous|
+          ExceptionNotifier.notify_exception(ex) unless ex.backtrace == previous.try(:backtrace)
+        end
+      else
+        @config[:notification] = 'custom'
       end
-    rescue Exception => e
-      STDERR.puts "Invalid configuration 'config/skiplock.yml': #{e.message}"
-      exit
+      Skiplock.on_errors.freeze unless Skiplock.on_errors.frozen?
     end
 
-    def self.standalone
-      if Settings['logging']
-        log_timestamp = (Settings['logging'].to_s == 'timestamp')
-        logfile = File.open('log/skiplock.log', 'a')
-        logfile.sync = true
-        $stdout = Demux.new(logfile, STDOUT, timestamp: log_timestamp)
-        errfile = File.open('log/skiplock.error.log', 'a')
-        errfile.sync = true
-        $stderr = Demux.new(errfile, STDERR, timestamp: log_timestamp)
-        logger = ActiveSupport::Logger.new($stdout)
-        logger.level = Rails.logger.level
-        Rails.logger.reopen('/dev/null')
-        Rails.logger.extend(ActiveSupport::Logger.broadcast(logger))
-      end
-      title = "Skiplock version: #{Skiplock::VERSION} (Ruby #{RUBY_VERSION}-p#{RUBY_PATCHLEVEL})"
-      puts "-"*(title.length)
-      puts title
-      puts "-"*(title.length)
-      puts "Purge completion: #{Settings['purge_completion']}"
-      puts "    Notification: #{Settings['notification']}"
-      puts "     Max retries: #{Settings['max_retries']}"
-      puts "     Min threads: #{Settings['min_threads']}"
-      puts "     Max threads: #{Settings['max_threads']}"
-      puts "     Environment: #{Rails.env}"
-      puts "         Logging: #{Settings['logging']}"
-      puts "         Workers: #{Settings['workers']}"
-      puts "          Queues: #{Settings['queues'].map {|k,v| k + '(' + v.to_s + ')'}.join(', ')}" if Settings['queues'].is_a?(Hash)
-      puts "             PID: #{Process.pid}"
-      puts "-"*(title.length)
+    def standalone
       parent_id = Process.pid
       shutdown = false
       Signal.trap("INT") { shutdown = true }
       Signal.trap("TERM") { shutdown = true }
-      worker_pids = []
-      (Settings['workers']-1).times do |n|
-        worker_pids << fork do
-          dispatcher = Dispatcher.new(master: false, worker_num: n+1)
+      (@config[:workers] - 1).times do |n|
+        fork do
+          sleep 1
+          worker = create_worker(master: false)
+          dispatcher = Dispatcher.new(worker: worker, worker_num: n + 1, **@config)
           thread = dispatcher.run
           loop do
             sleep 0.5
             break if shutdown || Process.ppid != parent_id
           end
-          dispatcher.shutdown(wait: true)
-          thread.join
+          dispatcher.shutdown
+          thread.join(@config[:graceful_shutdown])
+          worker.delete
           exit
         end
       end
-      sleep 0.1
-      dispatcher = Dispatcher.new(worker_pids: worker_pids)
+      dispatcher = Dispatcher.new(worker: @worker, **@config)
       thread = dispatcher.run
       loop do
         sleep 0.5
         break if shutdown
       end
+      @logger.info "[Skiplock] Terminating signal... Waiting for jobs to finish (up to #{@config[:graceful_shutdown]} seconds)..." if @config[:graceful_shutdown]
       Process.waitall
-      dispatcher.shutdown(wait: true)
-      thread.join
-    end
-
-    class Demux
-      def initialize(*targets, timestamp: true)
-        @targets = targets
-        @timestamp = timestamp
-      end
-
-      def close
-        @targets.each(&:close)
-      end
-
-      def flush
-        @targets.each(&:flush)
-      end
-  
-      def tty?
-        true
-      end
-
-      def write(*args)
-        args.prepend("[#{Time.now.utc}]: ") if @timestamp
-        @targets.each {|t| t.write(*args)}
-      end
+      dispatcher.shutdown
+      thread.join(@config[:graceful_shutdown])
+      @worker.delete
+      @logger.info "[Skiplock] Shutdown completed."
     end
   end
 end

data/lib/skiplock/version.rb

@@ -1,4 +1,4 @@
 module Skiplock
-  VERSION = Version = '1.0.7'
+  VERSION = Version = '1.0.11'
 end
 

data/lib/skiplock/worker.rb

@@ -1,5 +1,5 @@
 module Skiplock
   class Worker < ActiveRecord::Base
-    self.table_name = 'skiplock.workers'
+    self.implicit_order_column = 'created_at'
   end
 end

data/lib/skiplock.rb

@@ -1,26 +1,37 @@
 require 'active_job'
 require 'active_job/queue_adapters/skiplock_adapter'
 require 'active_record'
+require 'skiplock/counter'
 require 'skiplock/cron'
 require 'skiplock/dispatcher'
-require 'skiplock/manager'
+require 'skiplock/extension'
 require 'skiplock/job'
+require 'skiplock/manager'
 require 'skiplock/worker'
 require 'skiplock/version'
 
 module Skiplock
-  Settings = { 
-    'logging' => 'timestamp',
-    'min_threads' => 1,
-    'max_threads' => 5,
-    'max_retries' => 20,
-    'notification' => 'auto',
-    'purge_completion' => true,
-    'queues' => {
-      'default' => 200,
-      'mailers' => 100
-    },
-    'workers' => 0
-  }
-  mattr_accessor :on_error
+  DEFAULT_CONFIG = { 'extensions' => false, 'logfile' => 'log/skiplock.log', 'graceful_shutdown' => 15, 'min_threads' => 1, 'max_threads' => 5, 'max_retries' => 20, 'notification' => 'custom', 'purge_completion' => true, 'queues' => { 'default' => 100, 'mailers' => 999 }, 'workers' => 0 }.freeze
+
+  def self.logger=(l)
+    @logger = l
+  end
+
+  def self.logger
+    @logger
+  end
+
+  def self.on_error(&block)
+    @on_errors ||= []
+    @on_errors << block
+    block
+  end
+
+  def self.on_errors
+    @on_errors || []
+  end
+
+  def self.table_name_prefix
+    'skiplock.'
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skiplock
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.11
 platform: ruby
 authors:
 - Tin Vo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-23 00:00:00.000000000 Z
+date: 2021-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -82,8 +82,10 @@ files:
 - lib/generators/skiplock/install_generator.rb
 - lib/generators/skiplock/templates/migration.rb.erb
 - lib/skiplock.rb
+- lib/skiplock/counter.rb
 - lib/skiplock/cron.rb
 - lib/skiplock/dispatcher.rb
+- lib/skiplock/extension.rb
 - lib/skiplock/job.rb
 - lib/skiplock/manager.rb
 - lib/skiplock/version.rb
@@ -107,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: ActiveJob Queue Adapter for PostgreSQL