skiplock 1.0.22 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b99466d7f1848f9dcd750a4dceb6a39d7124707e941c74b6c75f0b3a57944ff
-  data.tar.gz: 3f3ed1c9327d41e675d5c170590b52fe32f1facc5b57667dba513e5324e6ee20
+  metadata.gz: 71cf3158d71dfe424c218da13a55d785365bee9027f827bb1498ba5e0110fda9
+  data.tar.gz: 1512863ab863b392f84d67a137959227a823ac33dd9ca8e1ab8e87c63fe04adf
 SHA512:
-  metadata.gz: '0129296507b0479ba28a6e4e0bce810340f0ff2559004da1385cf02b6d6023f7be5cb1543f3fb24cebdeb579d6be6ed1e5d7d72d43245de5374926607b23d7df'
-  data.tar.gz: 163faf212d06b6d7f8045eb347ccab77de5b16bfe77fe8780a5d73176ce3c5804cc25cbca8f962d92bc5dfccc1ada223d03fc0b443d0df48125a12bf4ec7ddeb
+  metadata.gz: 79ef005a284b8aabfff1e1435bba2019a3b3698f714803f6521562f143b3d5f4f6536991a040bbd44574a52fa3be2df7ac565c37cccf4da40da95fe4b150b1c6
+  data.tar.gz: 1f7fd443df703c98133b66838c6d6d49a9cfa3b54d33cafaff15d59fd3a538591400fdb917ccfb5ec4b8a2831d88b6a1f85ef622ecb7566842f2f761e97bba9f

data/README.md

@@ -4,7 +4,7 @@
 
 It only uses the `LISTEN/NOTIFY/SKIP LOCKED` features provided natively on PostgreSQL 9.5+ to efficiently and reliably dispatch jobs to worker processes and threads ensuring that each job can be completed successfully **only once**.  No other polling or timer is needed.
 
-The library is quite small compared to other PostgreSQL job queues (eg. *delay_job*, *queue_classic*, *que*, *good_job*) with less than 500 lines of codes; and it still provides similar set of features and more...
+The library is quite small compared to other PostgreSQL job queues (eg. *delay_job*, *queue_classic*, *que*, *good_job*) with less than 600 lines of codes; and it still provides similar set of features and more...
 
 #### Compatibility:
 
@@ -50,11 +50,13 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     ```yaml
     # config/skiplock.yml (default settings)
     ---
+    graceful_shutdown: 15
     min_threads: 1
     max_threads: 10
     max_retries: 20
     logfile: skiplock.log
     loglevel: info
+    namespace:
     notification: custom
     extensions: false
     purge_completion: true
@@ -64,11 +66,13 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     workers: 0
     ```
     Available configuration options are:
+    - **graceful_shutdown** (*integer*): sets the number of seconds to wait for jobs to finish before being killed during shutdown
     - **min_threads** (*integer*): sets minimum number of threads staying idle
     - **max_threads** (*integer*): sets the maximum number of threads allowed to run jobs
     - **max_retries** (*integer*): sets the maximum attempt a job will be retrying before it is marked expired.  See `Retry system` for more details
     - **logfile** (*string*): filename for skiplock logs; empty logfile will disable logging
     - **loglevel** (*string*): sets logging level (`debug, info, warn, error, fatal, unknown`)
+    - **namespace** (*string*): sets namespace for jobs (workers will only process jobs of specified namespace)
     - **notification** (*string*): sets the library to be used for notifying errors and exceptions (`auto, airbrake, bugsnag, exception_notification, custom`); using `auto` will detect library if available.  See `Notification system` for more details
     - **extensions** (*multi*): enable or disable the class method extension.  See `ClassMethod extension` for more details
     - **purge_completion** (*boolean*): when set to **true** will delete jobs after they were completed successfully; if set to **false** then the completed jobs should be purged periodically to maximize performance (eg. clean up old jobs after 3 months); queued jobs can manually override using `purge` option
@@ -85,6 +89,8 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     Usage: skiplock [options]
       -e, --environment STRING         Rails environment
       -l, --logfile STRING             Log filename
+      -L, --loglevel STRING            Log level (debug, info, warn, error, fatal, unknown)
+      -n, --namespace STRING           Job namespace
       -s, --graceful-shutdown NUM      Number of seconds to wait for graceful shutdown
       -r, --max-retries NUM            Number of maxixum retries
       -t, --max-threads NUM            Number of maximum threads
@@ -237,6 +243,10 @@ Code examples of gathering counters information:
     ```ruby
     Skiplock::Counter.sum(:expiries)
     ```
+  - get all information in one query
+    ```ruby
+    Skiplock::Counter.pluck("sum(completions), sum(dispatches), sum(expiries), sum(failures), sum(retries)").first
+    ```
 
 ## Contributing
 

data/bin/skiplock

@@ -8,6 +8,7 @@ begin
     opts.on('-e', '--environment STRING', String, 'Rails environment')
     opts.on('-l', '--logfile STRING', String, 'Log filename')
     opts.on('-L', '--loglevel STRING', String, 'Log level (debug, info, warn, error, fatal, unknown)')
+    opts.on('-n', '--namespace STRING', String, 'Job namespace')
     opts.on('-s', '--graceful-shutdown NUM', Integer, 'Number of seconds to wait for graceful shutdown')
     opts.on('-r', '--max-retries NUM', Integer, 'Number of maxixum retries')
     opts.on('-t', '--max-threads NUM', Integer, 'Number of maximum threads')

data/lib/generators/skiplock/templates/migration.rb.erb

@@ -7,11 +7,14 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
       t.integer   :expiries, null: false, default: 0
       t.integer   :failures, null: false, default: 0
       t.integer   :retries, null: false, default: 0
-      t.date      :day, null: false, index: { unique: true }
+      t.string    :namespace, null: false, default: '', index: true
+      t.date      :day, null: false
+      t.index     [ :namespace, :day ], unique: true
     end
     create_table 'skiplock.jobs', id: :uuid do |t|
       t.uuid      :worker_id, index: true
       t.string    :job_class, null: false
+      t.string    :namespace, null: false, default: '', index: true
       t.string    :queue_name, index: true
       t.string    :locale
       t.string    :timezone
@@ -31,6 +34,7 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
       t.integer   :sid, null: false
       t.integer   :capacity, null: false
       t.string    :hostname, null: false, index: true
+      t.string    :namespace, null: false, default: '', index: true
       t.boolean   :master, null: false, default: false, index: true
       t.jsonb     :data
       t.timestamps null: false, index: true, default: -> { 'now()' }
@@ -41,27 +45,29 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
           record RECORD;
         BEGIN
           record = NEW;
-          IF (TG_OP = 'DELETE') THEN
+          IF TG_OP = 'DELETE' THEN
             record = OLD;
-            IF (record.running = TRUE) THEN
-              INSERT INTO skiplock.counters (day,completions) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET completions = skiplock.counters.completions + 1;
+            IF record.running IS TRUE THEN
+              INSERT INTO skiplock.counters (namespace,day,completions) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET completions = skiplock.counters.completions + 1;
             END IF;
-          ELSIF (TG_OP = 'UPDATE') THEN
-            IF (OLD.running = FALSE AND record.running = TRUE) THEN
-              IF (record.executions > 0) THEN
-                INSERT INTO skiplock.counters (day,retries) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET retries = skiplock.counters.retries + 1;
+          ELSIF TG_OP = 'UPDATE' THEN
+            IF OLD.running IS FALSE AND record.running IS TRUE THEN
+              IF record.executions > 0 THEN
+                INSERT INTO skiplock.counters (namespace,day,retries) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET retries = skiplock.counters.retries + 1;
               ELSE
-                INSERT INTO skiplock.counters (day,dispatches) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET dispatches = skiplock.counters.dispatches + 1;
+                INSERT INTO skiplock.counters (namespace,day,dispatches) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET dispatches = skiplock.counters.dispatches + 1;
+              END IF;
+            ELSIF OLD.finished_at IS NULL AND record.finished_at IS NOT NULL THEN
+              INSERT INTO skiplock.counters (namespace,day,completions) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET completions = skiplock.counters.completions + 1;
+            ELSIF OLD.running IS TRUE AND record.running IS FALSE THEN
+              IF record.expired_at IS NOT NULL THEN
+                INSERT INTO skiplock.counters (namespace,day,expiries) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET expiries = skiplock.counters.expiries + 1;
+              ELSE
+                INSERT INTO skiplock.counters (namespace,day,failures) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET failures = skiplock.counters.failures + 1;
               END IF;
-            ELSIF (OLD.finished_at IS NULL AND record.finished_at IS NOT NULL) THEN
-              INSERT INTO skiplock.counters (day,completions) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET completions = skiplock.counters.completions + 1;
-            ELSIF (OLD.running = TRUE AND record.running = FALSE AND record.expired_at IS NOT NULL) THEN
-              INSERT INTO skiplock.counters (day,expiries) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET expiries = skiplock.counters.expiries + 1;
-            ELSIF (OLD.running = TRUE AND record.running = FALSE AND record.expired_at IS NULL AND record.finished_at IS NULL) THEN
-              INSERT INTO skiplock.counters (day,failures) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET failures = skiplock.counters.failures + 1;
             END IF;
           END IF;
-          PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.job_class,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM CASE WHEN record.scheduled_at IS NULL THEN record.updated_at ELSE record.scheduled_at END) AS FLOAT)::TEXT));
+          PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.namespace,',',record.job_class,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM CASE WHEN record.scheduled_at IS NULL THEN record.updated_at ELSE record.scheduled_at END) AS FLOAT)::TEXT));
           RETURN NULL;
         END;
       $$ LANGUAGE plpgsql
@@ -71,12 +77,12 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
         DECLARE
           record RECORD;
         BEGIN
-          IF (TG_OP = 'DELETE') THEN
+          IF TG_OP = 'DELETE' THEN
             record = OLD;
           ELSE
             record = NEW;
           END IF;
-          PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.hostname,',',record.master::TEXT,',',record.capacity,',',record.pid,',',record.sid,',',CAST(EXTRACT(EPOCH FROM record.created_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.updated_at) AS FLOAT)::TEXT));
+          PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.namespace,',',record.hostname,',',record.master::TEXT,',',record.capacity,',',record.pid,',',record.sid,',',CAST(EXTRACT(EPOCH FROM record.created_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.updated_at) AS FLOAT)::TEXT));
           RETURN NULL;
         END;
       $$ LANGUAGE plpgsql;
@@ -87,7 +93,7 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
     execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE running = FALSE AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
     execute "CREATE INDEX jobs_cron_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE cron IS NOT NULL AND finished_at IS NULL"
     execute "CREATE UNIQUE INDEX jobs_unique_cron_index ON skiplock.jobs (job_class) WHERE cron IS NOT NULL"
-    execute "CREATE UNIQUE INDEX workers_unique_master_index ON skiplock.workers(hostname) WHERE master = TRUE"
+    execute "CREATE UNIQUE INDEX workers_unique_master_index ON skiplock.workers(hostname,namespace) WHERE master = TRUE"
   end
 
   def down

data/lib/skiplock/cron.rb

@@ -19,6 +19,7 @@ module Skiplock
       query = Job.where('cron IS NOT NULL')
       query = query.where('job_class NOT IN (?)', cronjobs) if cronjobs.count > 0
       query.delete_all
+    rescue
     end
 
     def self.next_schedule_at(cron)

data/lib/skiplock/job.rb

@@ -3,14 +3,15 @@ module Skiplock
     self.implicit_order_column = 'updated_at'
     attribute :activejob_error
     attribute :exception
-    attribute :max_retries, :integer
-    attribute :purge, :boolean
+    attribute :max_retries
+    attribute :purge
     belongs_to :worker, inverse_of: :jobs, required: false
 
     def self.dispatch(purge_completion: true, max_retries: 20)
+      namespace_query = Skiplock.namespace.nil? ? "namespace IS NULL" : "namespace = '#{Skiplock.namespace}'"
       job = nil
       self.connection.transaction do
-        job = self.find_by_sql("SELECT id, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
+        job = self.find_by_sql("SELECT id, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL AND #{namespace_query} ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
         return if job.nil? || job.scheduled_at.to_f > Time.now.to_f
         job = self.find_by_sql("UPDATE skiplock.jobs SET running = TRUE, updated_at = NOW() WHERE id = '#{job.id}' RETURNING *").first
       end
@@ -33,7 +34,7 @@ module Skiplock
         Thread.current[:skiplock_job]
       else
         serialize = activejob.serialize
-        self.create!(serialize.slice(*self.column_names).merge('id' => serialize['job_id'], 'data' => { 'arguments' => serialize['arguments'], 'options' => options }, 'scheduled_at' => timestamp))
+        self.create!(serialize.slice(*self.column_names).merge('id' => serialize['job_id'], 'data' => { 'arguments' => serialize['arguments'], 'options' => options }, 'namespace' => Skiplock.namespace, 'scheduled_at' => timestamp))
       end
     end
 

data/lib/skiplock/manager.rb

@@ -10,7 +10,8 @@ module Skiplock
       elsif @config[:extensions].is_a?(Array)
         @config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
       end
-      async if (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0)
+      ActiveJob::Base.__send__(:include, Skiplock::Patch)
+      (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0) ? async : Cron.setup
     end
 
     def async
@@ -18,6 +19,7 @@ module Skiplock
       configure
       Worker.cleanup(@hostname)
       @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
+      Cron.setup if @worker.master
       @worker.start(**@config)
       at_exit { @worker.shutdown }
     rescue Exception => ex
@@ -42,7 +44,8 @@ module Skiplock
       ActiveRecord::Base.connection.disconnect! if @config[:workers] > 1
       (@config[:workers] - 1).times do |n|
         fork do
-          sleep 1
+          sleep(0.25*n + 1)
+          ActiveRecord::Base.establish_connection
           worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false)
           worker.start(worker_num: n + 1, **@config)
           loop do
@@ -73,24 +76,25 @@ module Skiplock
       @logger.info "-"*(title.length)
       @logger.info title
       @logger.info "-"*(title.length)
-      @logger.info "ClassMethod extensions: #{@config[:extensions]}"
-      @logger.info "      Purge completion: #{@config[:purge_completion]}"
-      @logger.info "          Notification: #{@config[:notification]}"
-      @logger.info "           Max retries: #{@config[:max_retries]}"
-      @logger.info "           Min threads: #{@config[:min_threads]}"
-      @logger.info "           Max threads: #{@config[:max_threads]}"
-      @logger.info "           Environment: #{Rails.env}"
-      @logger.info "              Loglevel: #{@config[:loglevel]}"
-      @logger.info "               Logfile: #{@config[:logfile] || '(disabled)'}"
-      @logger.info "               Workers: #{@config[:workers]}"
-      @logger.info "                Queues: #{@config[:queues].map {|k,v| k + '(' + v.to_s + ')'}.join(', ')}" if @config[:queues].is_a?(Hash)
-      @logger.info "                   PID: #{Process.pid}"
+      @logger.info "  ClassMethod extensions: #{@config[:extensions]}"
+      @logger.info "        Purge completion: #{@config[:purge_completion]}"
+      @logger.info "            Notification: #{@config[:notification]}"
+      @logger.info "             Max retries: #{@config[:max_retries]}"
+      @logger.info "             Min threads: #{@config[:min_threads]}"
+      @logger.info "             Max threads: #{@config[:max_threads]}"
+      @logger.info "             Environment: #{Rails.env}"
+      @logger.info "               Namespace: #{@config[:namespace] || '(nil)'}"
+      @logger.info "                Loglevel: #{@config[:loglevel]}"
+      @logger.info "                 Logfile: #{@config[:logfile] || '(disabled)'}"
+      @logger.info "                 Workers: #{@config[:workers]}"
+      @logger.info "                  Queues: #{@config[:queues].map {|k,v| k + '(' + v.to_s + ')'}.join(', ')}" if @config[:queues].is_a?(Hash)
+      @logger.info "                     PID: #{Process.pid}"
       @logger.info "-"*(title.length)
       @logger.warn "[Skiplock] Custom notification has no registered 'on_error' callback" if Skiplock.on_errors.count == 0
     end
 
     def configure
-      @hostname = Socket.gethostname
+      @hostname = "#{`hostname -f`.strip}|#{Socket.ip_address_list.reject(&:ipv4_loopback?).reject(&:ipv6?).map(&:ip_address).join('|')}"
       @config.transform_values! {|v| v.is_a?(String) ? v.downcase : v}
       @config[:graceful_shutdown] = 300 if @config[:graceful_shutdown] > 300
       @config[:graceful_shutdown] = nil if @config[:graceful_shutdown] < 0
@@ -131,6 +135,7 @@ module Skiplock
       else
         @config[:notification] = 'custom'
       end
+      Skiplock.namespace = @config[:namespace]
       Skiplock.on_errors.freeze
     end
 
@@ -145,6 +150,7 @@ module Skiplock
       end
       if @config[:standalone]
         Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
+        Rails.logger.level = @logger.level
         Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
       end
     rescue Exception => ex
@@ -153,4 +159,4 @@ module Skiplock
       Skiplock.on_errors.each { |p| p.call(ex) }
     end
   end
-end
+end

data/lib/skiplock/version.rb

@@ -1,4 +1,4 @@
 module Skiplock
-  VERSION = Version = '1.0.22'
+  VERSION = Version = '1.1.0'
 end
 

data/lib/skiplock/worker.rb

@@ -5,7 +5,7 @@ module Skiplock
 
     def self.cleanup(hostname = nil)
       delete_ids = []
-      self.where(hostname: hostname || Socket.gethostname).each do |worker|
+      self.where(namespace: Skiplock.namespace, hostname: hostname || Socket.gethostname).each do |worker|
         sid = Process.getsid(worker.pid) rescue nil
         delete_ids << worker.id if worker.sid != sid || worker.updated_at < 10.minutes.ago
       end
@@ -13,115 +13,119 @@ module Skiplock
     end
 
     def self.generate(capacity:, hostname:, master: true)
-      self.create!(pid: Process.pid, sid: Process.getsid(), master: master, hostname: hostname, capacity: capacity)
+      worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: master, hostname: hostname, capacity: capacity, namespace: Skiplock.namespace)
     rescue
-      self.create!(pid: Process.pid, sid: Process.getsid(), master: false, hostname: hostname, capacity: capacity)
+      worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: false, hostname: hostname, capacity: capacity, namespace: Skiplock.namespace)
     end
 
     def shutdown
       @running = false
       @executor.shutdown
       @executor.kill unless @executor.wait_for_termination(@config[:graceful_shutdown])
-      self.delete
       Skiplock.logger.info "[Skiplock] Shutdown of #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} (PID: #{self.pid}) was completed."
     end
 
     def start(worker_num: 0, **config)
       @num = worker_num
       @config = config
+      @pg_config = ActiveRecord::Base.connection.raw_connection.conninfo_hash.compact
+      @namespace_query = Skiplock.namespace.nil? ? "namespace IS NULL" : "namespace = '#{Skiplock.namespace}'"
       @queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
       @running = true
-      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: true, fallback_policy: :discard)
+      @map = ::PG::TypeMapByOid.new
+      @map.add_coder(::PG::TextDecoder::Boolean.new(oid: 16, name: 'bool'))
+      @map.add_coder(::PG::TextDecoder::Integer.new(oid: 20, name: 'int8'))
+      @map.add_coder(::PG::TextDecoder::Integer.new(oid: 21, name: 'int2'))
+      @map.add_coder(::PG::TextDecoder::Integer.new(oid: 23, name: 'int4'))
+      @map.add_coder(::PG::TextDecoder::TimestampUtc.new(oid: 1114, name: 'timestamp'))
+      @map.add_coder(::PG::TextDecoder::String.new(oid: 2950, name: 'uuid'))
+      @map.add_coder(::PG::TextDecoder::JSON.new(oid: 3802, name: 'jsonb'))
+      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: false, fallback_policy: :abort)
       @executor.post { run }
-      Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]") if @config[:standalone]
+      if @config[:standalone]
+        Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]")
+        ActiveRecord::Base.connection.throw_away!
+      end
     end
 
     private
 
+    def establish_connection
+      @connection = ::PG.connect(@pg_config)
+      @connection.type_map_for_results = @map
+      @connection.exec('LISTEN "skiplock::jobs"').clear
+      @connection.exec('LISTEN "skiplock::workers"').clear
+    end
+
     def run
       sleep 3
       Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
-      connection = nil
-      error = false
-      listen = false
       next_schedule_at = Time.now.to_f
       pg_exception_timestamp = nil
       timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       while @running
         Rails.application.reloader.wrap do
           begin
-            unless listen
-              connection = self.class.connection
-              connection.exec_query('LISTEN "skiplock::jobs"')
-              if self.master
-                Job.flush
-                Cron.setup
-              end
-              listen = true
-            end
-            if error
-              unless connection.active?
-                connection.reconnect!
-                sleep(0.5)
-                connection.exec_query('LISTEN "skiplock::jobs"')
-                Job.flush if self.master
-                pg_exception_timestamp = nil
-                next_schedule_at = Time.now.to_f
-              end
-              error = false
+            if @connection.nil? || @connection.status != ::PG::CONNECTION_OK
+              establish_connection
+              @executor.post { Rails.application.executor.wrap { Job.flush } } if self.master
+              pg_exception_timestamp = nil
+              next_schedule_at = Time.now.to_f
             end
             if Time.now.to_f >= next_schedule_at && @executor.remaining_capacity > 1  # reserves 1 slot in queue for Job.flush in case of pg_connection error
               result = nil
-              connection.transaction do
-                result = connection.select_all("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
-                result = connection.select_all("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *").first if result && result['scheduled_at'].to_f <= Time.now.to_f
+              @connection.transaction do |conn|
+                conn.exec("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL AND #{@namespace_query} ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1") do |r|
+                  result = r.first
+                  conn.exec("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *") { |r| result = r.first } if result && result['scheduled_at'].to_f <= Time.now.to_f
+                end
               end
               if result && result['running']
-                @executor.post do
-                  Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
-                end
+                @executor.post { Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) } }
               else
                 next_schedule_at = (result ? result['scheduled_at'].to_f : Float::INFINITY)
               end
             end
-            job_notifications = []
-            connection.raw_connection.wait_for_notify(0.2) do |channel, pid, payload|
-              job_notifications << payload if payload
+            notifications = { 'skiplock::jobs' => [], 'skiplock::workers' => [] }
+            @connection.wait_for_notify(0.2) do |channel, pid, payload|
+              notifications[channel] << payload if payload
               loop do
-                payload = connection.raw_connection.notifies
+                payload = @connection.notifies
                 break unless @running && payload
-                job_notifications << payload[:extra]
+                notifications[payload[:relname]] << payload[:extra]
               end
-              job_notifications.each do |n|
-                op, id, worker_id, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
-                next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0
+              notifications['skiplock::jobs'].each do |n|
+                op, id, worker_id, namespace, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
+                next if op == 'DELETE' || running == 'true' || namespace != Skiplock.namespace || expired_at.to_f > 0 || finished_at.to_f > 0
                 next_schedule_at = scheduled_at.to_f if scheduled_at.to_f < next_schedule_at
               end
             end
             if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
-              connection.exec_query("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'")
+              @connection.exec("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'").clear
               timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
             end
           rescue Exception => ex
-            Skiplock.logger.error(ex.to_s)
-            Skiplock.logger.error(ex.backtrace.join("\n"))
             report_exception = true
             # if error is with database connection then only report if it persists longer than 1 minute
-            if ex.is_a?(::PG::ConnectionBad) ||ex.is_a?(::PG::UnableToSend) || ex.message.include?('Bad file descriptor')
+            if @connection.nil? || @connection.status != ::PG::CONNECTION_OK
               report_exception = false if pg_exception_timestamp.nil? || Process.clock_gettime(Process::CLOCK_MONOTONIC) - pg_exception_timestamp <= 60
               pg_exception_timestamp ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
             end
-            Skiplock.on_errors.each { |p| p.call(ex) } if report_exception
-            error = true
+            if report_exception
+              Skiplock.logger.error(ex.to_s)
+              Skiplock.logger.error(ex.backtrace.join("\n"))
+              Skiplock.on_errors.each { |p| p.call(ex) }
+            end
             wait(5)
           end
           sleep(0.3)
         end
       end
-      connection.exec_query('UNLISTEN *')
+    ensure
+      @connection.close if @connection && !@connection.finished?
     end
 
-    def wait(timeout)
+    def wait(timeout = 1)
       t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       while @running
         sleep(0.5)
@@ -129,4 +133,4 @@ module Skiplock
       end
     end
   end
-end
+end

data/lib/skiplock.rb

@@ -11,7 +11,7 @@ require 'skiplock/worker'
 require 'skiplock/version'
 
 module Skiplock
-  DEFAULT_CONFIG = { 'extensions' => false, 'logfile' => 'skiplock.log', 'loglevel' => 'info', 'graceful_shutdown' => 15, 'min_threads' => 1, 'max_threads' => 10, 'max_retries' => 20, 'notification' => 'custom', 'purge_completion' => true, 'queues' => { 'default' => 100, 'mailers' => 999 }, 'workers' => 0 }.freeze
+  DEFAULT_CONFIG = { 'graceful_shutdown' => 15, 'min_threads' => 1, 'max_threads' => 10, 'max_retries' => 20, 'logfile' => 'skiplock.log', 'loglevel' => 'info', 'namespace' => nil, 'notification' => 'custom', 'extensions' => false, 'purge_completion' => true, 'queues' => { 'default' => 100, 'mailers' => 999 }, 'workers' => 0 }.freeze
 
   def self.logger=(l)
     @logger = l
@@ -21,6 +21,14 @@ module Skiplock
     @logger
   end
 
+  def self.namespace=(n)
+    @namespace = n
+  end
+
+  def self.namespace
+    @namespace || ''
+  end
+
   def self.on_error(&block)
     @on_errors ||= []
     @on_errors << block
@@ -34,5 +42,4 @@ module Skiplock
   def self.table_name_prefix
     'skiplock.'
   end
-end
-ActiveJob::Base.__send__(:include, Skiplock::Patch)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skiplock
 version: !ruby/object:Gem::Version
-  version: 1.0.22
+  version: 1.1.0
 platform: ruby
 authors:
 - Tin Vo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-18 00:00:00.000000000 Z
+date: 2022-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob