skiplock 1.0.20 → 1.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 982ed70ce969943c421cad0923a280c87c4c26ea90da5a3451ce73becd5db42e
-  data.tar.gz: 9f68becb9ab21e00816315d1e2ff20058d2f5942a3688ed78959018e3ccf940f
+  metadata.gz: a43b01cedd94ea395b0b173e2da3c5c7ab0baf6ccbd35e599127544084ca5589
+  data.tar.gz: a908c4eea5b2b75727a70fe5efc248f0b0183e9fe97a37c068b185b9f259a4ba
 SHA512:
-  metadata.gz: 0fb583418cf8c0a190f9b651ecd95effba2d4b415f91bf6fbd452d2271f02534e7c5823e360f1e7a258cdd0a22b14cc99d44915007e9c32a186371748c086144
-  data.tar.gz: 6aa546ad0a538213ae9587e667543027fb2ca080e7623b3f409fe6303c005085f39fd27c310a690a2da9fbb50321d97519ed9aee7bb4922a126c0a280f6ad5fe
+  metadata.gz: c5ae8b9bd79e37426710d707fadd2e4d658e541ffdd109a760687f2c14dacfa9d61614130d25be801264ecee86ea63d54ce4d6604c4cc80c7a47ac31752618f6
+  data.tar.gz: 42ba6d9047ac028c24389d1915f686f5d553751244bd557a98a8b28af311a53702d6c69078abafcb6696d5d00e441fd8eca43d2e18b11530945fa10a42d15cea

data/README.md

@@ -50,12 +50,14 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     ```yaml
     # config/skiplock.yml (default settings)
     ---
+    graceful_shutdown: 15
     min_threads: 1
     max_threads: 10
     max_retries: 20
     logfile: skiplock.log
     loglevel: info
     notification: custom
+    actioncable: false
     extensions: false
     purge_completion: true
     queues:
@@ -64,12 +66,14 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     workers: 0
     ```
     Available configuration options are:
+    - **graceful_shutdown** (*integer*): sets the number of seconds to wait for jobs to finish before being killed during shutdown
     - **min_threads** (*integer*): sets minimum number of threads staying idle
     - **max_threads** (*integer*): sets the maximum number of threads allowed to run jobs
     - **max_retries** (*integer*): sets the maximum attempt a job will be retrying before it is marked expired.  See `Retry system` for more details
     - **logfile** (*string*): filename for skiplock logs; empty logfile will disable logging
     - **loglevel** (*string*): sets logging level (`debug, info, warn, error, fatal, unknown`)
     - **notification** (*string*): sets the library to be used for notifying errors and exceptions (`auto, airbrake, bugsnag, exception_notification, custom`); using `auto` will detect library if available.  See `Notification system` for more details
+    - **actioncable** (*boolean*): enable or disable usage of ActionCable notification
     - **extensions** (*multi*): enable or disable the class method extension.  See `ClassMethod extension` for more details
     - **purge_completion** (*boolean*): when set to **true** will delete jobs after they were completed successfully; if set to **false** then the completed jobs should be purged periodically to maximize performance (eg. clean up old jobs after 3 months); queued jobs can manually override using `purge` option
     - **queues** (*hash*): defines the set of queues with priorities; lower priority takes precedence
@@ -83,8 +87,10 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     ```
     $ bundle exec skiplock -h
     Usage: skiplock [options]
+      -a, --actioncable YESNO          Actioncable notification
       -e, --environment STRING         Rails environment
       -l, --logfile STRING             Log filename
+      -L, --loglevel STRING            Log level (debug, info, warn, error, fatal, unknown)
       -s, --graceful-shutdown NUM      Number of seconds to wait for graceful shutdown
       -r, --max-retries NUM            Number of maxixum retries
       -t, --max-threads NUM            Number of maximum threads
@@ -172,14 +178,10 @@ If the `retry_on` block is not defined, then the built-in retry system of `Skipl
 `Skiplock` can use existing exception notification library to notify errors and exceptions.  It supports `airbrake`, `bugsnag`, and `exception_notification`.  Custom notification can also be called whenever an exception occurs; it can be configured in an initializer like below:
 ```ruby
   # config/initializers/skiplock.rb
-  Skiplock.on_error do |ex, previous|
-    if ex.backtrace != previous.try(:backtrace)
-      # sends text message using Amazon SNS on new exceptions only
-      # the same repeated exceptions will only be sent once to avoid SPAM
-      # NOTE: exceptions generated from Job executions will not provide 'previous' exceptions
-      sms = Aws::SNS::Client.new(region: 'us-west-2', access_key_id: Rails.application.credentials[:aws][:access_key_id], secret_access_key: Rails.application.credentials[:aws][:secret_access_key])
-      sms.publish({ phone_number: '+122233334444', message: "Exception: #{ex.message}"[0..130] })
-    end
+  Skiplock.on_error do |ex|
+    # sends text message using Amazon SNS
+    sms = Aws::SNS::Client.new(region: 'us-west-2', access_key_id: Rails.application.credentials[:aws][:access_key_id], secret_access_key: Rails.application.credentials[:aws][:secret_access_key])
+    sms.publish(phone_number: '+12223334444', message: "Exception: #{ex.message}"[0..130])
   end
   # supports multiple 'on_error' event callbacks
 ```
@@ -241,6 +243,10 @@ Code examples of gathering counters information:
     ```ruby
     Skiplock::Counter.sum(:expiries)
     ```
+  - get all information in one query
+    ```ruby
+    Skiplock::Counter.pluck("sum(completions), sum(dispatches), sum(expiries), sum(failures), sum(retries)").first
+    ```
 
 ## Contributing
 

data/bin/skiplock

@@ -5,6 +5,7 @@ options = {}
 begin
   op = OptionParser.new do |opts|
     opts.banner = "Usage: #{File.basename($0)} [options]"
+    opts.on('-a', '--actioncable YESNO', TrueClass, 'Actioncable notification')
     opts.on('-e', '--environment STRING', String, 'Rails environment')
     opts.on('-l', '--logfile STRING', String, 'Log filename')
     opts.on('-L', '--loglevel STRING', String, 'Log level (debug, info, warn, error, fatal, unknown)')

data/lib/skiplock/cron.rb

@@ -19,6 +19,7 @@ module Skiplock
       query = Job.where('cron IS NOT NULL')
       query = query.where('job_class NOT IN (?)', cronjobs) if cronjobs.count > 0
       query.delete_all
+    rescue
     end
 
     def self.next_schedule_at(cron)

data/lib/skiplock/job.rb

@@ -1,7 +1,10 @@
 module Skiplock
   class Job < ActiveRecord::Base
     self.implicit_order_column = 'updated_at'
-    attr_accessor :activejob_error
+    attribute :activejob_error
+    attribute :exception
+    attribute :max_retries, :integer
+    attribute :purge, :boolean
     belongs_to :worker, inverse_of: :jobs, required: false
 
     def self.dispatch(purge_completion: true, max_retries: 20)
@@ -40,7 +43,7 @@ module Skiplock
       Dir.glob('tmp/skiplock/*').each do |f|
         disposed = true
         if self.exists?(id: File.basename(f), running: true)
-          job = Marshal.load(File.binread(f)) rescue nil
+          job = YAML.load_file(f) rescue nil
           disposed = job.dispose if job.is_a?(Skiplock::Job)
         end
         (File.delete(f) rescue nil) if disposed
@@ -54,15 +57,15 @@ module Skiplock
     end
 
     def dispose
-      return unless @max_retries
-      dump = Marshal.dump(self)
+      return unless self.max_retries
+      yaml = self.to_yaml
       purging = false
       self.running = false
       self.worker_id = nil
       self.updated_at = Time.now > self.updated_at ? Time.now : self.updated_at + 1 # in case of clock drifting
-      if @exception
-        self.exception_executions["[#{@exception.class.name}]"] = self.exception_executions["[#{@exception.class.name}]"].to_i + 1 unless self.data.key?('activejob_error')
-        if (self.executions.to_i >= @max_retries + 1) || self.data.key?('activejob_error') || @exception.is_a?(Skiplock::Extension::ProxyError)
+      if self.exception
+        self.exception_executions["[#{self.exception.class.name}]"] = self.exception_executions["[#{self.exception.class.name}]"].to_i + 1 unless self.data.key?('activejob_error')
+        if (self.executions.to_i >= self.max_retries + 1) || self.data.key?('activejob_error') || self.exception.is_a?(Skiplock::Extension::ProxyError)
           self.expired_at = Time.now
         else
           self.scheduled_at = Time.now + (5 * 2**self.executions.to_i)
@@ -76,7 +79,7 @@ module Skiplock
           next_cron_at = Cron.next_schedule_at(self.cron)
           if next_cron_at
             # update job to record completions counter before resetting finished_at to nil
-            self.update_columns(self.attributes.slice(*self.changes.keys))
+            self.update_columns(self.attributes.slice(*(self.changes.keys & self.class.column_names)))
             self.finished_at = nil
             self.executions = nil
             self.exception_executions = nil
@@ -86,13 +89,13 @@ module Skiplock
             Skiplock.logger.error("[Skiplock] ERROR: Invalid CRON '#{self.cron}' for Job #{self.job_class}") if Skiplock.logger
             purging = true
           end
-        elsif @purge == true
+        elsif self.purge == true
           purging = true
         end
       end
-      purging ? self.delete : self.update_columns(self.attributes.slice(*self.changes.keys))
+      purging ? self.delete : self.update_columns(self.attributes.slice(*(self.changes.keys & self.class.column_names)))
     rescue Exception => e
-      File.binwrite("tmp/skiplock/#{self.id}", dump) rescue nil
+      File.write("tmp/skiplock/#{self.id}", yaml) rescue nil
       if Skiplock.logger
         Skiplock.logger.error(e.to_s)
         Skiplock.logger.error(e.backtrace.join("\n"))
@@ -109,9 +112,9 @@ module Skiplock
       self.data.delete('result')
       self.exception_executions ||= {}
       self.activejob_error = nil
-      @max_retries = (self.data['options'].key?('max_retries') ? self.data['options']['max_retries'].to_i : max_retries) rescue max_retries
-      @max_retries = 20 if @max_retries < 0 || @max_retries > 20
-      @purge = (self.data['options'].key?('purge') ? self.data['options']['purge'] : purge_completion) rescue purge_completion
+      self.max_retries = (self.data['options'].key?('max_retries') ? self.data['options']['max_retries'].to_i : max_retries) rescue max_retries
+      self.max_retries = 20 if self.max_retries < 0 || self.max_retries > 20
+      self.purge = (self.data['options'].key?('purge') ? self.data['options']['purge'] : purge_completion) rescue purge_completion
       job_data = self.attributes.slice('job_class', 'queue_name', 'locale', 'timezone', 'priority', 'executions', 'exception_executions').merge('job_id' => self.id, 'enqueued_at' => self.updated_at, 'arguments' => (self.data['arguments'] || []))
       self.executions = self.executions.to_i + 1
       Thread.current[:skiplock_job] = self
@@ -119,15 +122,15 @@ module Skiplock
       begin
         self.data['result'] = ActiveJob::Base.execute(job_data)
       rescue Exception => ex
-        @exception = ex
-        Skiplock.on_errors.each { |p| p.call(@exception) }
+        self.exception = ex
+        Skiplock.on_errors.each { |p| p.call(ex) }
       end
       if Skiplock.logger
-        if @exception || self.activejob_error
+        if self.exception || self.activejob_error
           Skiplock.logger.error("[Skiplock] Job #{self.job_class} (#{self.id}) was interrupted by an exception#{ ' (rescued and retried by ActiveJob)' if self.activejob_error }")
-          if @exception
-            Skiplock.logger.error(@exception.to_s)
-            Skiplock.logger.error(@exception.backtrace.join("\n"))
+          if self.exception
+            Skiplock.logger.error(self.exception.to_s)
+            Skiplock.logger.error(self.exception.backtrace.join("\n"))
           end
         else
           end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -139,7 +142,7 @@ module Skiplock
           Skiplock.logger.info "[Skiplock] Performed #{job_name} (#{self.id}) from queue '#{self.queue_name || 'default'}' in #{end_time - start_time} seconds"
         end
       end
-      @exception || self.activejob_error || self.data['result']
+      self.exception || self.activejob_error || self.data['result']
     ensure
       self.finished_at ||= Time.now if self.data.key?('result') && !self.activejob_error
       self.dispose

data/lib/skiplock/manager.rb

@@ -4,14 +4,22 @@ module Skiplock
       @config = Skiplock::DEFAULT_CONFIG.dup
       @config.merge!(YAML.load_file('config/skiplock.yml')) rescue nil
       @config.symbolize_keys!
-      async if (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0)
+      Rails.application.eager_load! if Rails.env.development?
+      if @config[:extensions] == true
+        Module.__send__(:include, Skiplock::Extension)
+      elsif @config[:extensions].is_a?(Array)
+        @config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
+      end
+      ActiveJob::Base.__send__(:include, Skiplock::Patch)
+      (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0) ? async : Cron.setup
     end
 
     def async
-      configure
       setup_logger
+      configure
       Worker.cleanup(@hostname)
-      @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
+      @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, actioncable: @config[:actioncable])
+      Cron.setup if @worker.master
       @worker.start(**@config)
       at_exit { @worker.shutdown }
     rescue Exception => ex
@@ -21,25 +29,24 @@ module Skiplock
 
     def standalone(**options)
       @config.merge!(options)
-      configure
-      setup_logger
-      Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
-      Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
-      @config[:workers] = 1 if @config[:workers] <= 0
       @config[:standalone] = true
+      @config[:workers] = 1 if @config[:workers] <= 0
+      setup_logger
+      configure
       banner
-      Worker.cleanup(@hostname)
-      @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
       @parent_id = Process.pid
       @shutdown = false
       Signal.trap('INT') { @shutdown = true }
       Signal.trap('TERM') { @shutdown = true }
       Signal.trap('HUP') { setup_logger }
+      Worker.cleanup(@hostname)
+      @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, actioncable: @config[:actioncable])
+      ActiveRecord::Base.connection.disconnect! if @config[:workers] > 1
       (@config[:workers] - 1).times do |n|
-        sleep 0.2
         fork do
-          sleep 1
-          worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false)
+          sleep(0.25*n + 1)
+          ActiveRecord::Base.establish_connection
+          worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false, actioncable: @config[:actioncable])
           worker.start(worker_num: n + 1, **@config)
           loop do
             sleep 0.5
@@ -48,6 +55,7 @@ module Skiplock
           worker.shutdown
         end
       end
+      ActiveRecord::Base.establish_connection if @config[:workers] > 1
       @worker.start(**@config)
       loop do
         sleep 0.5
@@ -68,24 +76,25 @@ module Skiplock
       @logger.info "-"*(title.length)
       @logger.info title
       @logger.info "-"*(title.length)
-      @logger.info "ClassMethod extensions: #{@config[:extensions]}"
-      @logger.info "      Purge completion: #{@config[:purge_completion]}"
-      @logger.info "          Notification: #{@config[:notification]}"
-      @logger.info "           Max retries: #{@config[:max_retries]}"
-      @logger.info "           Min threads: #{@config[:min_threads]}"
-      @logger.info "           Max threads: #{@config[:max_threads]}"
-      @logger.info "           Environment: #{Rails.env}"
-      @logger.info "              Loglevel: #{@config[:loglevel]}"
-      @logger.info "               Logfile: #{@config[:logfile] || '(disabled)'}"
-      @logger.info "               Workers: #{@config[:workers]}"
-      @logger.info "                Queues: #{@config[:queues].map {|k,v| k + '(' + v.to_s + ')'}.join(', ')}" if @config[:queues].is_a?(Hash)
-      @logger.info "                   PID: #{Process.pid}"
+      @logger.info "ActionCable notification: #{@config[:actioncable]}#{' (not available)' unless defined?(ActionCable)}"
+      @logger.info "  ClassMethod extensions: #{@config[:extensions]}"
+      @logger.info "        Purge completion: #{@config[:purge_completion]}"
+      @logger.info "            Notification: #{@config[:notification]}"
+      @logger.info "             Max retries: #{@config[:max_retries]}"
+      @logger.info "             Min threads: #{@config[:min_threads]}"
+      @logger.info "             Max threads: #{@config[:max_threads]}"
+      @logger.info "             Environment: #{Rails.env}"
+      @logger.info "                Loglevel: #{@config[:loglevel]}"
+      @logger.info "                 Logfile: #{@config[:logfile] || '(disabled)'}"
+      @logger.info "                 Workers: #{@config[:workers]}"
+      @logger.info "                  Queues: #{@config[:queues].map {|k,v| k + '(' + v.to_s + ')'}.join(', ')}" if @config[:queues].is_a?(Hash)
+      @logger.info "                     PID: #{Process.pid}"
       @logger.info "-"*(title.length)
       @logger.warn "[Skiplock] Custom notification has no registered 'on_error' callback" if Skiplock.on_errors.count == 0
     end
 
     def configure
-      @hostname = Socket.gethostname
+      @hostname = "#{`hostname -f`.strip}|#{Socket.ip_address_list.reject(&:ipv4_loopback?).reject(&:ipv6?).map(&:ip_address).join('|')}"
       @config.transform_values! {|v| v.is_a?(String) ? v.downcase : v}
       @config[:graceful_shutdown] = 300 if @config[:graceful_shutdown] > 300
       @config[:graceful_shutdown] = nil if @config[:graceful_shutdown] < 0
@@ -107,31 +116,26 @@ module Skiplock
           raise "Unable to detect any known exception notification library. Please define custom 'on_error' event callbacks and change to 'custom' notification in 'config/skiplock.yml'"
         end
       end
-      Rails.application.eager_load! if Rails.env.development?
-      if @config[:extensions] == true
-        Module.__send__(:include, Skiplock::Extension)
-      elsif @config[:extensions].is_a?(Array)
-        @config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
-      end
       case @config[:notification]
       when 'airbrake'
         raise 'airbrake gem not found' unless defined?(Airbrake)
-        Skiplock.on_error do |ex, previous|
-          Airbrake.notify_sync(ex) unless ex.backtrace == previous.try(:backtrace)
+        Skiplock.on_error do |ex|
+          Airbrake.notify_sync(ex)
         end
       when 'bugsnag'
         raise 'bugsnag gem not found' unless defined?(Bugsnag)
-        Skiplock.on_error do |ex, previous|
-          Bugsnag.notify(ex) unless ex.backtrace == previous.try(:backtrace)
+        Skiplock.on_error do |ex|
+          Bugsnag.notify(ex)
         end
       when 'exception_notification'
         raise 'exception_notification gem not found' unless defined?(ExceptionNotifier)
-        Skiplock.on_error do |ex, previous|
-          ExceptionNotifier.notify_exception(ex) unless ex.backtrace == previous.try(:backtrace)
+        Skiplock.on_error do |ex|
+          ExceptionNotifier.notify_exception(ex)
         end
       else
         @config[:notification] = 'custom'
       end
+      @logger.error 'ActionCable is not found!' if @config[:actioncable] && !defined?(ActionCable)
       Skiplock.on_errors.freeze
     end
 
@@ -144,6 +148,11 @@ module Skiplock
         @logger.extend(ActiveSupport::Logger.broadcast(::Logger.new(File.join(Rails.root, 'log', @config[:logfile].to_s), 'daily')))
         ActiveJob::Base.logger = nil
       end
+      if @config[:standalone]
+        Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
+        Rails.logger.level = @logger.level
+        Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
+      end
     rescue Exception => ex
       @logger.error "Exception with logger: #{ex.to_s}"
       @logger.error ex.backtrace.join("\n")

data/lib/skiplock/version.rb

@@ -1,4 +1,4 @@
 module Skiplock
-  VERSION = Version = '1.0.20'
+  VERSION = Version = '1.0.24'
 end
 

data/lib/skiplock/worker.rb

@@ -12,107 +12,130 @@ module Skiplock
       self.where(id: delete_ids).delete_all if delete_ids.count > 0
     end
 
-    def self.generate(capacity:, hostname:, master: true)
-      self.create!(pid: Process.pid, sid: Process.getsid(), master: master, hostname: hostname, capacity: capacity)
+    def self.generate(capacity:, hostname:, master: true, actioncable: false)
+      worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: master, hostname: hostname, capacity: capacity)
     rescue
-      self.create!(pid: Process.pid, sid: Process.getsid(), master: false, hostname: hostname, capacity: capacity)
-    end
-
-    def start(worker_num: 0, **config)
-      if self.master
-        Job.flush
-        Cron.setup
-      end
-      @num = worker_num
-      @config = config
-      @queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
-      @running = true
-      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads], idletime: 60, auto_terminate: true, fallback_policy: :discard)
-      @executor.post { run }
-      Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]") if @config[:standalone]
+      worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: false, hostname: hostname, capacity: capacity)
+    ensure
+      ActionCable.server.broadcast('skiplock', { worker: { op: 'CREATE', id: worker.id, hostname: worker.hostname, master: worker.master, capacity: worker.capacity, pid: worker.pid, sid: worker.sid, created_at: worker.created_at.to_f, updated_at: worker.updated_at.to_f } }) if actioncable && defined?(ActionCable)
     end
 
     def shutdown
       @running = false
       @executor.shutdown
       @executor.kill unless @executor.wait_for_termination(@config[:graceful_shutdown])
+      ActionCable.server.broadcast('skiplock', { worker: { op: 'DELETE', id: self.id, hostname: self.hostname, master: self.master, capacity: self.capacity, pid: self.pid, sid: self.sid, created_at: self.created_at.to_f, updated_at: self.updated_at.to_f } }) if @config[:actioncable] && defined?(ActionCable)
       self.delete
       Skiplock.logger.info "[Skiplock] Shutdown of #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} (PID: #{self.pid}) was completed."
     end
 
+    def start(worker_num: 0, **config)
+      @num = worker_num
+      @config = config
+      @pg_config = ActiveRecord::Base.connection.raw_connection.conninfo_hash.compact
+      @queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
+      @running = true
+      @map = ::PG::TypeMapByOid.new
+      @map.add_coder(::PG::TextDecoder::Boolean.new(oid: 16, name: 'bool'))
+      @map.add_coder(::PG::TextDecoder::Integer.new(oid: 20, name: 'int8'))
+      @map.add_coder(::PG::TextDecoder::Integer.new(oid: 21, name: 'int2'))
+      @map.add_coder(::PG::TextDecoder::Integer.new(oid: 23, name: 'int4'))
+      @map.add_coder(::PG::TextDecoder::TimestampUtc.new(oid: 1114, name: 'timestamp'))
+      @map.add_coder(::PG::TextDecoder::String.new(oid: 2950, name: 'uuid'))
+      @map.add_coder(::PG::TextDecoder::JSON.new(oid: 3802, name: 'jsonb'))
+      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: false, fallback_policy: :abort)
+      @executor.post { run }
+      if @config[:standalone]
+        Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]")
+        ActiveRecord::Base.connection.throw_away!
+      end
+    end
+
     private
 
+    def establish_connection
+      @connection = ::PG.connect(@pg_config)
+      @connection.type_map_for_results = @map
+      @connection.exec('LISTEN "skiplock::jobs"').clear
+      @connection.exec('LISTEN "skiplock::workers"').clear
+    end
+
     def run
       sleep 3
-      ActiveRecord::Base.connection_pool.with_connection do |connection|
-        Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
-        connection.exec_query('LISTEN "skiplock::jobs"')
-        error = false
-        next_schedule_at = Time.now.to_f
-        timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        while @running
-          Rails.application.reloader.wrap do
-            begin
-              if error
-                unless connection.active?
-                  connection.reconnect!
-                  sleep(0.5)
-                  connection.exec_query('LISTEN "skiplock::jobs"')
-                  next_schedule_at = Time.now.to_f
+      Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
+      next_schedule_at = Time.now.to_f
+      pg_exception_timestamp = nil
+      timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      while @running
+        Rails.application.reloader.wrap do
+          begin
+            if @connection.nil? || @connection.status != ::PG::CONNECTION_OK
+              establish_connection
+              @executor.post { Rails.application.executor.wrap { Job.flush } } if self.master
+              pg_exception_timestamp = nil
+              next_schedule_at = Time.now.to_f
+            end
+            if Time.now.to_f >= next_schedule_at && @executor.remaining_capacity > 1  # reserves 1 slot in queue for Job.flush in case of pg_connection error
+              result = nil
+              @connection.transaction do |conn|
+                conn.exec("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1") do |r|
+                  result = r.first
+                  conn.exec("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *") { |r| result = r.first } if result && result['scheduled_at'].to_f <= Time.now.to_f
                 end
-                Job.flush if self.master
-                error = false
               end
-              if Time.now.to_f >= next_schedule_at && @executor.remaining_capacity > 0
-                job = nil
-                connection.transaction do
-                  result = connection.select_all("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
-                  result = connection.select_all("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *").first if result && result['scheduled_at'].to_f <= Time.now.to_f
-                  job = Job.instantiate(result) if result
-                end
-                if job.try(:running)
-                  @executor.post do
-                    Rails.application.executor.wrap { job.execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
-                  end
-                else
-                  next_schedule_at = (job ? job.scheduled_at.to_f : Float::INFINITY)
-                end
+              if result && result['running']
+                @executor.post { Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) } }
+              else
+                next_schedule_at = (result ? result['scheduled_at'].to_f : Float::INFINITY)
               end
-              job_notifications = []
-              connection.raw_connection.wait_for_notify(0.2) do |channel, pid, payload|
-                job_notifications << payload if payload
-                loop do
-                  payload = connection.raw_connection.notifies
-                  break unless @running && payload
-                  job_notifications << payload[:extra]
-                end
-                job_notifications.each do |n|
-                  op, id, worker_id, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
-                  next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0
-                  next_schedule_at = scheduled_at.to_f if scheduled_at.to_f < next_schedule_at
-                end
+            end
+            notifications = { 'skiplock::jobs' => [], 'skiplock::workers' => [] }
+            @connection.wait_for_notify(0.2) do |channel, pid, payload|
+              notifications[channel] << payload if payload
+              loop do
+                payload = @connection.notifies
+                break unless @running && payload
+                notifications[payload[:relname]] << payload[:extra]
+              end
+              notifications['skiplock::jobs'].each do |n|
+                op, id, worker_id, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
+                ActionCable.server.broadcast('skiplock', { job: { op: op, id: id, worker_id: worker_id, job_class: job_class, queue_name: queue_name, running: (running == 'true'), expired_at: expired_at.to_f, finished_at: finished_at.to_f, scheduled_at: scheduled_at.to_f } }) if self.master && @config[:actioncable] && defined?(ActionCable)
+                next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0
+                next_schedule_at = scheduled_at.to_f if scheduled_at.to_f < next_schedule_at
               end
-              if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
-                self.touch
-                timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              if self.master && @config[:actioncable] && defined?(ActionCable)
+                notifications['skiplock::workers'].each do |w|
+                  op, id, hostname, master, capacity, pid, sid, created_at, updated_at = w.split(',')
+                  ActionCable.server.broadcast('skiplock', { worker: { op: op, id: id, hostname: hostname, master: (master == 'true'), capacity: capacity.to_i, pid: pid.to_i, sid: sid.to_i, created_at: created_at.to_f, updated_at: updated_at.to_f } })
+                end
               end
-            rescue Exception => ex
-              # most likely error with database connection
+            end
+            if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
+              @connection.exec("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'").clear
+              timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            end
+          rescue Exception => ex
+            report_exception = true
+            # if error is with database connection then only report if it persists longer than 1 minute
+            if @connection.nil? || @connection.status != ::PG::CONNECTION_OK
+              report_exception = false if pg_exception_timestamp.nil? || Process.clock_gettime(Process::CLOCK_MONOTONIC) - pg_exception_timestamp <= 60
+              pg_exception_timestamp ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            end
+            if report_exception
               Skiplock.logger.error(ex.to_s)
               Skiplock.logger.error(ex.backtrace.join("\n"))
-              Skiplock.on_errors.each { |p| p.call(ex, @last_exception) }
-              error = true
-              wait(5)
-              @last_exception = ex
+              Skiplock.on_errors.each { |p| p.call(ex) }
             end
-            sleep(0.3)
+            wait(5)
           end
+          sleep(0.3)
         end
-        connection.exec_query('UNLISTEN *')
       end
+    ensure
+      @connection.close if @connection && !@connection.finished?
     end
 
-    def wait(timeout)
+    def wait(timeout = 1)
       t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       while @running
         sleep(0.5)

data/lib/skiplock.rb

@@ -11,7 +11,7 @@ require 'skiplock/worker'
 require 'skiplock/version'
 
 module Skiplock
-  DEFAULT_CONFIG = { 'extensions' => false, 'logfile' => 'skiplock.log', 'loglevel' => 'info', 'graceful_shutdown' => 15, 'min_threads' => 1, 'max_threads' => 10, 'max_retries' => 20, 'notification' => 'custom', 'purge_completion' => true, 'queues' => { 'default' => 100, 'mailers' => 999 }, 'workers' => 0 }.freeze
+  DEFAULT_CONFIG = { 'graceful_shutdown' => 15, 'min_threads' => 1, 'max_threads' => 10, 'max_retries' => 20, 'logfile' => 'skiplock.log', 'loglevel' => 'info', 'notification' => 'custom', 'actioncable' => false, 'extensions' => false, 'purge_completion' => true, 'queues' => { 'default' => 100, 'mailers' => 999 }, 'workers' => 0 }.freeze
 
   def self.logger=(l)
     @logger = l
@@ -34,5 +34,4 @@ module Skiplock
   def self.table_name_prefix
     'skiplock.'
   end
-end
-ActiveJob::Base.__send__(:include, Skiplock::Patch)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skiplock
 version: !ruby/object:Gem::Version
-  version: 1.0.20
+  version: 1.0.24
 platform: ruby
 authors:
 - Tin Vo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-14 00:00:00.000000000 Z
+date: 2021-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob