skiplock 1.0.20 → 1.0.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 982ed70ce969943c421cad0923a280c87c4c26ea90da5a3451ce73becd5db42e
-  data.tar.gz: 9f68becb9ab21e00816315d1e2ff20058d2f5942a3688ed78959018e3ccf940f
+  metadata.gz: 69e771cf5508d4a5a648dab43eb33b980370e317a1bf370ceaff62ecee4592e3
+  data.tar.gz: d1df71a515d871073b8d039f1bcbc3fb00b398806ed9f3bd82dc0169737e2f52
 SHA512:
-  metadata.gz: 0fb583418cf8c0a190f9b651ecd95effba2d4b415f91bf6fbd452d2271f02534e7c5823e360f1e7a258cdd0a22b14cc99d44915007e9c32a186371748c086144
-  data.tar.gz: 6aa546ad0a538213ae9587e667543027fb2ca080e7623b3f409fe6303c005085f39fd27c310a690a2da9fbb50321d97519ed9aee7bb4922a126c0a280f6ad5fe
+  metadata.gz: cca4a86fc4c31e4054a920b32740f294e3f54e9de65fe3a0c31b43258fee6e6f1a790ae1a95f79d9b719119936fedac953afd695ef5e7ce9e990b7f29fa1c397
+  data.tar.gz: 6bdc5a38218b052a825ab2b5f5ef2c73a2c9a6a2d981d8aed7cc61244f8806fdad06876278aa320bafb956c9dc2f15b705d87a2f171cbb2bc3805bfada640ce4

data/README.md

@@ -172,14 +172,10 @@ If the `retry_on` block is not defined, then the built-in retry system of `Skipl
 `Skiplock` can use existing exception notification library to notify errors and exceptions.  It supports `airbrake`, `bugsnag`, and `exception_notification`.  Custom notification can also be called whenever an exception occurs; it can be configured in an initializer like below:
 ```ruby
   # config/initializers/skiplock.rb
-  Skiplock.on_error do |ex, previous|
-    if ex.backtrace != previous.try(:backtrace)
-      # sends text message using Amazon SNS on new exceptions only
-      # the same repeated exceptions will only be sent once to avoid SPAM
-      # NOTE: exceptions generated from Job executions will not provide 'previous' exceptions
-      sms = Aws::SNS::Client.new(region: 'us-west-2', access_key_id: Rails.application.credentials[:aws][:access_key_id], secret_access_key: Rails.application.credentials[:aws][:secret_access_key])
-      sms.publish({ phone_number: '+122233334444', message: "Exception: #{ex.message}"[0..130] })
-    end
+  Skiplock.on_error do |ex|
+    # sends text message using Amazon SNS
+    sms = Aws::SNS::Client.new(region: 'us-west-2', access_key_id: Rails.application.credentials[:aws][:access_key_id], secret_access_key: Rails.application.credentials[:aws][:secret_access_key])
+    sms.publish(phone_number: '+12223334444', message: "Exception: #{ex.message}"[0..130])
   end
   # supports multiple 'on_error' event callbacks
 ```

data/lib/skiplock/manager.rb

@@ -116,18 +116,18 @@ module Skiplock
       case @config[:notification]
       when 'airbrake'
         raise 'airbrake gem not found' unless defined?(Airbrake)
-        Skiplock.on_error do |ex, previous|
-          Airbrake.notify_sync(ex) unless ex.backtrace == previous.try(:backtrace)
+        Skiplock.on_error do |ex|
+          Airbrake.notify_sync(ex)
         end
       when 'bugsnag'
         raise 'bugsnag gem not found' unless defined?(Bugsnag)
-        Skiplock.on_error do |ex, previous|
-          Bugsnag.notify(ex) unless ex.backtrace == previous.try(:backtrace)
+        Skiplock.on_error do |ex|
+          Bugsnag.notify(ex)
         end
       when 'exception_notification'
         raise 'exception_notification gem not found' unless defined?(ExceptionNotifier)
-        Skiplock.on_error do |ex, previous|
-          ExceptionNotifier.notify_exception(ex) unless ex.backtrace == previous.try(:backtrace)
+        Skiplock.on_error do |ex|
+          ExceptionNotifier.notify_exception(ex)
         end
       else
         @config[:notification] = 'custom'

data/lib/skiplock/version.rb

@@ -1,4 +1,4 @@
 module Skiplock
-  VERSION = Version = '1.0.20'
+  VERSION = Version = '1.0.21'
 end
 

data/lib/skiplock/worker.rb

@@ -18,95 +18,100 @@ module Skiplock
       self.create!(pid: Process.pid, sid: Process.getsid(), master: false, hostname: hostname, capacity: capacity)
     end
 
+    def shutdown
+      @running = false
+      @executor.shutdown
+      @executor.kill unless @executor.wait_for_termination(@config[:graceful_shutdown])
+      self.delete
+      Skiplock.logger.info "[Skiplock] Shutdown of #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} (PID: #{self.pid}) was completed."
+    end
+
     def start(worker_num: 0, **config)
-      if self.master
-        Job.flush
-        Cron.setup
-      end
       @num = worker_num
       @config = config
       @queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
       @running = true
-      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads], idletime: 60, auto_terminate: true, fallback_policy: :discard)
+      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: true, fallback_policy: :discard)
+      if self.master
+        Job.flush
+        Cron.setup
+      end
       @executor.post { run }
       Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]") if @config[:standalone]
     end
 
-    def shutdown
-      @running = false
-      @executor.shutdown
-      @executor.kill unless @executor.wait_for_termination(@config[:graceful_shutdown])
-      self.delete
-      Skiplock.logger.info "[Skiplock] Shutdown of #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} (PID: #{self.pid}) was completed."
-    end
-
     private
 
+    def reloader_post
+      Rails.application.reloader.wrap { @executor.post { Rails.application.executor.wrap { yield } } } if block_given?
+    end
+
     def run
       sleep 3
+      Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
+      error = false
+      next_schedule_at = Time.now.to_f
+      pg_exception_timestamp = nil
+      timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       ActiveRecord::Base.connection_pool.with_connection do |connection|
-        Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
         connection.exec_query('LISTEN "skiplock::jobs"')
-        error = false
-        next_schedule_at = Time.now.to_f
-        timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         while @running
-          Rails.application.reloader.wrap do
-            begin
-              if error
-                unless connection.active?
-                  connection.reconnect!
-                  sleep(0.5)
-                  connection.exec_query('LISTEN "skiplock::jobs"')
-                  next_schedule_at = Time.now.to_f
-                end
-                Job.flush if self.master
-                error = false
+          begin
+            if error
+              unless connection.active?
+                connection.reconnect!
+                sleep(0.5)
+                connection.exec_query('LISTEN "skiplock::jobs"')
+                reloader_post { Job.flush } if self.master
+                pg_exception_timestamp = nil
+                next_schedule_at = Time.now.to_f
               end
-              if Time.now.to_f >= next_schedule_at && @executor.remaining_capacity > 0
-                job = nil
-                connection.transaction do
-                  result = connection.select_all("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
-                  result = connection.select_all("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *").first if result && result['scheduled_at'].to_f <= Time.now.to_f
-                  job = Job.instantiate(result) if result
-                end
-                if job.try(:running)
-                  @executor.post do
-                    Rails.application.executor.wrap { job.execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
-                  end
-                else
-                  next_schedule_at = (job ? job.scheduled_at.to_f : Float::INFINITY)
-                end
+              error = false
+            end
+            if Time.now.to_f >= next_schedule_at && @executor.remaining_capacity > 1  # reserves 1 slot in queue for Job.flush in case of pg_connection error
+              result = nil
+              connection.transaction do
+                result = connection.select_all("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
+                result = connection.select_all("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *").first if result && result['scheduled_at'].to_f <= Time.now.to_f
               end
-              job_notifications = []
-              connection.raw_connection.wait_for_notify(0.2) do |channel, pid, payload|
-                job_notifications << payload if payload
-                loop do
-                  payload = connection.raw_connection.notifies
-                  break unless @running && payload
-                  job_notifications << payload[:extra]
-                end
-                job_notifications.each do |n|
-                  op, id, worker_id, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
-                  next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0
-                  next_schedule_at = scheduled_at.to_f if scheduled_at.to_f < next_schedule_at
-                end
+              if result && result['running']
+                reloader_post { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
+              else
+                next_schedule_at = (result ? result['scheduled_at'].to_f : Float::INFINITY)
               end
-              if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
-                self.touch
-                timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            end
+            job_notifications = []
+            connection.raw_connection.wait_for_notify(0.2) do |channel, pid, payload|
+              job_notifications << payload if payload
+              loop do
+                payload = connection.raw_connection.notifies
+                break unless @running && payload
+                job_notifications << payload[:extra]
               end
-            rescue Exception => ex
-              # most likely error with database connection
-              Skiplock.logger.error(ex.to_s)
-              Skiplock.logger.error(ex.backtrace.join("\n"))
-              Skiplock.on_errors.each { |p| p.call(ex, @last_exception) }
-              error = true
-              wait(5)
-              @last_exception = ex
+              job_notifications.each do |n|
+                op, id, worker_id, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
+                next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0
+                next_schedule_at = scheduled_at.to_f if scheduled_at.to_f < next_schedule_at
+              end
+            end
+            if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
+              connection.exec_query("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'")
+              timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            end
+          rescue Exception => ex
+            Skiplock.logger.error(ex.to_s)
+            Skiplock.logger.error(ex.backtrace.join("\n"))
+            report_exception = true
+            # if error is with database connection then only report if it persists longer than 1 minute
+            if ex.is_a?(::PG::ConnectionBad)
+              report_exception = false if pg_exception_timestamp.nil? || Process.clock_gettime(Process::CLOCK_MONOTONIC) - pg_exception_timestamp <= 60
+              pg_exception_timestamp ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
             end
-            sleep(0.3)
+            Skiplock.on_errors.each { |p| p.call(ex) } if report_exception
+            error = true
+            wait(5)
           end
+          sleep(0.3)
         end
         connection.exec_query('UNLISTEN *')
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skiplock
 version: !ruby/object:Gem::Version
-  version: 1.0.20
+  version: 1.0.21
 platform: ruby
 authors:
 - Tin Vo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-14 00:00:00.000000000 Z
+date: 2021-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob