skiplock 1.0.22 → 1.0.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b99466d7f1848f9dcd750a4dceb6a39d7124707e941c74b6c75f0b3a57944ff
-  data.tar.gz: 3f3ed1c9327d41e675d5c170590b52fe32f1facc5b57667dba513e5324e6ee20
+  metadata.gz: 275e1ddd1a59648e86378889e2e2cc6a4e22943a3fb4211209f20a674744b1b3
+  data.tar.gz: 7327f5893f36cc4b05163cfd0df47992344b2ca6b97c2ef10ca4f00b219a9ac9
 SHA512:
-  metadata.gz: '0129296507b0479ba28a6e4e0bce810340f0ff2559004da1385cf02b6d6023f7be5cb1543f3fb24cebdeb579d6be6ed1e5d7d72d43245de5374926607b23d7df'
-  data.tar.gz: 163faf212d06b6d7f8045eb347ccab77de5b16bfe77fe8780a5d73176ce3c5804cc25cbca8f962d92bc5dfccc1ada223d03fc0b443d0df48125a12bf4ec7ddeb
+  metadata.gz: e9b8afd563ec4ff46838e73ae031a75a69142ee392a05d0e5b612008153b10a6a62a931e22643e817722d0017548573f9cafed6ba943f41da3cf8d0872d76481
+  data.tar.gz: 8b04173649d9e4a21edb9d47e23d37d5c8d0b08ba8658f46c0fd8341270abf1148a2131e3f57251421caf024d5ce8d9bfb35a648ee9a904f999d6b21bf0fe5ec

data/README.md

@@ -85,6 +85,7 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
     Usage: skiplock [options]
       -e, --environment STRING         Rails environment
       -l, --logfile STRING             Log filename
+      -L, --loglevel STRING            Log level (debug, info, warn, error, fatal, unknown)
       -s, --graceful-shutdown NUM      Number of seconds to wait for graceful shutdown
       -r, --max-retries NUM            Number of maxixum retries
       -t, --max-threads NUM            Number of maximum threads

data/lib/skiplock/manager.rb

@@ -10,7 +10,7 @@ module Skiplock
       elsif @config[:extensions].is_a?(Array)
         @config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
       end
-      async if (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0)
+      (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0) ? async : Cron.setup
     end
 
     def async
@@ -18,6 +18,7 @@ module Skiplock
       configure
       Worker.cleanup(@hostname)
       @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
+      Cron.setup if @worker.master
       @worker.start(**@config)
       at_exit { @worker.shutdown }
     rescue Exception => ex
@@ -42,7 +43,8 @@ module Skiplock
       ActiveRecord::Base.connection.disconnect! if @config[:workers] > 1
       (@config[:workers] - 1).times do |n|
         fork do
-          sleep 1
+          sleep(0.25*n + 1)
+          ActiveRecord::Base.establish_connection
           worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false)
           worker.start(worker_num: n + 1, **@config)
           loop do
@@ -90,7 +92,7 @@ module Skiplock
     end
 
     def configure
-      @hostname = Socket.gethostname
+      @hostname = Socket.ip_address_list.reject(&:ipv4_loopback?).reject(&:ipv6?).map(&:ip_address).join('|')
       @config.transform_values! {|v| v.is_a?(String) ? v.downcase : v}
       @config[:graceful_shutdown] = 300 if @config[:graceful_shutdown] > 300
       @config[:graceful_shutdown] = nil if @config[:graceful_shutdown] < 0
@@ -145,6 +147,7 @@ module Skiplock
       end
       if @config[:standalone]
         Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
+        Rails.logger.level = @logger.level
         Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
       end
     rescue Exception => ex

data/lib/skiplock/version.rb

@@ -1,4 +1,4 @@
 module Skiplock
-  VERSION = Version = '1.0.22'
+  VERSION = Version = '1.0.23'
 end
 

data/lib/skiplock/worker.rb

@@ -29,66 +29,67 @@ module Skiplock
     def start(worker_num: 0, **config)
       @num = worker_num
       @config = config
+      @pg_config = ActiveRecord::Base.connection.raw_connection.conninfo_hash.compact
       @queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
       @running = true
-      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: true, fallback_policy: :discard)
+      @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: true, fallback_policy: :abort)
       @executor.post { run }
-      Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]") if @config[:standalone]
+      if @config[:standalone]
+        Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]")
+        ActiveRecord::Base.connection.throw_away!
+      end
     end
 
     private
 
+    def establish_connection
+      map = ::PG::TypeMapByOid.new
+      map.add_coder(::PG::TextDecoder::Boolean.new(oid: 16, name: 'bool'))
+      map.add_coder(::PG::TextDecoder::Integer.new(oid: 20, name: 'int8'))
+      map.add_coder(::PG::TextDecoder::Integer.new(oid: 21, name: 'int2'))
+      map.add_coder(::PG::TextDecoder::Integer.new(oid: 23, name: 'int4'))
+      map.add_coder(::PG::TextDecoder::TimestampUtc.new(oid: 1114, name: 'timestamp'))
+      map.add_coder(::PG::TextDecoder::String.new(oid: 2950, name: 'uuid'))
+      map.add_coder(::PG::TextDecoder::JSON.new(oid: 3802, name: 'jsonb'))
+      @connection = ::PG.connect(@pg_config)
+      @connection.type_map_for_results = map
+      @connection.exec('LISTEN "skiplock::jobs"').clear
+    end
+
     def run
       sleep 3
       Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
-      connection = nil
-      error = false
-      listen = false
       next_schedule_at = Time.now.to_f
       pg_exception_timestamp = nil
       timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       while @running
         Rails.application.reloader.wrap do
           begin
-            unless listen
-              connection = self.class.connection
-              connection.exec_query('LISTEN "skiplock::jobs"')
-              if self.master
-                Job.flush
-                Cron.setup
-              end
-              listen = true
-            end
-            if error
-              unless connection.active?
-                connection.reconnect!
-                sleep(0.5)
-                connection.exec_query('LISTEN "skiplock::jobs"')
-                Job.flush if self.master
-                pg_exception_timestamp = nil
-                next_schedule_at = Time.now.to_f
-              end
-              error = false
+            if @connection.nil? || @connection.status != ::PG::CONNECTION_OK
+              establish_connection
+              @executor.post { Rails.application.executor.wrap { Job.flush } } if self.master
+              pg_exception_timestamp = nil
+              next_schedule_at = Time.now.to_f
             end
             if Time.now.to_f >= next_schedule_at && @executor.remaining_capacity > 1  # reserves 1 slot in queue for Job.flush in case of pg_connection error
               result = nil
-              connection.transaction do
-                result = connection.select_all("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
-                result = connection.select_all("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *").first if result && result['scheduled_at'].to_f <= Time.now.to_f
+              @connection.transaction do |conn|
+                conn.exec("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1") do |r|
+                  result = r.first
+                  conn.exec("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *") { |r| result = r.first } if result && result['scheduled_at'].to_f <= Time.now.to_f
+                end
               end
               if result && result['running']
-                @executor.post do
-                  Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
-                end
+                @executor.post { Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) } }
               else
                 next_schedule_at = (result ? result['scheduled_at'].to_f : Float::INFINITY)
               end
             end
             job_notifications = []
-            connection.raw_connection.wait_for_notify(0.2) do |channel, pid, payload|
+            @connection.wait_for_notify(0.2) do |channel, pid, payload|
               job_notifications << payload if payload
               loop do
-                payload = connection.raw_connection.notifies
+                payload = @connection.notifies
                 break unless @running && payload
                 job_notifications << payload[:extra]
               end
@@ -99,29 +100,31 @@ module Skiplock
               end
             end
             if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
-              connection.exec_query("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'")
+              @connection.exec("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'").clear
               timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
             end
           rescue Exception => ex
-            Skiplock.logger.error(ex.to_s)
-            Skiplock.logger.error(ex.backtrace.join("\n"))
             report_exception = true
             # if error is with database connection then only report if it persists longer than 1 minute
-            if ex.is_a?(::PG::ConnectionBad) ||ex.is_a?(::PG::UnableToSend) || ex.message.include?('Bad file descriptor')
+            if @connection.nil? || @connection.status != ::PG::CONNECTION_OK
               report_exception = false if pg_exception_timestamp.nil? || Process.clock_gettime(Process::CLOCK_MONOTONIC) - pg_exception_timestamp <= 60
               pg_exception_timestamp ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
             end
-            Skiplock.on_errors.each { |p| p.call(ex) } if report_exception
-            error = true
+            if report_exception
+              Skiplock.logger.error(ex.to_s)
+              Skiplock.logger.error(ex.backtrace.join("\n"))
+              Skiplock.on_errors.each { |p| p.call(ex) }
+            end
             wait(5)
           end
           sleep(0.3)
         end
       end
-      connection.exec_query('UNLISTEN *')
+    ensure
+      @connection.close if @connection && !@connection.finished?
     end
 
-    def wait(timeout)
+    def wait(timeout = 1)
       t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       while @running
         sleep(0.5)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skiplock
 version: !ruby/object:Gem::Version
-  version: 1.0.22
+  version: 1.0.23
 platform: ruby
 authors:
 - Tin Vo