skiplock 1.0.4 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c53cdb42608339ffd7ba7532393e933ce9128757ee35679a665fd2f1e1d0fba3
-  data.tar.gz: e7dfcc14eed82fef12747d9c0172de006cdb8823fdcaa1d7e944b8d0bde71be1
+  metadata.gz: 572dcd7b3a55c259ed181c8d893b38b0d27e682d7647347fd020e1f4cc3b7cd8
+  data.tar.gz: 227e1f528be6ee5c86eb79173f0922fbcf5e436caab3158eafa5c40ed7e658c3
 SHA512:
-  metadata.gz: b32ac8c7b3cf90be24c86310d4e2b2fe250a5ec7b244aad267144f49e09f14d9a46e64214f45afde8093d4a867f1c93245c5e216ac214cacca555f3965770f7c
-  data.tar.gz: 0a61bf8d14f3162fe981420d43278642ac98a4ab5ac6705bc9a9f65ddd3330e26b6ec1d3ce9a196155e2316ed972954cd657f4d36a48bda83554ea97eb205237
+  metadata.gz: 3ddaba15001823730cf79cd644398ec14db0997d16413e694b4578ac599099e3ff39f754b18b103f6ed0005803e718b050ac2b9b45bb3b095977fef1f1fe46ef
+  data.tar.gz: 5ff4be776646109fad4ffe1476ee982f8fb47ed111e13cb836bb7c07988c9c4a8753ae43cfbd05f8694a2759264a59daedf094cf645325bada99dfa930d5d333

data/lib/generators/skiplock/templates/migration.rb.erb

@@ -4,10 +4,10 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
     create_table 'skiplock.jobs', id: :uuid do |t|
       t.uuid      :worker_id, index: true
       t.string    :job_class, null: false
-      t.string    :cron
-      t.string    :queue_name
+      t.string    :queue_name, index: true
       t.string    :locale
       t.string    :timezone
+      t.string    :cron
       t.integer   :priority
       t.integer   :executions
       t.jsonb     :exception_executions
@@ -18,26 +18,48 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
       t.timestamp :scheduled_at
       t.timestamps null: false, default: -> { 'now()' }
     end
-    create_table 'skiplob.workers', id: :uuid do |t|
+    create_table 'skiplock.workers', id: :uuid do |t|
       t.integer   :pid, null: false, index: true
       t.integer   :ppid, index: true
-      t.integer   :remaining_capacity, null: false
+      t.integer   :capacity, null: false
       t.string    :hostname, null: false, index: true
       t.jsonb     :data
       t.timestamps null: false, index: true, default: -> { 'now()' }
     end    
-    execute %(CREATE OR REPLACE FUNCTION skiplock.notify() RETURNS TRIGGER AS $$
-      BEGIN
-        IF (NEW.finished_at IS NULL AND NEW.expired_at IS NULL) THEN
-          PERFORM pg_notify('skiplock', CONCAT(TG_OP,',',NEW.id::TEXT,',',NEW.queue_name,',',NEW.priority,',',CAST(EXTRACT(EPOCH FROM NEW.scheduled_at) AS FLOAT)::text));
-        END IF;
-        RETURN NULL;
-      END;
+    execute <<~ENDFUNC
+      CREATE OR REPLACE FUNCTION skiplock.notify_jobs() RETURNS TRIGGER AS $$
+        DECLARE
+          record RECORD;
+        BEGIN
+          IF (TG_OP = 'DELETE') THEN
+            record = OLD;
+          ELSE
+            record = NEW;
+          END IF;
+          PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.scheduled_at) AS FLOAT)::TEXT));
+          RETURN NULL;
+        END;
       $$ LANGUAGE plpgsql
-    )
-    execute "CREATE TRIGGER notify_job AFTER INSERT OR UPDATE ON skiplock.jobs FOR EACH ROW EXECUTE PROCEDURE skiplock.notify()"
-    execute "CREATE INDEX jobs_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE expired_at IS NULL AND finished_at IS NULL"
-    execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
+    ENDFUNC
+    execute <<~ENDFUNC
+      CREATE OR REPLACE FUNCTION skiplock.notify_workers() RETURNS TRIGGER AS $$
+        DECLARE
+          record RECORD;
+        BEGIN
+          IF (TG_OP = 'DELETE') THEN
+            record = OLD;
+          ELSE
+            record = NEW;
+          END IF;
+          PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.hostname,',',record.capacity,',',record.pid,',',record.ppid));
+          RETURN NULL;
+        END;
+      $$ LANGUAGE plpgsql;
+    ENDFUNC
+    execute "CREATE TRIGGER notify_job AFTER INSERT OR UPDATE OR DELETE ON skiplock.jobs FOR EACH ROW EXECUTE PROCEDURE skiplock.notify_jobs()"
+    execute "CREATE TRIGGER notify_worker AFTER INSERT OR UPDATE OR DELETE ON skiplock.workers FOR EACH ROW EXECUTE PROCEDURE skiplock.notify_workers()"
+    execute "CREATE INDEX jobs_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL"
+    execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE running = FALSE AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
     execute "CREATE INDEX jobs_cron_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE cron IS NOT NULL AND finished_at IS NULL"
     execute "CREATE UNIQUE INDEX jobs_unique_cron_index ON skiplock.jobs (job_class) WHERE cron IS NOT NULL"
   end

data/lib/skiplock.rb

@@ -16,6 +16,10 @@ module Skiplock
     'max_threads' => 5,
     'max_retries' => 20,
     'purge_completion' => true,
+    'queues' => {
+      'default' => 100,
+      'mailers' => 999
+    },
     'workers' => 0
   }
 end

data/lib/skiplock/dispatcher.rb

@@ -1,6 +1,7 @@
 module Skiplock
   class Dispatcher
     def initialize(master: true, worker_num: nil, worker_pids: [])
+      @queues_order_query = Skiplock::Settings['queues'].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ')
       @executor = Concurrent::ThreadPoolExecutor.new(min_threads: Settings['min_threads'], max_threads: Settings['max_threads'], max_queue: Settings['max_threads'], idletime: 60, auto_terminate: true, fallback_policy: :discard)
       @master = master
       if @master
@@ -62,7 +63,7 @@ module Skiplock
                     if scheduled_at.to_f <= Time.now.to_f
                       @next_schedule_at = Time.now.to_f
                     elsif scheduled_at.to_f < @next_schedule_at
-                      @next_schedule_at = time.to_f
+                      @next_schedule_at = scheduled_at.to_f
                     end
                   end
                 end
@@ -94,7 +95,7 @@ module Skiplock
 
     def do_work
       while @running
-        result = Job.dispatch(worker_id: @worker.id)
+        result = Job.dispatch(queues_order_query: @queues_order_query, worker_id: @worker.id)
         next if result.is_a?(Hash)
         @next_schedule_at = result if result.is_a?(Float)
         break

data/lib/skiplock/job.rb

@@ -2,10 +2,10 @@ module Skiplock
   class Job < ActiveRecord::Base
     self.table_name = 'skiplock.jobs'
 
-    # Return: Attributes hash of the Job if it was executed; otherwise returns the next Job's schedule time in FLOAT
-    def self.dispatch(worker_id: nil)
+    # Return: Skiplock::Job if it was executed; otherwise returns the next Job's schedule time in FLOAT
+    def self.dispatch(queues_order_query: nil, worker_id: nil)
       self.connection.exec_query('BEGIN')
-      job = self.find_by_sql("SELECT id, scheduled_at FROM #{self.table_name} WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
+      job = self.find_by_sql("SELECT id, scheduled_at FROM #{self.table_name} WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{queues_order_query ? 'CASE ' + queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
       if job.nil? || job.scheduled_at.to_f > Time.now.to_f
         self.connection.exec_query('END')
         return (job ? job.scheduled_at.to_f : Float::INFINITY)
@@ -54,7 +54,7 @@ module Skiplock
       end
       job
     ensure
-      Thread.current[:skiplock_dispatch_data] = nil
+      Thread.current[:skiplock_dispatch_job] = nil
     end
 
     def self.enqueue_at(activejob, timestamp)

data/lib/skiplock/manager.rb

@@ -31,7 +31,11 @@ module Skiplock
       Settings['max_threads'] = 20 if Settings['max_threads'] > 20
       Settings['min_threads'] = 0 if Settings['min_threads'] < 0
       Settings['workers'] = 0 if Settings['workers'] < 0
+      Settings['queues'].values.each { |v| raise 'Queue value must be an integer' unless v.is_a?(Integer) }
       Settings.freeze
+    rescue Exception => e
+      STDERR.puts "Invalid configuration 'config/skiplock.yml': #{e.message}"
+      exit
     end
 
     def self.standalone
@@ -46,6 +50,7 @@ module Skiplock
       puts "       Max threads: #{Settings['max_threads']}"
       puts "       Environment: #{Rails.env}"
       puts "           Logging: #{Settings['logging']}"
+      puts "            Queues: #{Settings['queues'].map {|k,v| k + '(' + v.to_s + ')'}.join(', ')}"
       puts "               PID: #{Process.pid}"
       puts "-"*(title.length)
       if Settings['logging']

data/lib/skiplock/version.rb

@@ -1,4 +1,4 @@
 module Skiplock
-  VERSION = Version = '1.0.4'
+  VERSION = Version = '1.0.5'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skiplock
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.5
 platform: ruby
 authors:
 - Tin Vo