skiplock 1.0.4 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c53cdb42608339ffd7ba7532393e933ce9128757ee35679a665fd2f1e1d0fba3
4
- data.tar.gz: e7dfcc14eed82fef12747d9c0172de006cdb8823fdcaa1d7e944b8d0bde71be1
3
+ metadata.gz: 572dcd7b3a55c259ed181c8d893b38b0d27e682d7647347fd020e1f4cc3b7cd8
4
+ data.tar.gz: 227e1f528be6ee5c86eb79173f0922fbcf5e436caab3158eafa5c40ed7e658c3
5
5
  SHA512:
6
- metadata.gz: b32ac8c7b3cf90be24c86310d4e2b2fe250a5ec7b244aad267144f49e09f14d9a46e64214f45afde8093d4a867f1c93245c5e216ac214cacca555f3965770f7c
7
- data.tar.gz: 0a61bf8d14f3162fe981420d43278642ac98a4ab5ac6705bc9a9f65ddd3330e26b6ec1d3ce9a196155e2316ed972954cd657f4d36a48bda83554ea97eb205237
6
+ metadata.gz: 3ddaba15001823730cf79cd644398ec14db0997d16413e694b4578ac599099e3ff39f754b18b103f6ed0005803e718b050ac2b9b45bb3b095977fef1f1fe46ef
7
+ data.tar.gz: 5ff4be776646109fad4ffe1476ee982f8fb47ed111e13cb836bb7c07988c9c4a8753ae43cfbd05f8694a2759264a59daedf094cf645325bada99dfa930d5d333
@@ -4,10 +4,10 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
4
4
  create_table 'skiplock.jobs', id: :uuid do |t|
5
5
  t.uuid :worker_id, index: true
6
6
  t.string :job_class, null: false
7
- t.string :cron
8
- t.string :queue_name
7
+ t.string :queue_name, index: true
9
8
  t.string :locale
10
9
  t.string :timezone
10
+ t.string :cron
11
11
  t.integer :priority
12
12
  t.integer :executions
13
13
  t.jsonb :exception_executions
@@ -18,26 +18,48 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
18
18
  t.timestamp :scheduled_at
19
19
  t.timestamps null: false, default: -> { 'now()' }
20
20
  end
21
- create_table 'skiplob.workers', id: :uuid do |t|
21
+ create_table 'skiplock.workers', id: :uuid do |t|
22
22
  t.integer :pid, null: false, index: true
23
23
  t.integer :ppid, index: true
24
- t.integer :remaining_capacity, null: false
24
+ t.integer :capacity, null: false
25
25
  t.string :hostname, null: false, index: true
26
26
  t.jsonb :data
27
27
  t.timestamps null: false, index: true, default: -> { 'now()' }
28
28
  end
29
- execute %(CREATE OR REPLACE FUNCTION skiplock.notify() RETURNS TRIGGER AS $$
30
- BEGIN
31
- IF (NEW.finished_at IS NULL AND NEW.expired_at IS NULL) THEN
32
- PERFORM pg_notify('skiplock', CONCAT(TG_OP,',',NEW.id::TEXT,',',NEW.queue_name,',',NEW.priority,',',CAST(EXTRACT(EPOCH FROM NEW.scheduled_at) AS FLOAT)::text));
33
- END IF;
34
- RETURN NULL;
35
- END;
29
+ execute <<~ENDFUNC
30
+ CREATE OR REPLACE FUNCTION skiplock.notify_jobs() RETURNS TRIGGER AS $$
31
+ DECLARE
32
+ record RECORD;
33
+ BEGIN
34
+ IF (TG_OP = 'DELETE') THEN
35
+ record = OLD;
36
+ ELSE
37
+ record = NEW;
38
+ END IF;
39
+ PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.scheduled_at) AS FLOAT)::TEXT));
40
+ RETURN NULL;
41
+ END;
36
42
  $$ LANGUAGE plpgsql
37
- )
38
- execute "CREATE TRIGGER notify_job AFTER INSERT OR UPDATE ON skiplock.jobs FOR EACH ROW EXECUTE PROCEDURE skiplock.notify()"
39
- execute "CREATE INDEX jobs_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE expired_at IS NULL AND finished_at IS NULL"
40
- execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
43
+ ENDFUNC
44
+ execute <<~ENDFUNC
45
+ CREATE OR REPLACE FUNCTION skiplock.notify_workers() RETURNS TRIGGER AS $$
46
+ DECLARE
47
+ record RECORD;
48
+ BEGIN
49
+ IF (TG_OP = 'DELETE') THEN
50
+ record = OLD;
51
+ ELSE
52
+ record = NEW;
53
+ END IF;
54
+ PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.hostname,',',record.capacity,',',record.pid,',',record.ppid));
55
+ RETURN NULL;
56
+ END;
57
+ $$ LANGUAGE plpgsql;
58
+ ENDFUNC
59
+ execute "CREATE TRIGGER notify_job AFTER INSERT OR UPDATE OR DELETE ON skiplock.jobs FOR EACH ROW EXECUTE PROCEDURE skiplock.notify_jobs()"
60
+ execute "CREATE TRIGGER notify_worker AFTER INSERT OR UPDATE OR DELETE ON skiplock.workers FOR EACH ROW EXECUTE PROCEDURE skiplock.notify_workers()"
61
+ execute "CREATE INDEX jobs_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL"
62
+ execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE running = FALSE AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
41
63
  execute "CREATE INDEX jobs_cron_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE cron IS NOT NULL AND finished_at IS NULL"
42
64
  execute "CREATE UNIQUE INDEX jobs_unique_cron_index ON skiplock.jobs (job_class) WHERE cron IS NOT NULL"
43
65
  end
data/lib/skiplock.rb CHANGED
@@ -16,6 +16,10 @@ module Skiplock
16
16
  'max_threads' => 5,
17
17
  'max_retries' => 20,
18
18
  'purge_completion' => true,
19
+ 'queues' => {
20
+ 'default' => 100,
21
+ 'mailers' => 999
22
+ },
19
23
  'workers' => 0
20
24
  }
21
25
  end
@@ -1,6 +1,7 @@
1
1
  module Skiplock
2
2
  class Dispatcher
3
3
  def initialize(master: true, worker_num: nil, worker_pids: [])
4
+ @queues_order_query = Skiplock::Settings['queues'].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ')
4
5
  @executor = Concurrent::ThreadPoolExecutor.new(min_threads: Settings['min_threads'], max_threads: Settings['max_threads'], max_queue: Settings['max_threads'], idletime: 60, auto_terminate: true, fallback_policy: :discard)
5
6
  @master = master
6
7
  if @master
@@ -62,7 +63,7 @@ module Skiplock
62
63
  if scheduled_at.to_f <= Time.now.to_f
63
64
  @next_schedule_at = Time.now.to_f
64
65
  elsif scheduled_at.to_f < @next_schedule_at
65
- @next_schedule_at = time.to_f
66
+ @next_schedule_at = scheduled_at.to_f
66
67
  end
67
68
  end
68
69
  end
@@ -94,7 +95,7 @@ module Skiplock
94
95
 
95
96
  def do_work
96
97
  while @running
97
- result = Job.dispatch(worker_id: @worker.id)
98
+ result = Job.dispatch(queues_order_query: @queues_order_query, worker_id: @worker.id)
98
99
  next if result.is_a?(Hash)
99
100
  @next_schedule_at = result if result.is_a?(Float)
100
101
  break
data/lib/skiplock/job.rb CHANGED
@@ -2,10 +2,10 @@ module Skiplock
2
2
  class Job < ActiveRecord::Base
3
3
  self.table_name = 'skiplock.jobs'
4
4
 
5
- # Return: Attributes hash of the Job if it was executed; otherwise returns the next Job's schedule time in FLOAT
6
- def self.dispatch(worker_id: nil)
5
+ # Return: Skiplock::Job if it was executed; otherwise returns the next Job's schedule time in FLOAT
6
+ def self.dispatch(queues_order_query: nil, worker_id: nil)
7
7
  self.connection.exec_query('BEGIN')
8
- job = self.find_by_sql("SELECT id, scheduled_at FROM #{self.table_name} WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
8
+ job = self.find_by_sql("SELECT id, scheduled_at FROM #{self.table_name} WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{queues_order_query ? 'CASE ' + queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
9
9
  if job.nil? || job.scheduled_at.to_f > Time.now.to_f
10
10
  self.connection.exec_query('END')
11
11
  return (job ? job.scheduled_at.to_f : Float::INFINITY)
@@ -54,7 +54,7 @@ module Skiplock
54
54
  end
55
55
  job
56
56
  ensure
57
- Thread.current[:skiplock_dispatch_data] = nil
57
+ Thread.current[:skiplock_dispatch_job] = nil
58
58
  end
59
59
 
60
60
  def self.enqueue_at(activejob, timestamp)
@@ -31,7 +31,11 @@ module Skiplock
31
31
  Settings['max_threads'] = 20 if Settings['max_threads'] > 20
32
32
  Settings['min_threads'] = 0 if Settings['min_threads'] < 0
33
33
  Settings['workers'] = 0 if Settings['workers'] < 0
34
+ Settings['queues'].values.each { |v| raise 'Queue value must be an integer' unless v.is_a?(Integer) }
34
35
  Settings.freeze
36
+ rescue Exception => e
37
+ STDERR.puts "Invalid configuration 'config/skiplock.yml': #{e.message}"
38
+ exit
35
39
  end
36
40
 
37
41
  def self.standalone
@@ -46,6 +50,7 @@ module Skiplock
46
50
  puts " Max threads: #{Settings['max_threads']}"
47
51
  puts " Environment: #{Rails.env}"
48
52
  puts " Logging: #{Settings['logging']}"
53
+ puts " Queues: #{Settings['queues'].map {|k,v| k + '(' + v.to_s + ')'}.join(', ')}"
49
54
  puts " PID: #{Process.pid}"
50
55
  puts "-"*(title.length)
51
56
  if Settings['logging']
@@ -1,4 +1,4 @@
1
1
  module Skiplock
2
- VERSION = Version = '1.0.4'
2
+ VERSION = Version = '1.0.5'
3
3
  end
4
4
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: skiplock
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.4
4
+ version: 1.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tin Vo