que-locks 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -0
- data/Gemfile.lock +50 -9
- data/lib/que/locks/active_job_extensions.rb +29 -0
- data/lib/que/locks/execution_lock.rb +45 -9
- data/lib/que/locks/job_extensions.rb +9 -5
- data/lib/que/locks/lock_middleware.rb +1 -1
- data/lib/que/locks/railtie.rb +13 -0
- data/lib/que/locks/version.rb +1 -1
- data/lib/que/locks.rb +2 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9d0ffcedfe00f4e3e4e6b609c345bfb8f5af2f13c31c7f21394e1e43c71458c1
|
4
|
+
data.tar.gz: 80c89326dfbab470400c543be8760e4966103d1d781602db4135e903f1a38234
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4eedc15d8acf9d9a37cb0ccda407b26c2d7505f259ccabac9d469c9862070a90bd592dd2f6b9dca16fa51bc967077ce12319cf82924a713d2c700d253c8ddf5a
|
7
|
+
data.tar.gz: c4f436197a6546866b8b3d25441f5a548d007c1ae64c7adb218c9b95ea5792be4c68b88f4692cc862c4b32a2e8f56e30d26a4b87d23415b84c4e1755ac297367
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
que-locks (0.
|
4
|
+
que-locks (0.4.0)
|
5
5
|
neatjson (~> 0.9)
|
6
6
|
que (~> 1.0)
|
7
7
|
xxhash (~> 0.4)
|
@@ -9,42 +9,81 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
|
13
|
-
|
12
|
+
actionpack (6.1.4.7)
|
13
|
+
actionview (= 6.1.4.7)
|
14
|
+
activesupport (= 6.1.4.7)
|
15
|
+
rack (~> 2.0, >= 2.0.9)
|
16
|
+
rack-test (>= 0.6.3)
|
17
|
+
rails-dom-testing (~> 2.0)
|
18
|
+
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
19
|
+
actionview (6.1.4.7)
|
20
|
+
activesupport (= 6.1.4.7)
|
21
|
+
builder (~> 3.1)
|
22
|
+
erubi (~> 1.4)
|
23
|
+
rails-dom-testing (~> 2.0)
|
24
|
+
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
25
|
+
activejob (6.1.4.7)
|
26
|
+
activesupport (= 6.1.4.7)
|
14
27
|
globalid (>= 0.3.6)
|
15
|
-
activemodel (6.1.
|
16
|
-
activesupport (= 6.1.
|
17
|
-
activerecord (6.1.
|
18
|
-
activemodel (= 6.1.
|
19
|
-
activesupport (= 6.1.
|
20
|
-
activesupport (6.1.
|
28
|
+
activemodel (6.1.4.7)
|
29
|
+
activesupport (= 6.1.4.7)
|
30
|
+
activerecord (6.1.4.7)
|
31
|
+
activemodel (= 6.1.4.7)
|
32
|
+
activesupport (= 6.1.4.7)
|
33
|
+
activesupport (6.1.4.7)
|
21
34
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
22
35
|
i18n (>= 1.6, < 2)
|
23
36
|
minitest (>= 5.1)
|
24
37
|
tzinfo (~> 2.0)
|
25
38
|
zeitwerk (~> 2.3)
|
26
39
|
ast (2.4.2)
|
40
|
+
builder (3.2.4)
|
27
41
|
byebug (11.1.3)
|
28
42
|
concurrent-ruby (1.1.10)
|
43
|
+
crass (1.0.6)
|
29
44
|
database_cleaner (2.0.1)
|
30
45
|
database_cleaner-active_record (~> 2.0.0)
|
31
46
|
database_cleaner-active_record (2.0.1)
|
32
47
|
activerecord (>= 5.a)
|
33
48
|
database_cleaner-core (~> 2.0.0)
|
34
49
|
database_cleaner-core (2.0.1)
|
50
|
+
erubi (1.10.0)
|
35
51
|
globalid (1.0.0)
|
36
52
|
activesupport (>= 5.0)
|
37
53
|
i18n (1.12.0)
|
38
54
|
concurrent-ruby (~> 1.0)
|
39
55
|
json (2.6.2)
|
56
|
+
loofah (2.18.0)
|
57
|
+
crass (~> 1.0.2)
|
58
|
+
nokogiri (>= 1.5.9)
|
59
|
+
method_source (1.0.0)
|
60
|
+
mini_portile2 (2.8.0)
|
40
61
|
minitest (5.16.2)
|
41
62
|
mocha (1.14.0)
|
42
63
|
neatjson (0.9)
|
64
|
+
nokogiri (1.13.8)
|
65
|
+
mini_portile2 (~> 2.8.0)
|
66
|
+
racc (~> 1.4)
|
43
67
|
parallel (1.22.1)
|
44
68
|
parser (3.1.2.0)
|
45
69
|
ast (~> 2.4.1)
|
46
70
|
pg (1.4.2)
|
47
71
|
que (1.4.1)
|
72
|
+
racc (1.6.0)
|
73
|
+
rack (2.2.4)
|
74
|
+
rack-test (2.0.2)
|
75
|
+
rack (>= 1.3)
|
76
|
+
rails-dom-testing (2.0.3)
|
77
|
+
activesupport (>= 4.2.0)
|
78
|
+
nokogiri (>= 1.6)
|
79
|
+
rails-html-sanitizer (1.4.3)
|
80
|
+
loofah (~> 2.3)
|
81
|
+
railties (6.1.4.7)
|
82
|
+
actionpack (= 6.1.4.7)
|
83
|
+
activesupport (= 6.1.4.7)
|
84
|
+
method_source
|
85
|
+
rake (>= 0.13)
|
86
|
+
thor (~> 1.0)
|
48
87
|
rainbow (3.1.1)
|
49
88
|
rake (10.5.0)
|
50
89
|
regexp_parser (2.5.0)
|
@@ -66,6 +105,7 @@ GEM
|
|
66
105
|
rubocop-ast (>= 0.4.0)
|
67
106
|
ruby-progressbar (1.11.0)
|
68
107
|
rufo (0.13.0)
|
108
|
+
thor (1.2.1)
|
69
109
|
tzinfo (2.0.5)
|
70
110
|
concurrent-ruby (~> 1.0)
|
71
111
|
unicode-display_width (2.2.0)
|
@@ -85,6 +125,7 @@ DEPENDENCIES
|
|
85
125
|
mocha
|
86
126
|
pg
|
87
127
|
que-locks!
|
128
|
+
railties
|
88
129
|
rake (~> 10.0)
|
89
130
|
rubocop
|
90
131
|
rubocop-performance
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Que::Locks
|
2
|
+
module ActiveJobExtensions
|
3
|
+
class ExclusiveJobWrapper < ::ActiveJob::QueueAdapters::QueAdapter::JobWrapper
|
4
|
+
# Opt into the locking functionality provided by que-locks
|
5
|
+
self.exclusive_execution_lock = true
|
6
|
+
end
|
7
|
+
|
8
|
+
def enqueue(job)
|
9
|
+
return super unless job.class.exclusive_execution_lock
|
10
|
+
do_enqueue job
|
11
|
+
end
|
12
|
+
|
13
|
+
def enqueue_at(job, timestamp)
|
14
|
+
return super unless job.class.exclusive_execution_lock
|
15
|
+
do_enqueue job, run_at: Time.at(timestamp)
|
16
|
+
end
|
17
|
+
|
18
|
+
def do_enqueue(job, **job_options)
|
19
|
+
job_options[:priority] = job.priority if job.respond_to? :priority
|
20
|
+
job_options[:queue] = job.queue_name if job.respond_to? :queue_name
|
21
|
+
|
22
|
+
que_job = ExclusiveJobWrapper.enqueue job.serialize, job_options: job_options
|
23
|
+
if que_job && job.respond_to?(:provider_job_id=)
|
24
|
+
job.provider_job_id = que_job.attrs["job_id"]
|
25
|
+
end
|
26
|
+
que_job
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -2,10 +2,14 @@ require "xxhash"
|
|
2
2
|
|
3
3
|
module Que
|
4
4
|
SQL[:args_already_enqueued] = %{
|
5
|
-
SELECT COUNT(*) FROM public.que_jobs WHERE job_class = $1 AND args = $2 AND finished_at IS NULL AND expired_at IS NULL;
|
5
|
+
SELECT COUNT(*) FROM public.que_jobs WHERE job_class = $1 AND args = $2 AND finished_at IS NULL AND expired_at IS NULL LIMIT 1;
|
6
6
|
}
|
7
7
|
|
8
|
-
SQL[:
|
8
|
+
SQL[:active_job_args_already_enqueued] = %{
|
9
|
+
SELECT COUNT(*) FROM public.que_jobs WHERE job_class = $1 AND args->0 @> $2 AND finished_at IS NULL AND expired_at IS NULL LIMIT 1;
|
10
|
+
}
|
11
|
+
|
12
|
+
SQL[:try_acquire_execution_lock] = %{
|
9
13
|
SELECT pg_try_advisory_lock(42, $1) AS locked
|
10
14
|
}
|
11
15
|
|
@@ -16,19 +20,25 @@ module Que
|
|
16
20
|
module Locks::ExecutionLock
|
17
21
|
class << self
|
18
22
|
def already_enqueued_job_wanting_lock?(klass, args)
|
23
|
+
query = :args_already_enqueued
|
24
|
+
if active_job_class?(klass)
|
25
|
+
args = active_jobless_args(args)
|
26
|
+
query = :active_job_args_already_enqueued
|
27
|
+
end
|
28
|
+
|
19
29
|
args_string = Que.serialize_json(args)
|
20
|
-
values = Que.execute(
|
30
|
+
values = Que.execute(query, [klass.name, args_string]).first
|
21
31
|
values[:count] != 0
|
22
32
|
end
|
23
33
|
|
24
|
-
def
|
25
|
-
|
34
|
+
def can_acquire?(klass, args)
|
35
|
+
can_acquire_key?(lock_key(klass, args))
|
26
36
|
end
|
27
37
|
|
28
|
-
def
|
38
|
+
def can_acquire_key?(key)
|
29
39
|
result = false
|
30
40
|
begin
|
31
|
-
result =
|
41
|
+
result = acquire!(key)
|
32
42
|
ensure
|
33
43
|
if result
|
34
44
|
release!(key)
|
@@ -38,17 +48,43 @@ module Que
|
|
38
48
|
end
|
39
49
|
|
40
50
|
def lock_key(klass, args)
|
51
|
+
if active_job_class?(klass)
|
52
|
+
args = active_jobless_args(args)
|
53
|
+
end
|
41
54
|
XXhash.xxh32(klass.name + ":" + Que.serialize_json(args), 42) / 2
|
42
55
|
end
|
43
56
|
|
44
|
-
def
|
45
|
-
result = Que.execute(:
|
57
|
+
def acquire!(key)
|
58
|
+
result = Que.execute(:try_acquire_execution_lock, [key]).first
|
46
59
|
result[:locked]
|
47
60
|
end
|
48
61
|
|
49
62
|
def release!(key)
|
50
63
|
Que.execute(:release_execution_lock, [key])
|
51
64
|
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
def active_job_class?(klass)
|
69
|
+
if Object.const_defined?("ActiveJob::QueueAdapters::QueAdapter::JobWrapper")
|
70
|
+
return klass.ancestors.include? ::ActiveJob::QueueAdapters::QueAdapter::JobWrapper
|
71
|
+
end
|
72
|
+
false
|
73
|
+
end
|
74
|
+
|
75
|
+
def active_jobless_args(args)
|
76
|
+
# ActiveJob handles its own arguments, and thus comes in as one argument:
|
77
|
+
# a single serialized hash representing the job. We want to use this hash
|
78
|
+
# to check and see if we already have an equivalent job queued up, but that
|
79
|
+
# requires us to toss out irrelevant ActiveJob-specific parameters that will
|
80
|
+
# throw our check off. There are enough of these
|
81
|
+
# (see ActiveJob::Core#serialize) that it's easier to maintain a whitelist;
|
82
|
+
# toss everything else.
|
83
|
+
hash = args.first
|
84
|
+
okay_keys = ["job_class", "arguments"]
|
85
|
+
# Careful, this is a shallow copy, don't actually modify that hash
|
86
|
+
hash.reject { |key| !okay_keys.include?(key) }
|
87
|
+
end
|
52
88
|
end
|
53
89
|
end
|
54
90
|
end
|
@@ -6,7 +6,7 @@ module Que::Locks
|
|
6
6
|
args << kwargs if kwargs.any?
|
7
7
|
return true unless self.exclusive_execution_lock
|
8
8
|
return false if Que::Locks::ExecutionLock.already_enqueued_job_wanting_lock?(self, args)
|
9
|
-
return Que::Locks::ExecutionLock.
|
9
|
+
return Que::Locks::ExecutionLock.can_acquire?(self, args)
|
10
10
|
end
|
11
11
|
|
12
12
|
def enqueue(*args, queue: nil, priority: nil, run_at: nil, job_class: nil, tags: nil, job_options: {}, **kwargs)
|
@@ -25,12 +25,16 @@ module Que::Locks
|
|
25
25
|
|
26
26
|
if Que::Locks::ExecutionLock.already_enqueued_job_wanting_lock?(self, args_list)
|
27
27
|
Que.log(level: :info, event: :skipped_enqueue_due_to_preemptive_lock_check, args: args_list)
|
28
|
-
|
29
|
-
|
28
|
+
# This technically breaks API compatibility with que, which always
|
29
|
+
# returns a job. It could be argued that we should return the
|
30
|
+
# already-enqueued job, but then we'd lose the ability to signal to
|
31
|
+
# the caller that a job wasn't actually enqueued. Let's see how
|
32
|
+
# far we can get with this.
|
33
|
+
return
|
30
34
|
end
|
31
|
-
else
|
32
|
-
super(*args, **forwardable_kwargs)
|
33
35
|
end
|
36
|
+
|
37
|
+
super(*args, **forwardable_kwargs)
|
34
38
|
end
|
35
39
|
end
|
36
40
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Que::Locks
|
2
|
+
class Railtie < Rails::Railtie
|
3
|
+
initializer "que-locks.patch_active_job" do
|
4
|
+
ActiveSupport.on_load(:active_job) do
|
5
|
+
class_attribute :exclusive_execution_lock
|
6
|
+
|
7
|
+
require_relative "active_job_extensions"
|
8
|
+
ActiveJob::QueueAdapters::QueAdapter.prepend(Que::Locks::ActiveJobExtensions)
|
9
|
+
ActiveJob::QueueAdapters::QueAdapter.singleton_class.prepend(Que::Locks::ActiveJobExtensions) # for rails 4
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
data/lib/que/locks/version.rb
CHANGED
data/lib/que/locks.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
+
require "que"
|
1
2
|
require "que/locks/version"
|
2
3
|
require "que/locks/execution_lock"
|
3
4
|
require "que/locks/json_extensions"
|
4
5
|
require "que/locks/job_extensions"
|
5
6
|
require "que/locks/lock_middleware"
|
7
|
+
require "que/locks/railtie" if defined?(Rails::Railtie)
|
6
8
|
|
7
9
|
module Que
|
8
10
|
module Locks
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: que-locks
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Harry Brundage
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-08-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -103,10 +103,12 @@ files:
|
|
103
103
|
- bin/setup
|
104
104
|
- docker-compose.yml
|
105
105
|
- lib/que/locks.rb
|
106
|
+
- lib/que/locks/active_job_extensions.rb
|
106
107
|
- lib/que/locks/execution_lock.rb
|
107
108
|
- lib/que/locks/job_extensions.rb
|
108
109
|
- lib/que/locks/json_extensions.rb
|
109
110
|
- lib/que/locks/lock_middleware.rb
|
111
|
+
- lib/que/locks/railtie.rb
|
110
112
|
- lib/que/locks/version.rb
|
111
113
|
- que-locks.gemspec
|
112
114
|
homepage: https://github.com/superpro-inc/que-locks
|