sidekiq-unique-jobs 6.0.8 → 6.0.9
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.codeclimate.yml +10 -13
- data/.editorconfig +1 -1
- data/.gitignore +11 -22
- data/.mdlrc +1 -0
- data/.reek.yml +1 -0
- data/.rspec +1 -1
- data/.rubocop.yml +58 -30
- data/.travis.yml +14 -7
- data/.yardopts +1 -7
- data/CHANGELOG.md +704 -160
- data/Gemfile +19 -17
- data/Guardfile +0 -29
- data/README.md +77 -65
- data/Rakefile +22 -5
- data/bin/bench +5 -5
- data/bin/uniquejobs +2 -2
- data/examples/custom_queue_job.rb +1 -1
- data/examples/custom_queue_job_with_filter_method.rb +1 -1
- data/examples/custom_queue_job_with_filter_proc.rb +2 -2
- data/examples/my_unique_job_with_filter_method.rb +1 -1
- data/examples/my_unique_job_with_filter_proc.rb +1 -1
- data/examples/unique_job_with_filter_method.rb +1 -1
- data/lib/sidekiq-unique-jobs.rb +1 -1
- data/lib/sidekiq_unique_jobs.rb +32 -107
- data/lib/sidekiq_unique_jobs/cli.rb +15 -10
- data/lib/sidekiq_unique_jobs/client/middleware.rb +1 -1
- data/lib/sidekiq_unique_jobs/constants.rb +25 -20
- data/lib/sidekiq_unique_jobs/digests.rb +5 -4
- data/lib/sidekiq_unique_jobs/lock/base_lock.rb +3 -3
- data/lib/sidekiq_unique_jobs/lock/until_executed.rb +1 -1
- data/lib/sidekiq_unique_jobs/lock/while_executing.rb +1 -1
- data/lib/sidekiq_unique_jobs/locksmith.rb +4 -4
- data/lib/sidekiq_unique_jobs/logging.rb +1 -1
- data/lib/sidekiq_unique_jobs/middleware.rb +9 -4
- data/lib/sidekiq_unique_jobs/normalizer.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict.rb +12 -7
- data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
- data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +3 -3
- data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +1 -1
- data/lib/sidekiq_unique_jobs/options_with_fallback.rb +1 -1
- data/lib/sidekiq_unique_jobs/scripts.rb +5 -5
- data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +1 -1
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +75 -0
- data/lib/sidekiq_unique_jobs/testing.rb +3 -3
- data/lib/sidekiq_unique_jobs/timeout.rb +1 -1
- data/lib/sidekiq_unique_jobs/unique_args.rb +2 -2
- data/lib/sidekiq_unique_jobs/util.rb +3 -3
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- data/lib/sidekiq_unique_jobs/web.rb +13 -8
- data/lib/sidekiq_unique_jobs/web/helpers.rb +2 -2
- data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +4 -0
- data/lib/tasks/changelog.rake +23 -0
- data/sidekiq-unique-jobs.gemspec +48 -27
- data/update_docs.sh +37 -0
- metadata +68 -33
- data/.csslintrc +0 -2
- data/.dockerignore +0 -4
- data/.eslintignore +0 -1
- data/.eslintrc +0 -213
@@ -53,9 +53,9 @@ module SidekiqUniqueJobs
|
|
53
53
|
def push_to_deadset
|
54
54
|
Sidekiq.redis do |conn|
|
55
55
|
conn.multi do
|
56
|
-
conn.zadd(
|
57
|
-
conn.zremrangebyscore(
|
58
|
-
conn.zremrangebyrank(
|
56
|
+
conn.zadd("dead", current_time, payload)
|
57
|
+
conn.zremrangebyscore("dead", "-inf", current_time - Sidekiq::DeadSet.timeout)
|
58
|
+
conn.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
|
59
59
|
end
|
60
60
|
end
|
61
61
|
end
|
@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
|
|
21
21
|
# Use strategy on conflict
|
22
22
|
# @raise [NotImplementedError] needs to be implemented in child class
|
23
23
|
def call
|
24
|
-
|
24
|
+
raise NotImplementedError, "needs to be implemented in child class"
|
25
25
|
end
|
26
26
|
|
27
27
|
def replace?
|
@@ -1,15 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "pathname"
|
4
|
+
require "digest/sha1"
|
5
|
+
require "concurrent/map"
|
6
6
|
|
7
7
|
module SidekiqUniqueJobs
|
8
8
|
# Interface to dealing with .lua files
|
9
9
|
#
|
10
10
|
# @author Mikael Henriksson <mikael@zoolutions.se>
|
11
11
|
module Scripts
|
12
|
-
LUA_PATHNAME ||= Pathname.new(__FILE__).dirname.join(
|
12
|
+
LUA_PATHNAME ||= Pathname.new(__FILE__).dirname.join("../../redis").freeze
|
13
13
|
SCRIPT_SHAS ||= Concurrent::Map.new
|
14
14
|
|
15
15
|
include SidekiqUniqueJobs::Connection
|
@@ -64,7 +64,7 @@ module SidekiqUniqueJobs
|
|
64
64
|
# @param [Symbol] file_name the name of the lua script
|
65
65
|
# @raise [ScriptError] when the error isn't handled
|
66
66
|
def handle_error(ex, file_name)
|
67
|
-
if ex.message ==
|
67
|
+
if ex.message == "NOSCRIPT No matching script. Please use EVAL."
|
68
68
|
SCRIPT_SHAS.delete(file_name)
|
69
69
|
return yield if block_given?
|
70
70
|
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
#
|
4
|
+
# Contains configuration and utility methods that belongs top level
|
5
|
+
#
|
6
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
7
|
+
module SidekiqUniqueJobs
|
8
|
+
include SidekiqUniqueJobs::Connection
|
9
|
+
|
10
|
+
module_function
|
11
|
+
|
12
|
+
Config = Concurrent::MutableStruct.new(
|
13
|
+
:default_lock_timeout,
|
14
|
+
:enabled,
|
15
|
+
:unique_prefix,
|
16
|
+
:logger,
|
17
|
+
)
|
18
|
+
|
19
|
+
# The current configuration (See: {.configure} on how to configure)
|
20
|
+
def config
|
21
|
+
# Arguments here need to match the definition of the new class (see above)
|
22
|
+
@config ||= Config.new(
|
23
|
+
0,
|
24
|
+
true,
|
25
|
+
"uniquejobs",
|
26
|
+
Sidekiq.logger,
|
27
|
+
)
|
28
|
+
end
|
29
|
+
|
30
|
+
# The current logger
|
31
|
+
# @return [Logger] the configured logger
|
32
|
+
def logger
|
33
|
+
config.logger
|
34
|
+
end
|
35
|
+
|
36
|
+
# Set a new logger
|
37
|
+
# @param [Logger] other a new logger
|
38
|
+
def logger=(other)
|
39
|
+
config.logger = other
|
40
|
+
end
|
41
|
+
|
42
|
+
# Change global configuration while yielding
|
43
|
+
# @yield control to the caller
|
44
|
+
def use_config(tmp_config)
|
45
|
+
raise ::ArgumentError, "#{name}.#{__method__} needs a block" unless block_given?
|
46
|
+
|
47
|
+
old_config = config.to_h
|
48
|
+
configure(tmp_config)
|
49
|
+
yield
|
50
|
+
configure(old_config)
|
51
|
+
end
|
52
|
+
|
53
|
+
# Configure the gem
|
54
|
+
#
|
55
|
+
# This is usually called once at startup of an application
|
56
|
+
# @param [Hash] options global gem options
|
57
|
+
# @option options [Integer] :default_lock_timeout (default is 0)
|
58
|
+
# @option options [true,false] :enabled (default is true)
|
59
|
+
# @option options [String] :unique_prefix (default is 'uniquejobs')
|
60
|
+
# @option options [Logger] :logger (default is Sidekiq.logger)
|
61
|
+
# @yield control to the caller when given block
|
62
|
+
def configure(options = {})
|
63
|
+
if block_given?
|
64
|
+
yield config
|
65
|
+
else
|
66
|
+
options.each do |key, val|
|
67
|
+
config.send("#{key}=", val)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def redis_version
|
73
|
+
@redis_version ||= redis { |conn| conn.info("server")["redis_version"] }
|
74
|
+
end
|
75
|
+
end
|
@@ -2,8 +2,8 @@
|
|
2
2
|
|
3
3
|
# :nocov:
|
4
4
|
|
5
|
-
require
|
6
|
-
require
|
5
|
+
require "sidekiq"
|
6
|
+
require "sidekiq/testing"
|
7
7
|
|
8
8
|
module Sidekiq
|
9
9
|
def self.use_options(tmp_config = {})
|
@@ -52,7 +52,7 @@ module Sidekiq
|
|
52
52
|
module Testing
|
53
53
|
def clear_all_ext
|
54
54
|
clear_all_orig
|
55
|
-
SidekiqUniqueJobs::Util.del(
|
55
|
+
SidekiqUniqueJobs::Util.del("*", 1000)
|
56
56
|
end
|
57
57
|
end
|
58
58
|
end
|
@@ -7,7 +7,7 @@ module SidekiqUniqueJobs
|
|
7
7
|
# @author Mikael Henriksson <mikael@zoolutions.se>
|
8
8
|
module Util
|
9
9
|
DEFAULT_COUNT = 1_000
|
10
|
-
SCAN_PATTERN =
|
10
|
+
SCAN_PATTERN = "*"
|
11
11
|
|
12
12
|
include SidekiqUniqueJobs::Logging
|
13
13
|
include SidekiqUniqueJobs::Connection
|
@@ -44,7 +44,7 @@ module SidekiqUniqueJobs
|
|
44
44
|
# @param [Integer] count the maximum number of keys to delete
|
45
45
|
# @return [Integer] the number of keys deleted
|
46
46
|
def del(pattern = SCAN_PATTERN, count = 0)
|
47
|
-
raise ArgumentError,
|
47
|
+
raise ArgumentError, "Please provide a number of keys to delete greater than zero" if count.zero?
|
48
48
|
|
49
49
|
pattern = suffix(pattern)
|
50
50
|
|
@@ -91,7 +91,7 @@ module SidekiqUniqueJobs
|
|
91
91
|
end
|
92
92
|
|
93
93
|
def suffix(key)
|
94
|
-
return "#{key}*" unless key.end_with?(
|
94
|
+
return "#{key}*" unless key.end_with?(":*")
|
95
95
|
|
96
96
|
key
|
97
97
|
end
|
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
begin
|
4
|
-
require
|
4
|
+
require "sidekiq/web"
|
5
5
|
rescue LoadError # rubocop:disable Lint/HandleExceptions
|
6
6
|
# client-only usage
|
7
7
|
end
|
8
8
|
|
9
|
-
require_relative
|
9
|
+
require_relative "web/helpers"
|
10
10
|
|
11
11
|
module SidekiqUniqueJobs
|
12
12
|
# Utility module to help manage unique keys in redis.
|
@@ -19,9 +19,9 @@ module SidekiqUniqueJobs
|
|
19
19
|
include Web::Helpers
|
20
20
|
end
|
21
21
|
|
22
|
-
app.get
|
23
|
-
@filter = params[:filter] ||
|
24
|
-
@filter =
|
22
|
+
app.get "/unique_digests" do
|
23
|
+
@filter = params[:filter] || "*"
|
24
|
+
@filter = "*" if @filter == ""
|
25
25
|
@count = (params[:count] || 100).to_i
|
26
26
|
@current_cursor = params[:cursor]
|
27
27
|
@prev_cursor = params[:prev_cursor]
|
@@ -31,14 +31,19 @@ module SidekiqUniqueJobs
|
|
31
31
|
erb(unique_template(:unique_digests))
|
32
32
|
end
|
33
33
|
|
34
|
-
app.get
|
34
|
+
app.get "/unique_digests/delete_all" do
|
35
|
+
Digests.del(pattern: "*", count: Digests.count)
|
36
|
+
redirect_to :unique_digests
|
37
|
+
end
|
38
|
+
|
39
|
+
app.get "/unique_digests/:digest" do
|
35
40
|
@digest = params[:digest]
|
36
41
|
@unique_keys = Util.keys("#{@digest}*", 1000)
|
37
42
|
|
38
43
|
erb(unique_template(:unique_digest))
|
39
44
|
end
|
40
45
|
|
41
|
-
app.get
|
46
|
+
app.get "/unique_digests/:digest/delete" do
|
42
47
|
Digests.del(digest: params[:digest])
|
43
48
|
redirect_to :unique_digests
|
44
49
|
end
|
@@ -48,6 +53,6 @@ end
|
|
48
53
|
|
49
54
|
if defined?(Sidekiq::Web)
|
50
55
|
Sidekiq::Web.register SidekiqUniqueJobs::Web
|
51
|
-
Sidekiq::Web.tabs[
|
56
|
+
Sidekiq::Web.tabs["Unique Digests"] = "unique_digests"
|
52
57
|
# Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), 'locales')
|
53
58
|
end
|
@@ -3,7 +3,7 @@
|
|
3
3
|
module SidekiqUniqueJobs
|
4
4
|
module Web
|
5
5
|
module Helpers
|
6
|
-
VIEW_PATH = File.expand_path(
|
6
|
+
VIEW_PATH = File.expand_path("../web/views", __dir__)
|
7
7
|
|
8
8
|
def unique_template(name)
|
9
9
|
File.open(File.join(VIEW_PATH, "#{name}.erb")).read
|
@@ -21,7 +21,7 @@ module SidekiqUniqueJobs
|
|
21
21
|
next unless SAFE_CPARAMS.include?(key)
|
22
22
|
|
23
23
|
"#{key}=#{CGI.escape(value.to_s)}"
|
24
|
-
end.compact.join(
|
24
|
+
end.compact.join("&")
|
25
25
|
end
|
26
26
|
|
27
27
|
def redirect_to(subpath)
|
@@ -38,5 +38,9 @@
|
|
38
38
|
</tr>
|
39
39
|
<% end %>
|
40
40
|
</table>
|
41
|
+
|
42
|
+
<form action="<%= root_path %>unique_digests/delete_all" method="get">
|
43
|
+
<input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
|
44
|
+
</form>
|
41
45
|
</div>
|
42
46
|
<% end %>
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
desc "Generate a Changelog"
|
4
|
+
task :changelog do
|
5
|
+
# rubocop:disable Style/MutableConstant
|
6
|
+
CHANGELOG_CMD ||= %w[
|
7
|
+
github_changelog_generator
|
8
|
+
-u
|
9
|
+
mhenrixon
|
10
|
+
-p
|
11
|
+
sidekiq-unique-jobs
|
12
|
+
--no-verbose
|
13
|
+
--token
|
14
|
+
]
|
15
|
+
ADD_CHANGELOG_CMD ||= "git add --all"
|
16
|
+
COMMIT_CHANGELOG_CMD ||= "git commit -a -m 'Update changelog'"
|
17
|
+
# rubocop:enable Style/MutableConstant
|
18
|
+
|
19
|
+
sh("git checkout master")
|
20
|
+
sh(*CHANGELOG_CMD.push(ENV["CHANGELOG_GITHUB_TOKEN"]))
|
21
|
+
sh(ADD_CHANGELOG_CMD)
|
22
|
+
sh(COMMIT_CHANGELOG_CMD)
|
23
|
+
end
|
data/sidekiq-unique-jobs.gemspec
CHANGED
@@ -1,42 +1,63 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
lib = File.expand_path("
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
4
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
5
|
|
6
|
-
require
|
6
|
+
require "sidekiq_unique_jobs/version"
|
7
7
|
|
8
8
|
Gem::Specification.new do |spec|
|
9
|
-
spec.name
|
10
|
-
spec.version
|
11
|
-
spec.authors
|
12
|
-
spec.email
|
13
|
-
|
14
|
-
spec.
|
15
|
-
spec.
|
16
|
-
|
17
|
-
|
9
|
+
spec.name = "sidekiq-unique-jobs"
|
10
|
+
spec.version = SidekiqUniqueJobs::VERSION
|
11
|
+
spec.authors = ["Mikael Henriksson"]
|
12
|
+
spec.email = ["mikael@zoolutions.se"]
|
13
|
+
spec.homepage = "https://mhenrixon.github.io/sidekiq-unique-jobs"
|
14
|
+
spec.license = "MIT"
|
15
|
+
spec.summary = <<~SUMMARY
|
16
|
+
Sidekiq middleware that prevents duplicates jobs
|
17
|
+
SUMMARY
|
18
|
+
spec.description = <<~DESCRIPTION
|
19
|
+
Prevents simultaneous Sidekiq jobs with the same unique arguments to run.
|
20
|
+
Highly configurable to suite your specific needs.
|
21
|
+
DESCRIPTION
|
22
|
+
|
23
|
+
if spec.respond_to?(:metadata)
|
24
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
25
|
+
spec.metadata["bug_tracker_uri"] = "https://github.com/mhenrixon/sidekiq-unique-jobs/issues"
|
26
|
+
spec.metadata["documentation_uri"] = "https://mhenrixon.github.io/sidekiq-unique-jobs"
|
27
|
+
spec.metadata["source_code_uri"] = "https://github.com/mhenrixon/sidekiq-unique-jobs"
|
28
|
+
spec.metadata["changelog_uri"] = "https://github.com/mhenrixon/sidekiq-unique-jobs/CHANGELOG.md"
|
29
|
+
else
|
30
|
+
raise "RubyGems 2.0 or newer is required to protect against " \
|
31
|
+
"public gem pushes."
|
32
|
+
end
|
18
33
|
|
19
|
-
spec.bindir =
|
34
|
+
spec.bindir = "bin"
|
20
35
|
spec.executables = %w[uniquejobs]
|
21
36
|
|
22
|
-
spec.files = Dir.chdir(File.expand_path(
|
37
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
23
38
|
`git ls-files -z`.split("\x0").reject do |file|
|
24
39
|
file.match(%r{^(test|spec|features|gemfiles|pkg|rails_example|tmp)/})
|
25
40
|
end
|
26
41
|
end
|
27
42
|
|
28
|
-
spec.require_paths = [
|
29
|
-
spec.add_dependency
|
30
|
-
spec.add_dependency
|
31
|
-
spec.add_dependency
|
32
|
-
|
33
|
-
spec.add_development_dependency
|
34
|
-
spec.add_development_dependency
|
35
|
-
spec.add_development_dependency
|
36
|
-
spec.add_development_dependency
|
37
|
-
spec.add_development_dependency
|
38
|
-
spec.add_development_dependency
|
39
|
-
|
40
|
-
|
41
|
-
spec.add_development_dependency
|
43
|
+
spec.require_paths = ["lib"]
|
44
|
+
spec.add_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.5"
|
45
|
+
spec.add_dependency "sidekiq", ">= 4.0", "< 6.0"
|
46
|
+
spec.add_dependency "thor", "~> 0"
|
47
|
+
|
48
|
+
spec.add_development_dependency "bundler", ">= 2.0"
|
49
|
+
spec.add_development_dependency "rack-test"
|
50
|
+
spec.add_development_dependency "rake", "~> 12.3"
|
51
|
+
spec.add_development_dependency "rspec", "~> 3.7"
|
52
|
+
spec.add_development_dependency "sinatra"
|
53
|
+
spec.add_development_dependency "timecop", "~> 0.9"
|
54
|
+
|
55
|
+
# ===== Documentation =====
|
56
|
+
spec.add_development_dependency "yard", "~> 0.9.18"
|
57
|
+
spec.add_development_dependency "redcarpet", "~> 3.4"
|
58
|
+
spec.add_development_dependency "github-markup", "~> 3.0"
|
59
|
+
spec.add_development_dependency "github_changelog_generator", "~> 1.14"
|
60
|
+
|
61
|
+
# ===== Release Management =====
|
62
|
+
spec.add_development_dependency "gem-release", ">= 2.0"
|
42
63
|
end
|
data/update_docs.sh
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
git checkout master
|
4
|
+
git fetch
|
5
|
+
stash_created=0
|
6
|
+
|
7
|
+
if [[ "$(git diff --stat)" != "" ]]; then
|
8
|
+
stash_created=1
|
9
|
+
git stash push -u -a -m "Before updating docs"
|
10
|
+
fi;
|
11
|
+
|
12
|
+
git pull --rebase
|
13
|
+
|
14
|
+
rake yard
|
15
|
+
|
16
|
+
git checkout gh-pages
|
17
|
+
|
18
|
+
if [[ "$(git branch | grep \* | cut -d ' ' -f2)" != "gh-pages" ]]; then
|
19
|
+
git checkout -b gh-pages
|
20
|
+
fi;
|
21
|
+
|
22
|
+
echo "Cleaning up current documentation"
|
23
|
+
find . ! -path '*/.git*' ! -path '*/doc*' ! -path '*/update_docs.sh*' ! -path '*/_config.yml*' ! -path '*/_index.html*' ! -path '.' | xargs rm -rf
|
24
|
+
|
25
|
+
echo "Copying new documentation"
|
26
|
+
mv doc/* ./
|
27
|
+
|
28
|
+
echo "Sending new documentation to github"
|
29
|
+
git add --all
|
30
|
+
git commit -a -m 'Update documentation'
|
31
|
+
git push --set-upstream origin gh-pages --force
|
32
|
+
|
33
|
+
if [[ $stash_created == 1 ]]; then
|
34
|
+
git stash pop
|
35
|
+
fi;
|
36
|
+
|
37
|
+
git checkout master
|