sidekiq-unique-jobs 6.0.8 → 6.0.9

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +10 -13
  3. data/.editorconfig +1 -1
  4. data/.gitignore +11 -22
  5. data/.mdlrc +1 -0
  6. data/.reek.yml +1 -0
  7. data/.rspec +1 -1
  8. data/.rubocop.yml +58 -30
  9. data/.travis.yml +14 -7
  10. data/.yardopts +1 -7
  11. data/CHANGELOG.md +704 -160
  12. data/Gemfile +19 -17
  13. data/Guardfile +0 -29
  14. data/README.md +77 -65
  15. data/Rakefile +22 -5
  16. data/bin/bench +5 -5
  17. data/bin/uniquejobs +2 -2
  18. data/examples/custom_queue_job.rb +1 -1
  19. data/examples/custom_queue_job_with_filter_method.rb +1 -1
  20. data/examples/custom_queue_job_with_filter_proc.rb +2 -2
  21. data/examples/my_unique_job_with_filter_method.rb +1 -1
  22. data/examples/my_unique_job_with_filter_proc.rb +1 -1
  23. data/examples/unique_job_with_filter_method.rb +1 -1
  24. data/lib/sidekiq-unique-jobs.rb +1 -1
  25. data/lib/sidekiq_unique_jobs.rb +32 -107
  26. data/lib/sidekiq_unique_jobs/cli.rb +15 -10
  27. data/lib/sidekiq_unique_jobs/client/middleware.rb +1 -1
  28. data/lib/sidekiq_unique_jobs/constants.rb +25 -20
  29. data/lib/sidekiq_unique_jobs/digests.rb +5 -4
  30. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +3 -3
  31. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +1 -1
  32. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +1 -1
  33. data/lib/sidekiq_unique_jobs/locksmith.rb +4 -4
  34. data/lib/sidekiq_unique_jobs/logging.rb +1 -1
  35. data/lib/sidekiq_unique_jobs/middleware.rb +9 -4
  36. data/lib/sidekiq_unique_jobs/normalizer.rb +1 -1
  37. data/lib/sidekiq_unique_jobs/on_conflict.rb +12 -7
  38. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
  39. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +3 -3
  40. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +1 -1
  41. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +1 -1
  42. data/lib/sidekiq_unique_jobs/scripts.rb +5 -5
  43. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +1 -1
  44. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +75 -0
  45. data/lib/sidekiq_unique_jobs/testing.rb +3 -3
  46. data/lib/sidekiq_unique_jobs/timeout.rb +1 -1
  47. data/lib/sidekiq_unique_jobs/unique_args.rb +2 -2
  48. data/lib/sidekiq_unique_jobs/util.rb +3 -3
  49. data/lib/sidekiq_unique_jobs/version.rb +1 -1
  50. data/lib/sidekiq_unique_jobs/web.rb +13 -8
  51. data/lib/sidekiq_unique_jobs/web/helpers.rb +2 -2
  52. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +4 -0
  53. data/lib/tasks/changelog.rake +23 -0
  54. data/sidekiq-unique-jobs.gemspec +48 -27
  55. data/update_docs.sh +37 -0
  56. metadata +68 -33
  57. data/.csslintrc +0 -2
  58. data/.dockerignore +0 -4
  59. data/.eslintignore +0 -1
  60. data/.eslintrc +0 -213
@@ -10,7 +10,7 @@ class MyUniqueJobWithFilterProc
10
10
  retry: true,
11
11
  unique_args: (lambda do |args|
12
12
  options = args.extract_options!
13
- [args.first, options['type']]
13
+ [args.first, options["type"]]
14
14
  end)
15
15
 
16
16
  def perform(*)
@@ -16,6 +16,6 @@ class UniqueJobWithFilterMethod
16
16
 
17
17
  def self.filtered_args(args)
18
18
  options = args.extract_options!
19
- [args.first, options['type']]
19
+ [args.first, options["type"]]
20
20
  end
21
21
  end
@@ -1,3 +1,3 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'sidekiq_unique_jobs'
3
+ require "sidekiq_unique_jobs"
@@ -1,109 +1,34 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'yaml' if RUBY_VERSION.include?('2.0.0')
4
- require 'forwardable'
5
- require 'concurrent/mutable_struct'
6
- require 'ostruct'
7
-
8
- require 'sidekiq_unique_jobs/version'
9
- require 'sidekiq_unique_jobs/constants'
10
- require 'sidekiq_unique_jobs/logging'
11
- require 'sidekiq_unique_jobs/sidekiq_worker_methods'
12
- require 'sidekiq_unique_jobs/connection'
13
- require 'sidekiq_unique_jobs/exceptions'
14
- require 'sidekiq_unique_jobs/job'
15
- require 'sidekiq_unique_jobs/util'
16
- require 'sidekiq_unique_jobs/digests'
17
- require 'sidekiq_unique_jobs/cli'
18
- require 'sidekiq_unique_jobs/core_ext'
19
- require 'sidekiq_unique_jobs/timeout'
20
- require 'sidekiq_unique_jobs/scripts'
21
- require 'sidekiq_unique_jobs/unique_args'
22
- require 'sidekiq_unique_jobs/unlockable'
23
- require 'sidekiq_unique_jobs/locksmith'
24
- require 'sidekiq_unique_jobs/lock/base_lock'
25
- require 'sidekiq_unique_jobs/lock/until_executed'
26
- require 'sidekiq_unique_jobs/lock/until_executing'
27
- require 'sidekiq_unique_jobs/lock/until_expired'
28
- require 'sidekiq_unique_jobs/lock/while_executing'
29
- require 'sidekiq_unique_jobs/lock/while_executing_reject'
30
- require 'sidekiq_unique_jobs/lock/until_and_while_executing'
31
- require 'sidekiq_unique_jobs/options_with_fallback'
32
- require 'sidekiq_unique_jobs/middleware'
33
- require 'sidekiq_unique_jobs/sidekiq_unique_ext'
34
- require 'sidekiq_unique_jobs/on_conflict'
35
-
36
- # Namespace for this gem
37
- #
38
- # Contains configuration and utility methods that belongs top level
39
- #
40
- # @author Mikael Henriksson <mikael@zoolutions.se>
41
- module SidekiqUniqueJobs
42
- include SidekiqUniqueJobs::Connection
43
-
44
- module_function
45
-
46
- Config = Concurrent::MutableStruct.new(
47
- :default_lock_timeout,
48
- :enabled,
49
- :unique_prefix,
50
- :logger,
51
- )
52
-
53
- # The current configuration (See: {.configure} on how to configure)
54
- def config
55
- # Arguments here need to match the definition of the new class (see above)
56
- @config ||= Config.new(
57
- 0,
58
- true,
59
- 'uniquejobs',
60
- Sidekiq.logger,
61
- )
62
- end
63
-
64
- # The current logger
65
- # @return [Logger] the configured logger
66
- def logger
67
- config.logger
68
- end
69
-
70
- # Set a new logger
71
- # @param [Logger] other a new logger
72
- def logger=(other)
73
- config.logger = other
74
- end
75
-
76
- # Change global configuration while yielding
77
- # @yield control to the caller
78
- def use_config(tmp_config)
79
- fail ::ArgumentError, "#{name}.#{__method__} needs a block" unless block_given?
80
-
81
- old_config = config.to_h
82
- configure(tmp_config)
83
- yield
84
- configure(old_config)
85
- end
86
-
87
- # Configure the gem
88
- #
89
- # This is usually called once at startup of an application
90
- # @param [Hash] options global gem options
91
- # @option options [Integer] :default_lock_timeout (default is 0)
92
- # @option options [true,false] :enabled (default is true)
93
- # @option options [String] :unique_prefix (default is 'uniquejobs')
94
- # @option options [Logger] :logger (default is Sidekiq.logger)
95
- # @yield control to the caller when given block
96
- def configure(options = {})
97
- if block_given?
98
- yield config
99
- else
100
- options.each do |key, val|
101
- config.send("#{key}=", val)
102
- end
103
- end
104
- end
105
-
106
- def redis_version
107
- @redis_version ||= redis { |conn| conn.info('server')['redis_version'] }
108
- end
109
- end
3
+ require "forwardable"
4
+ require "concurrent/mutable_struct"
5
+
6
+ require "sidekiq_unique_jobs/version"
7
+ require "sidekiq_unique_jobs/constants"
8
+ require "sidekiq_unique_jobs/logging"
9
+ require "sidekiq_unique_jobs/sidekiq_worker_methods"
10
+ require "sidekiq_unique_jobs/connection"
11
+ require "sidekiq_unique_jobs/exceptions"
12
+ require "sidekiq_unique_jobs/job"
13
+ require "sidekiq_unique_jobs/util"
14
+ require "sidekiq_unique_jobs/digests"
15
+ require "sidekiq_unique_jobs/cli"
16
+ require "sidekiq_unique_jobs/core_ext"
17
+ require "sidekiq_unique_jobs/timeout"
18
+ require "sidekiq_unique_jobs/scripts"
19
+ require "sidekiq_unique_jobs/unique_args"
20
+ require "sidekiq_unique_jobs/unlockable"
21
+ require "sidekiq_unique_jobs/locksmith"
22
+ require "sidekiq_unique_jobs/lock/base_lock"
23
+ require "sidekiq_unique_jobs/lock/until_executed"
24
+ require "sidekiq_unique_jobs/lock/until_executing"
25
+ require "sidekiq_unique_jobs/lock/until_expired"
26
+ require "sidekiq_unique_jobs/lock/while_executing"
27
+ require "sidekiq_unique_jobs/lock/while_executing_reject"
28
+ require "sidekiq_unique_jobs/lock/until_and_while_executing"
29
+ require "sidekiq_unique_jobs/options_with_fallback"
30
+ require "sidekiq_unique_jobs/middleware"
31
+ require "sidekiq_unique_jobs/sidekiq_unique_ext"
32
+ require "sidekiq_unique_jobs/on_conflict"
33
+
34
+ require "sidekiq_unique_jobs/sidekiq_unique_jobs"
@@ -1,24 +1,29 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'thor'
3
+ require "thor"
4
4
 
5
5
  module SidekiqUniqueJobs
6
+ #
7
+ # Command line interface for unique jobs
8
+ #
9
+ # @author Mikael Henriksson <mikael@zoolutions.se>
10
+ #
6
11
  class Cli < Thor
7
12
  def self.banner(command, _namespace = nil, _subcommand = false)
8
13
  "jobs #{@package_name} #{command.usage}"
9
14
  end
10
15
 
11
- desc 'keys PATTERN', 'list all unique keys and their expiry time'
12
- option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
13
- def keys(pattern = '*')
16
+ desc "keys PATTERN", "list all unique keys and their expiry time"
17
+ option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
18
+ def keys(pattern = "*")
14
19
  keys = Util.keys(pattern, options[:count])
15
20
  say "Found #{keys.size} keys matching '#{pattern}':"
16
21
  print_in_columns(keys.sort) if keys.any?
17
22
  end
18
23
 
19
- desc 'del PATTERN', 'deletes unique keys from redis by pattern'
20
- option :dry_run, aliases: :d, type: :boolean, desc: 'set to false to perform deletion'
21
- option :count, aliases: :c, type: :numeric, default: 1000, desc: 'The max number of keys to return'
24
+ desc "del PATTERN", "deletes unique keys from redis by pattern"
25
+ option :dry_run, aliases: :d, type: :boolean, desc: "set to false to perform deletion"
26
+ option :count, aliases: :c, type: :numeric, default: 1000, desc: "The max number of keys to return"
22
27
  def del(pattern)
23
28
  max_count = options[:count]
24
29
  if options[:dry_run]
@@ -30,7 +35,7 @@ module SidekiqUniqueJobs
30
35
  end
31
36
  end
32
37
 
33
- desc 'console', 'drop into a console with easy access to helper methods'
38
+ desc "console", "drop into a console with easy access to helper methods"
34
39
  def console
35
40
  say "Use `keys '*', 1000 to display the first 1000 unique keys matching '*'"
36
41
  say "Use `del '*', 1000, true (default) to see how many keys would be deleted for the pattern '*'"
@@ -41,10 +46,10 @@ module SidekiqUniqueJobs
41
46
 
42
47
  no_commands do
43
48
  def console_class
44
- require 'pry'
49
+ require "pry"
45
50
  Pry
46
51
  rescue LoadError
47
- require 'irb'
52
+ require "irb"
48
53
  IRB
49
54
  end
50
55
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'sidekiq_unique_jobs/server/middleware'
3
+ require "sidekiq_unique_jobs/server/middleware"
4
4
 
5
5
  module SidekiqUniqueJobs
6
6
  module Client
@@ -1,24 +1,29 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ #
4
+ # Module with constants to avoid string duplication
5
+ #
6
+ # @author Mikael Henriksson <mikael@zoolutions.se>
7
+ #
3
8
  module SidekiqUniqueJobs
4
- ARGS_KEY ||= 'args'
5
- AT_KEY ||= 'at'
6
- CLASS_KEY ||= 'class'
7
- JID_KEY ||= 'jid'
8
- LOCK_EXPIRATION_KEY ||= 'lock_expiration'
9
- LOCK_TIMEOUT_KEY ||= 'lock_timeout'
10
- LOG_DUPLICATE_KEY ||= 'log_duplicate_payload'
11
- QUEUE_KEY ||= 'queue'
12
- UNIQUE_ACROSS_QUEUES_KEY ||= 'unique_across_queues'
13
- UNIQUE_ACROSS_WORKERS_KEY ||= 'unique_across_workers'
14
- UNIQUE_ARGS_KEY ||= 'unique_args'
15
- UNIQUE_DIGEST_KEY ||= 'unique_digest'
16
- UNIQUE_KEY ||= 'unique'
17
- UNIQUE_SET ||= 'unique:keys'
18
- LOCK_KEY ||= 'lock'
19
- ON_CONFLICT_KEY ||= 'on_conflict'
20
- UNIQUE_ON_ALL_QUEUES_KEY ||= 'unique_on_all_queues' # TODO: Remove in v6.1
21
- UNIQUE_PREFIX_KEY ||= 'unique_prefix'
22
- RETRY_SET ||= 'retry'
23
- SCHEDULE_SET ||= 'schedule'
9
+ ARGS_KEY ||= "args"
10
+ AT_KEY ||= "at"
11
+ CLASS_KEY ||= "class"
12
+ JID_KEY ||= "jid"
13
+ LOCK_EXPIRATION_KEY ||= "lock_expiration"
14
+ LOCK_TIMEOUT_KEY ||= "lock_timeout"
15
+ LOG_DUPLICATE_KEY ||= "log_duplicate_payload"
16
+ QUEUE_KEY ||= "queue"
17
+ UNIQUE_ACROSS_QUEUES_KEY ||= "unique_across_queues"
18
+ UNIQUE_ACROSS_WORKERS_KEY ||= "unique_across_workers"
19
+ UNIQUE_ARGS_KEY ||= "unique_args"
20
+ UNIQUE_DIGEST_KEY ||= "unique_digest"
21
+ UNIQUE_KEY ||= "unique"
22
+ UNIQUE_SET ||= "unique:keys"
23
+ LOCK_KEY ||= "lock"
24
+ ON_CONFLICT_KEY ||= "on_conflict"
25
+ UNIQUE_ON_ALL_QUEUES_KEY ||= "unique_on_all_queues" # TODO: Remove in v6.1
26
+ UNIQUE_PREFIX_KEY ||= "unique_prefix"
27
+ RETRY_SET ||= "retry"
28
+ SCHEDULE_SET ||= "schedule"
24
29
  end
@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
6
6
  # @author Mikael Henriksson <mikael@zoolutions.se>
7
7
  module Digests
8
8
  DEFAULT_COUNT = 1_000
9
- SCAN_PATTERN = '*'
9
+ SCAN_PATTERN = "*"
10
10
  CHUNK_SIZE = 100
11
11
 
12
12
  include SidekiqUniqueJobs::Logging
@@ -25,8 +25,9 @@ module SidekiqUniqueJobs
25
25
  # Paginate unique digests
26
26
  #
27
27
  # @param [String] pattern a pattern to match with
28
- # @param [Integer] page the current cursor position
29
- # @param [Integer] count the maximum number to match
28
+ # @param [Integer] cursor the maximum number to match
29
+ # @param [Integer] page_size the current cursor position
30
+ #
30
31
  # @return [Array<String>] with unique digests
31
32
  def page(pattern: SCAN_PATTERN, cursor: 0, page_size: 100)
32
33
  redis do |conn|
@@ -57,7 +58,7 @@ module SidekiqUniqueJobs
57
58
  return delete_by_pattern(pattern, count: count) if pattern
58
59
  return delete_by_digest(digest) if digest
59
60
 
60
- raise ArgumentError, 'either digest or pattern need to be provided'
61
+ raise ArgumentError, "either digest or pattern need to be provided"
61
62
  end
62
63
 
63
64
  private
@@ -108,14 +108,14 @@ module SidekiqUniqueJobs
108
108
  def with_cleanup
109
109
  yield
110
110
  rescue Sidekiq::Shutdown
111
- log_info('Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!')
111
+ log_info("Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!")
112
112
  raise
113
113
  else
114
114
  unlock_with_callback
115
115
  end
116
116
 
117
117
  def unlock_with_callback
118
- return log_warn('might need to be unlocked manually') unless unlock
118
+ return log_warn("might need to be unlocked manually") unless unlock
119
119
 
120
120
  callback_safely
121
121
  item[JID_KEY]
@@ -124,7 +124,7 @@ module SidekiqUniqueJobs
124
124
  def callback_safely
125
125
  callback&.call
126
126
  rescue StandardError
127
- log_warn('unlocked successfully but the #after_unlock callback failed!')
127
+ log_warn("unlocked successfully but the #after_unlock callback failed!")
128
128
  raise
129
129
  end
130
130
 
@@ -8,7 +8,7 @@ module SidekiqUniqueJobs
8
8
  #
9
9
  # @author Mikael Henriksson <mikael@zoolutions.se>
10
10
  class UntilExecuted < BaseLock
11
- OK ||= 'OK'
11
+ OK ||= "OK"
12
12
 
13
13
  # Executes in the Sidekiq server process
14
14
  # @yield to the worker class perform method
@@ -11,7 +11,7 @@ module SidekiqUniqueJobs
11
11
  #
12
12
  # @author Mikael Henriksson <mikael@zoolutions.se>
13
13
  class WhileExecuting < BaseLock
14
- RUN_SUFFIX ||= ':RUN'
14
+ RUN_SUFFIX ||= ":RUN"
15
15
 
16
16
  # @param [Hash] item the Sidekiq job hash
17
17
  # @param [Proc] callback callback to call after unlock
@@ -133,19 +133,19 @@ module SidekiqUniqueJobs
133
133
  end
134
134
 
135
135
  def available_key
136
- @available_key ||= namespaced_key('AVAILABLE')
136
+ @available_key ||= namespaced_key("AVAILABLE")
137
137
  end
138
138
 
139
139
  def exists_key
140
- @exists_key ||= namespaced_key('EXISTS')
140
+ @exists_key ||= namespaced_key("EXISTS")
141
141
  end
142
142
 
143
143
  def grabbed_key
144
- @grabbed_key ||= namespaced_key('GRABBED')
144
+ @grabbed_key ||= namespaced_key("GRABBED")
145
145
  end
146
146
 
147
147
  def version_key
148
- @version_key ||= namespaced_key('VERSION')
148
+ @version_key ||= namespaced_key("VERSION")
149
149
  end
150
150
 
151
151
  def namespaced_key(variable)
@@ -51,7 +51,7 @@ module SidekiqUniqueJobs
51
51
  end
52
52
 
53
53
  def logging_context(middleware_class, job_hash)
54
- digest = job_hash['unique_digest']
54
+ digest = job_hash["unique_digest"]
55
55
  "#{middleware_class} #{"DIG-#{digest}" if digest}"
56
56
  end
57
57
  end
@@ -1,8 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'sidekiq'
3
+ require "sidekiq"
4
4
 
5
5
  module SidekiqUniqueJobs
6
+ #
7
+ # Provides the sidekiq middleware that makes the gem work
8
+ #
9
+ # @author Mikael Henriksson <mikael@zoolutions.se>
10
+ #
6
11
  module Middleware
7
12
  def self.extended(base)
8
13
  base.class_eval do
@@ -18,12 +23,12 @@ module SidekiqUniqueJobs
18
23
  def configure_server_middleware
19
24
  Sidekiq.configure_server do |config|
20
25
  config.client_middleware do |chain|
21
- require 'sidekiq_unique_jobs/client/middleware'
26
+ require "sidekiq_unique_jobs/client/middleware"
22
27
  chain.add SidekiqUniqueJobs::Client::Middleware
23
28
  end
24
29
 
25
30
  config.server_middleware do |chain|
26
- require 'sidekiq_unique_jobs/server/middleware'
31
+ require "sidekiq_unique_jobs/server/middleware"
27
32
  chain.add SidekiqUniqueJobs::Server::Middleware
28
33
  end
29
34
  end
@@ -32,7 +37,7 @@ module SidekiqUniqueJobs
32
37
  def configure_client_middleware
33
38
  Sidekiq.configure_client do |config|
34
39
  config.client_middleware do |chain|
35
- require 'sidekiq_unique_jobs/client/middleware'
40
+ require "sidekiq_unique_jobs/client/middleware"
36
41
  chain.add SidekiqUniqueJobs::Client::Middleware
37
42
  end
38
43
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'json'
3
+ require "json"
4
4
 
5
5
  module SidekiqUniqueJobs
6
6
  # Normalizes hashes by dumping them to json and loading them from json
@@ -1,14 +1,19 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative 'on_conflict/strategy'
4
- require_relative 'on_conflict/null_strategy'
5
- require_relative 'on_conflict/log'
6
- require_relative 'on_conflict/raise'
7
- require_relative 'on_conflict/reject'
8
- require_relative 'on_conflict/replace'
9
- require_relative 'on_conflict/reschedule'
3
+ require_relative "on_conflict/strategy"
4
+ require_relative "on_conflict/null_strategy"
5
+ require_relative "on_conflict/log"
6
+ require_relative "on_conflict/raise"
7
+ require_relative "on_conflict/reject"
8
+ require_relative "on_conflict/replace"
9
+ require_relative "on_conflict/reschedule"
10
10
 
11
11
  module SidekiqUniqueJobs
12
+ #
13
+ # Provides lock conflict resolutions
14
+ #
15
+ # @author Mikael Henriksson <mikael@zoolutions.se>
16
+ #
12
17
  module OnConflict
13
18
  STRATEGIES = {
14
19
  log: OnConflict::Log,