sidekiq-unique-jobs 6.0.6 → 6.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f82a371b5f835c77c75c753533472d6b7654cb5f403ebeb686502046a0884681
4
- data.tar.gz: 982ad72a4c7f7a164c9dadc7e5979b26562b53d7a934a6a04c2baf0bd35f7732
3
+ metadata.gz: 10e850f31b0ccdb3fb851cd075ab25ff6121245c7b0eace83e7075b637bbccf2
4
+ data.tar.gz: 6b6d7592acd4822c915b54c29d4fde9715709edbececdd7ff3324401ce8cfb8c
5
5
  SHA512:
6
- metadata.gz: 6e6e9740d60600bdf0ab87b1c24dca5c680f866bbdee55449dcfe2dd143956b85ecf8db33c5bbdbc1c4ec59dcd9ab8ee481af06c93b234419612005ce1628f62
7
- data.tar.gz: 58e048744c22ebf4b288863919dad091a122f9511b7d578b06327060e88e4c377542ebae00311381ec2cbda4276d8c6e08cf51033c6369532499aa2c2e49815b
6
+ metadata.gz: c68d91eaa41ab900ab2063a1a7f514283743f11546fff0ba479869c4479c8ff2ccb1b4d04636dcc6bb8f9dd8345562a09c488380dcadd5e1d1fb745eb5ec3564
7
+ data.tar.gz: a36b53f36718f17e2ab4f25d5456aae403e51a53cecdacd142c26dc28c177f147bce6c960db8b9ce31328c621ab0379251e27b0fed9b0de04d3175e23d8134e3
data/.gitignore CHANGED
@@ -24,3 +24,5 @@ rails_example/spec/examples.txt
24
24
  /.byebug_history
25
25
 
26
26
  /.yardoc/
27
+
28
+ /spec/examples.txt
data/.rubocop.yml CHANGED
@@ -9,7 +9,7 @@ AllCops:
9
9
  Exclude:
10
10
  - 'Gemfile.lock'
11
11
  - 'gemfiles/**/*'
12
- TargetRubyVersion: 2.5
12
+ TargetRubyVersion: 2.3
13
13
 
14
14
  Lint/HandleExceptions:
15
15
  Enabled: true
data/.travis.yml CHANGED
@@ -8,13 +8,10 @@ dist: trusty
8
8
  sudo: required
9
9
  language: ruby
10
10
  cache: bundler
11
+ services:
12
+ - redis-server
11
13
 
12
14
  before_install:
13
- - sudo rm -rf /etc/apt/sources.list.d/rwky-redis.list
14
- - sudo add-apt-repository ppa:chris-lea/redis-server -y
15
- - sudo apt-get update -qy
16
- - sudo apt-get install redis-server
17
- - sudo service redis-server start
18
15
  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
19
16
  - chmod +x ./cc-test-reporter
20
17
  before_script:
@@ -42,6 +39,7 @@ gemfile:
42
39
  - gemfiles/sidekiq_4.2.gemfile
43
40
  - gemfiles/sidekiq_5.0.gemfile
44
41
  - gemfiles/sidekiq_5.1.gemfile
42
+ - gemfiles/sidekiq_5.2.gemfile
45
43
 
46
44
  notifications:
47
45
  email:
data/Appraisals CHANGED
@@ -23,3 +23,7 @@ end
23
23
  appraise 'sidekiq-5.1' do
24
24
  gem 'sidekiq', '~> 5.1.0'
25
25
  end
26
+
27
+ appraise 'sidekiq-5.2' do
28
+ gem 'sidekiq', '~> 5.2.0'
29
+ end
data/CHANGELOG.md CHANGED
@@ -1,3 +1,13 @@
1
+ ## v6.0.6
2
+
3
+ - Fixes a bug with the command line utility ([#321](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/321))
4
+ - Internal refactoring to improve performance ([#318](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/318))
5
+ - Adds coverage for retrying jobs
6
+
7
+ ## v6.0.5
8
+
9
+ - Fixes a bug with keys not being deleted ([#317](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/317))
10
+
1
11
  ## v6.0.4
2
12
 
3
13
  - Prevent locks from stealing each other (#316)
data/README.md CHANGED
@@ -443,7 +443,7 @@ RSpec.describe Workers::CoolOne do
443
443
 
444
444
  it 'prevents duplicate jobs from being scheduled' do
445
445
  SidekiqUniqueJobs.use_config(enabled: true) do
446
- expect(described_class.perform_async(1)).not_to eq(nil)
446
+ expect(described_class.perform_in(3600, 1)).not_to eq(nil)
447
447
  expect(described_class.perform_async(1)).to eq(nil)
448
448
  end
449
449
  end
@@ -11,6 +11,7 @@ require 'sidekiq_unique_jobs/logging'
11
11
  require 'sidekiq_unique_jobs/sidekiq_worker_methods'
12
12
  require 'sidekiq_unique_jobs/connection'
13
13
  require 'sidekiq_unique_jobs/exceptions'
14
+ require 'sidekiq_unique_jobs/job'
14
15
  require 'sidekiq_unique_jobs/util'
15
16
  require 'sidekiq_unique_jobs/digests'
16
17
  require 'sidekiq_unique_jobs/cli'
@@ -42,8 +43,7 @@ module SidekiqUniqueJobs
42
43
 
43
44
  module_function
44
45
 
45
- Concurrent::MutableStruct.new(
46
- 'Config',
46
+ Config = Concurrent::MutableStruct.new(
47
47
  :default_lock_timeout,
48
48
  :enabled,
49
49
  :unique_prefix,
@@ -53,7 +53,7 @@ module SidekiqUniqueJobs
53
53
  # The current configuration (See: {.configure} on how to configure)
54
54
  def config
55
55
  # Arguments here need to match the definition of the new class (see above)
56
- @config ||= Concurrent::MutableStruct::Config.new(
56
+ @config ||= Config.new(
57
57
  0,
58
58
  true,
59
59
  'uniquejobs',
@@ -38,13 +38,17 @@ module SidekiqUniqueJobs
38
38
  end
39
39
 
40
40
  def locked?
41
- locked = lock.lock
42
- warn_about_duplicate unless locked
43
- locked
41
+ SidekiqUniqueJobs::Job.add_uniqueness(item)
42
+ Sidekiq::Logging.with_context(logging_context(self.class, item)) do
43
+ locked = lock.lock
44
+ warn_about_duplicate unless locked
45
+ locked
46
+ end
44
47
  end
45
48
 
46
49
  def warn_about_duplicate
47
50
  return unless log_duplicate_payload?
51
+
48
52
  log_warn "payload is not unique #{item}"
49
53
  end
50
54
  end
@@ -19,4 +19,6 @@ module SidekiqUniqueJobs
19
19
  ON_CONFLICT_KEY ||= 'on_conflict'
20
20
  UNIQUE_ON_ALL_QUEUES_KEY ||= 'unique_on_all_queues' # TODO: Remove in v6.1
21
21
  UNIQUE_PREFIX_KEY ||= 'unique_prefix'
22
+ RETRY_SET ||= 'retry'
23
+ SCHEDULE_SET ||= 'schedule'
22
24
  end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Utility class to append uniqueness to the sidekiq job hash
5
+ #
6
+ # @author Mikael Henriksson <mikael@zoolutions.se>
7
+ module Job
8
+ extend self # rubocop:disable Style/ModuleFunction
9
+
10
+ # Adds timeout, expiration, unique_args, unique_prefix and unique_digest to the sidekiq job hash
11
+ # @return [void] nothing returned here matters
12
+ def add_uniqueness(item)
13
+ add_timeout_and_expiration(item)
14
+ add_unique_args_and_digest(item)
15
+ end
16
+
17
+ private
18
+
19
+ def add_timeout_and_expiration(item)
20
+ calculator = SidekiqUniqueJobs::Timeout::Calculator.new(item)
21
+ item[LOCK_TIMEOUT_KEY] = calculator.lock_timeout
22
+ item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
23
+ end
24
+
25
+ def add_unique_args_and_digest(item)
26
+ SidekiqUniqueJobs::UniqueArgs.digest(item)
27
+ end
28
+ end
29
+ end
@@ -13,9 +13,10 @@ module SidekiqUniqueJobs
13
13
  # @param [Proc] callback the callback to use after unlock
14
14
  # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
15
15
  def initialize(item, callback, redis_pool = nil)
16
- @item = prepare_item(item)
16
+ @item = item
17
17
  @callback = callback
18
18
  @redis_pool = redis_pool
19
+ add_uniqueness_when_missing # Used to ease testing
19
20
  end
20
21
 
21
22
  # Handles locking of sidekiq jobs.
@@ -65,6 +66,14 @@ module SidekiqUniqueJobs
65
66
 
66
67
  private
67
68
 
69
+ def add_uniqueness_when_missing
70
+ return if item.key?(UNIQUE_DIGEST_KEY)
71
+
72
+ # The below should only be done to ease testing
73
+ # in production this will be done by the middleware
74
+ SidekiqUniqueJobs::Job.add_uniqueness(item)
75
+ end
76
+
68
77
  def call_strategy
69
78
  @attempt += 1
70
79
  strategy.call { lock if replace? }
@@ -99,27 +108,14 @@ module SidekiqUniqueJobs
99
108
  def with_cleanup
100
109
  yield
101
110
  rescue Sidekiq::Shutdown
102
- notify_about_manual_unlock
111
+ log_info('Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!')
103
112
  raise
104
113
  else
105
114
  unlock_with_callback
106
115
  end
107
116
 
108
- def prepare_item(item)
109
- calculator = SidekiqUniqueJobs::Timeout::Calculator.new(item)
110
- item[LOCK_TIMEOUT_KEY] = calculator.lock_timeout
111
- item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
112
- SidekiqUniqueJobs::UniqueArgs.digest(item)
113
- item
114
- end
115
-
116
- def notify_about_manual_unlock
117
- log_fatal("the unique_key: #{item[UNIQUE_DIGEST_KEY]} needs to be unlocked manually")
118
- false
119
- end
120
-
121
117
  def unlock_with_callback
122
- return notify_about_manual_unlock unless unlock
118
+ return log_warn('might need to be unlocked manually') unless unlock
123
119
 
124
120
  callback_safely
125
121
  item[JID_KEY]
@@ -128,7 +124,7 @@ module SidekiqUniqueJobs
128
124
  def callback_safely
129
125
  callback&.call
130
126
  rescue StandardError
131
- log_warn("The unique_key: #{item[UNIQUE_DIGEST_KEY]} has been unlocked but the #after_unlock callback failed!")
127
+ log_warn('unlocked successfully but the #after_unlock callback failed!')
132
128
  raise
133
129
  end
134
130
 
@@ -17,6 +17,7 @@ module SidekiqUniqueJobs
17
17
  # @yield to the worker class perform method
18
18
  def execute
19
19
  return unless locked?
20
+
20
21
  unlock
21
22
 
22
23
  runtime_lock.execute { yield }
@@ -14,6 +14,7 @@ module SidekiqUniqueJobs
14
14
  # @yield to the worker class perform method
15
15
  def execute
16
16
  return unless locked?
17
+
17
18
  with_cleanup { yield }
18
19
  end
19
20
  end
@@ -4,7 +4,7 @@ module SidekiqUniqueJobs
4
4
  class Lock
5
5
  # Locks jobs until {#execute} starts
6
6
  # - Locks on perform_in or perform_async
7
- # - Unlocks after yielding to the worker's perform method
7
+ # - Unlocks before yielding to the worker's perform method
8
8
  #
9
9
  # @author Mikael Henriksson <mikael@zoolutions.se>
10
10
  class UntilExecuting < BaseLock
@@ -21,6 +21,7 @@ module SidekiqUniqueJobs
21
21
  # @yield to the worker class perform method
22
22
  def execute
23
23
  return unless locked?
24
+
24
25
  yield
25
26
  # this lock does not handle after_unlock since we don't know when that would happen
26
27
  end
@@ -33,6 +33,7 @@ module SidekiqUniqueJobs
33
33
  # @yield to the worker class perform method
34
34
  def execute
35
35
  return strategy.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
36
+
36
37
  with_cleanup { yield }
37
38
  end
38
39
 
@@ -37,6 +37,7 @@ module SidekiqUniqueJobs
37
37
  # Deletes the lock unless it has an expiration set
38
38
  def delete
39
39
  return if expiration
40
+
40
41
  delete!
41
42
  end
42
43
 
@@ -72,6 +73,7 @@ module SidekiqUniqueJobs
72
73
  def unlock(token = nil)
73
74
  token ||= jid
74
75
  return false unless locked?(token)
76
+
75
77
  unlock!(token)
76
78
  end
77
79
 
@@ -49,5 +49,10 @@ module SidekiqUniqueJobs
49
49
  def log_fatal(message_or_exception = nil, &block)
50
50
  logger.fatal(message_or_exception, &block)
51
51
  end
52
+
53
+ def logging_context(middleware_class, job_hash)
54
+ digest = job_hash['unique_digest']
55
+ "#{middleware_class} #{"DIG-#{digest}" if digest}"
56
+ end
52
57
  end
53
58
  end
@@ -19,13 +19,18 @@ module SidekiqUniqueJobs
19
19
  # @yield to retry the lock after deleting the old one
20
20
  def call(&block)
21
21
  return unless delete_job_by_digest
22
+
22
23
  delete_lock
23
24
  block&.call
24
25
  end
25
26
 
26
27
  # Delete the job from either schedule, retry or the queue
27
28
  def delete_job_by_digest
28
- Scripts.call(:delete_job_by_digest, nil, keys: [queue, unique_digest])
29
+ Scripts.call(
30
+ :delete_job_by_digest,
31
+ nil,
32
+ keys: ["#{QUEUE_KEY}:#{queue}", SCHEDULE_SET, RETRY_SET], argv: [unique_digest],
33
+ )
29
34
  end
30
35
 
31
36
  # Delete the keys belonging to the job
@@ -11,12 +11,12 @@ module SidekiqUniqueJobs
11
11
  module OptionsWithFallback
12
12
  LOCKS = {
13
13
  until_and_while_executing: SidekiqUniqueJobs::Lock::UntilAndWhileExecuting,
14
- until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
15
- until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
16
- until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
17
- until_timeout: SidekiqUniqueJobs::Lock::UntilExpired,
18
- while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
19
- while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
14
+ until_executed: SidekiqUniqueJobs::Lock::UntilExecuted,
15
+ until_executing: SidekiqUniqueJobs::Lock::UntilExecuting,
16
+ until_expired: SidekiqUniqueJobs::Lock::UntilExpired,
17
+ until_timeout: SidekiqUniqueJobs::Lock::UntilExpired,
18
+ while_executing: SidekiqUniqueJobs::Lock::WhileExecuting,
19
+ while_executing_reject: SidekiqUniqueJobs::Lock::WhileExecutingReject,
20
20
  }.freeze
21
21
 
22
22
  def self.included(base)
@@ -6,6 +6,7 @@ module SidekiqUniqueJobs
6
6
  #
7
7
  # @author Mikael Henriksson <mikael@zoolutions.se>
8
8
  class Middleware
9
+ include Logging
9
10
  include OptionsWithFallback
10
11
 
11
12
  # Runs the server middleware
@@ -21,13 +22,18 @@ module SidekiqUniqueJobs
21
22
  @queue = queue
22
23
  return yield if unique_disabled?
23
24
 
24
- lock.execute do
25
- yield
25
+ SidekiqUniqueJobs::Job.add_uniqueness(item)
26
+ Sidekiq::Logging.with_context(logging_context(self.class, item)) do
27
+ lock.execute do
28
+ yield
29
+ end
26
30
  end
27
31
  end
28
32
 
29
- protected
33
+ private
30
34
 
35
+ # The sidekiq job hash
36
+ # @return [Hash] the Sidekiq job hash
31
37
  attr_reader :item
32
38
  end
33
39
  end
@@ -15,6 +15,7 @@ module SidekiqUniqueJobs
15
15
  # @return [Hash] of the worker class sidekiq options
16
16
  def worker_options
17
17
  return {} unless sidekiq_worker_class?
18
+
18
19
  worker_class.get_sidekiq_options.stringify_keys
19
20
  end
20
21
 
@@ -43,6 +44,7 @@ module SidekiqUniqueJobs
43
44
  # @return [Sidekiq::Worker]
44
45
  def worker_class_constantize(klazz = @worker_class)
45
46
  return klazz unless klazz.is_a?(String)
47
+
46
48
  Object.const_get(klazz)
47
49
  rescue NameError => ex
48
50
  case ex.message
@@ -25,6 +25,7 @@ module SidekiqUniqueJobs
25
25
  # @return [Integer] the number of seconds until job is scheduled
26
26
  def time_until_scheduled
27
27
  return 0 unless scheduled_at
28
+
28
29
  scheduled_at.to_i - Time.now.utc.to_i
29
30
  end
30
31
 
@@ -39,7 +40,7 @@ module SidekiqUniqueJobs
39
40
  @lock_expiration ||= begin
40
41
  expiration = item[LOCK_EXPIRATION_KEY]
41
42
  expiration ||= worker_options[LOCK_EXPIRATION_KEY]
42
- expiration && expiration + time_until_scheduled
43
+ expiration && expiration.to_i + time_until_scheduled
43
44
  end
44
45
  end
45
46
 
@@ -70,6 +70,7 @@ module SidekiqUniqueJobs
70
70
  # @return [Array] the arguments filters by the {#filtered_args} method if {#unique_args_enabled?}
71
71
  def unique_args(args)
72
72
  return filtered_args(args) if unique_args_enabled?
73
+
73
74
  args
74
75
  end
75
76
 
@@ -99,6 +100,7 @@ module SidekiqUniqueJobs
99
100
  # @return [Array] args unfiltered when neither of the above
100
101
  def filtered_args(args)
101
102
  return args if args.empty?
103
+
102
104
  json_args = Normalizer.jsonify(args)
103
105
 
104
106
  case unique_args_method
@@ -20,6 +20,7 @@ module SidekiqUniqueJobs
20
20
  # @return [Array<String>] an array with active unique keys
21
21
  def keys(pattern = SCAN_PATTERN, count = DEFAULT_COUNT)
22
22
  return redis(&:keys) if pattern.nil?
23
+
23
24
  redis { |conn| conn.scan_each(match: prefix(pattern), count: count).to_a }
24
25
  end
25
26
 
@@ -44,6 +45,7 @@ module SidekiqUniqueJobs
44
45
  # @return [Integer] the number of keys deleted
45
46
  def del(pattern = SCAN_PATTERN, count = 0)
46
47
  raise ArgumentError, 'Please provide a number of keys to delete greater than zero' if count.zero?
48
+
47
49
  pattern = suffix(pattern)
48
50
 
49
51
  log_debug { "Deleting keys by: #{pattern}" }
@@ -84,11 +86,13 @@ module SidekiqUniqueJobs
84
86
  def prefix(key)
85
87
  return key if unique_prefix.nil?
86
88
  return key if key.start_with?("#{unique_prefix}:")
89
+
87
90
  "#{unique_prefix}:#{key}"
88
91
  end
89
92
 
90
93
  def suffix(key)
91
94
  return "#{key}*" unless key.end_with?(':*')
95
+
92
96
  key
93
97
  end
94
98
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SidekiqUniqueJobs
4
- VERSION = '6.0.6'
4
+ VERSION = '6.0.7'
5
5
  end
@@ -19,6 +19,7 @@ module SidekiqUniqueJobs
19
19
 
20
20
  params.merge(options).map do |key, value|
21
21
  next unless SAFE_CPARAMS.include?(key)
22
+
22
23
  "#{key}=#{CGI.escape(value.to_s)}"
23
24
  end.compact.join('&')
24
25
  end
@@ -1,5 +1,7 @@
1
- local queue = "queue:" .. KEYS[1]
2
- local unique_digest = KEYS[2]
1
+ local queue = KEYS[1]
2
+ local schedule_set = KEYS[2]
3
+ local retry_set = KEYS[3]
4
+ local unique_digest = ARGV[1]
3
5
 
4
6
  local function delete_from_sorted_set(name, digest)
5
7
  local per = 50
@@ -49,10 +51,10 @@ if result then
49
51
  return result
50
52
  end
51
53
 
52
- result = delete_from_sorted_set('schedule', unique_digest)
54
+ result = delete_from_sorted_set(schedule_set, unique_digest)
53
55
  if result then
54
56
  return result
55
57
  end
56
58
 
57
- result = delete_from_sorted_set('retry', unique_digest)
59
+ result = delete_from_sorted_set(retry_set, unique_digest)
58
60
  return result
data/redis/unlock.lua CHANGED
@@ -10,7 +10,6 @@ local expiration = tonumber(ARGV[2])
10
10
 
11
11
  redis.call('HDEL', grabbed_key, token)
12
12
  redis.call('SREM', unique_keys, unique_digest)
13
- local available_count = redis.call('LPUSH', available_key, token)
14
13
 
15
14
  if expiration then
16
15
  redis.log(redis.LOG_DEBUG, "signal_locks.lua - expiring stale locks")
@@ -29,5 +28,5 @@ else
29
28
  redis.call('DEL', unique_digest) -- TODO: Legacy support (Remove in v6.1)
30
29
  end
31
30
 
32
- return available_count
31
+ return redis.call('LPUSH', available_key, token)
33
32
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq-unique-jobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.0.6
4
+ version: 6.0.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mikael Henriksson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-08-09 00:00:00.000000000 Z
11
+ date: 2018-11-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
@@ -273,6 +273,7 @@ files:
273
273
  - lib/sidekiq_unique_jobs/core_ext.rb
274
274
  - lib/sidekiq_unique_jobs/digests.rb
275
275
  - lib/sidekiq_unique_jobs/exceptions.rb
276
+ - lib/sidekiq_unique_jobs/job.rb
276
277
  - lib/sidekiq_unique_jobs/lock/base_lock.rb
277
278
  - lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb
278
279
  - lib/sidekiq_unique_jobs/lock/until_executed.rb