sidekiq-unique-jobs 8.0.3 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +157 -1
  3. data/README.md +2 -4
  4. data/lib/sidekiq_unique_jobs/batch_delete.rb +1 -1
  5. data/lib/sidekiq_unique_jobs/changelog.rb +1 -1
  6. data/lib/sidekiq_unique_jobs/config.rb +4 -4
  7. data/lib/sidekiq_unique_jobs/constants.rb +1 -0
  8. data/lib/sidekiq_unique_jobs/digests.rb +46 -13
  9. data/lib/sidekiq_unique_jobs/job.rb +4 -4
  10. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +1 -1
  11. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +3 -0
  12. data/lib/sidekiq_unique_jobs/lock.rb +14 -11
  13. data/lib/sidekiq_unique_jobs/lock_config.rb +10 -4
  14. data/lib/sidekiq_unique_jobs/locksmith.rb +10 -5
  15. data/lib/sidekiq_unique_jobs/lua/delete.lua +6 -5
  16. data/lib/sidekiq_unique_jobs/lua/lock.lua +16 -7
  17. data/lib/sidekiq_unique_jobs/lua/queue.lua +9 -8
  18. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +10 -10
  19. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +20 -10
  20. data/lib/sidekiq_unique_jobs/lua/unlock.lua +26 -18
  21. data/lib/sidekiq_unique_jobs/middleware/client.rb +1 -1
  22. data/lib/sidekiq_unique_jobs/middleware/server.rb +1 -1
  23. data/lib/sidekiq_unique_jobs/middleware.rb +5 -5
  24. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +1 -1
  25. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +5 -1
  26. data/lib/sidekiq_unique_jobs/orphans/manager.rb +8 -7
  27. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +2 -1
  28. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +26 -7
  29. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +12 -3
  30. data/lib/sidekiq_unique_jobs/script/caller.rb +1 -1
  31. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  32. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  33. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  34. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  35. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  36. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  37. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  38. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  39. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  40. data/lib/sidekiq_unique_jobs/script.rb +32 -1
  41. data/lib/sidekiq_unique_jobs/server.rb +2 -0
  42. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +13 -35
  43. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +4 -4
  44. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +3 -3
  45. data/lib/sidekiq_unique_jobs/testing.rb +4 -4
  46. data/lib/sidekiq_unique_jobs/version.rb +1 -1
  47. data/lib/sidekiq_unique_jobs/web/helpers.rb +3 -3
  48. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +1 -1
  49. data/lib/sidekiq_unique_jobs/web/views/lock.erb +2 -2
  50. data/lib/sidekiq_unique_jobs/web/views/locks.erb +1 -1
  51. data/lib/sidekiq_unique_jobs/web.rb +18 -17
  52. data/lib/sidekiq_unique_jobs.rb +5 -4
  53. metadata +12 -23
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2462acb2448a07c45eb8a1789239f859a09e079cff316fe61dc205d9ea8856a4
4
- data.tar.gz: 57449a594cd034c79001815883806098e04a92869c41cbe6044c90ab6f1fa3ae
3
+ metadata.gz: 8cd95f03398a1663b2afbba47f61736cfd3a0ab91fcc0d5b90d353d4f283db07
4
+ data.tar.gz: ced0a60e5fad52b5f2a66dcc29b10a221c6d39f4c4089d812327922d3b6b4e8b
5
5
  SHA512:
6
- metadata.gz: b200d3544047b6080147b19b71ffd83a0ac42087ed29f1d84fb35b226b1418f50e37c94b309a946504c32d26b11b5fd173ade85e77c568931d33b948f1c656f9
7
- data.tar.gz: 9d12cbf3317acd4e5bd6d934616622e5ac3336ecba4e51b2b29fbf6782deab2b1af291af1471e16da7271fef42b33804b7a3c7f3475016de88bd75e2d0668684
6
+ metadata.gz: d72a8b181407ccc09e4f11c9d8f0860df65f80373c2064ec0f7d743d2bde4273cde1e117f2983b8f71b3a2c9f17c8b9ab8808395991030617bb6b380073457be
7
+ data.tar.gz: 2908759b402bff47b1ace5c9ab666a10779085c89b90c6261b4c935fde97cbf0f8291c8bf2df48e0b46c302eaa66f169785fdf0503ad97fcf0d43a0e0fd1b47d
data/CHANGELOG.md CHANGED
@@ -1,5 +1,162 @@
1
1
  # Changelog
2
2
 
3
+ ## [v8.0.9](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.9) (2024-02-12)
4
+
5
+ [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v8.0.8...v8.0.9)
6
+
7
+ **Fixed bugs:**
8
+
9
+ - note: The RCE vulnerability was a false alarm, `sidekiq-unique-jobs` was not vulnerable to RCE. You can find additional information in the PR linked below.
10
+ - fix\(rce\): prevent remot code execution [\#833](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/833) ([mhenrixon](https://github.com/mhenrixon))
11
+
12
+ ## [v8.0.8](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.8) (2024-02-12)
13
+
14
+ [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v8.0.7...v8.0.8)
15
+
16
+ **Implemented enhancements:**
17
+
18
+ - fix: ensure a new lock isn't conflicting with itself [\#830](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/830) ([mhenrixon](https://github.com/mhenrixon))
19
+
20
+ **Fixed bugs:**
21
+
22
+ - until\_and\_while\_executing not entering perform method on initial run [\#824](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/824)
23
+ - fix\(digest\): write digest on middleware call [\#774](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/774) ([mhenrixon](https://github.com/mhenrixon))
24
+
25
+ **Closed issues:**
26
+
27
+ - incompatibility with sidekiq-failures [\#790](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/790)
28
+ - Jobs queued during existing job inherit lock digest [\#766](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/766)
29
+
30
+ ## [v8.0.7](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.7) (2024-02-05)
31
+
32
+ [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v8.0.6...v8.0.7)
33
+
34
+ **Implemented enhancements:**
35
+
36
+ - chore\(ci\): add test coverage from bropoplpush [\#828](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/828) ([mhenrixon](https://github.com/mhenrixon))
37
+
38
+ **Fixed bugs:**
39
+
40
+ - fix\(xss\): sanitize parameters [\#829](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/829) ([mhenrixon](https://github.com/mhenrixon))
41
+
42
+ **Closed issues:**
43
+
44
+ - No 'Changelog' link is being displayed on https://rubygems.org/gems/sidekiq-unique-jobs [\#825](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/825)
45
+ - Using rspec matcher and getting: `NameError: uninitialized constant SidekiqUniqueJobs::Lock::BaseLock::Validator` [\#741](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/741)
46
+
47
+ **Merged pull requests:**
48
+
49
+ - Fix Testing Instructions [\#827](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/827) ([jherdman](https://github.com/jherdman))
50
+
51
+ ## [v8.0.6](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.6) (2024-01-24)
52
+
53
+ [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v8.0.5...v8.0.6)
54
+
55
+ **Implemented enhancements:**
56
+
57
+ - Returning same job id [\#814](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/814)
58
+ - fix: various minor issues [\#826](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/826) ([mhenrixon](https://github.com/mhenrixon))
59
+
60
+ **Closed issues:**
61
+
62
+ - should respond to `has_valid_sidekiq_options?` [\#822](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/822)
63
+ - Reaper manager registration is subject to race conditions [\#801](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/801)
64
+ - `while_executing` with `on_conflict: :reschedule` Reschedule job after job execution [\#800](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/800)
65
+ - Large retry queue causes reaper to run too slow [\#759](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/759)
66
+
67
+ **Merged pull requests:**
68
+
69
+ - fix: skip unless reaper was registered [\#820](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/820) ([mhenrixon](https://github.com/mhenrixon))
70
+
71
+ ## [v8.0.5](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.5) (2023-11-11)
72
+
73
+ [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v8.0.4...v8.0.5)
74
+
75
+ **Merged pull requests:**
76
+
77
+ - Bump @babel/traverse from 7.22.8 to 7.23.3 in /myapp [\#819](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/819) ([dependabot[bot]](https://github.com/apps/dependabot))
78
+ - Bump postcss from 8.4.21 to 8.4.31 in /myapp [\#811](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/811) ([dependabot[bot]](https://github.com/apps/dependabot))
79
+ - fix: `while_executing` should not invoke conflict strategy when the job was successfully executed \[v8\] [\#810](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/810) ([cuzik](https://github.com/cuzik))
80
+ - Bump actions/checkout from 3 to 4 [\#808](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/808) ([dependabot[bot]](https://github.com/apps/dependabot))
81
+ - Bump semver from 6.3.0 to 6.3.1 in /myapp [\#798](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/798) ([dependabot[bot]](https://github.com/apps/dependabot))
82
+ - Because `replace` is a client strategy, it should only remove client locks aka queue locks. [\#778](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/778) ([bigzed](https://github.com/bigzed))
83
+
84
+ ## [v8.0.4](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.4) (2023-11-11)
85
+
86
+ [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.31...v8.0.4)
87
+
88
+ **Implemented enhancements:**
89
+
90
+ - fix: ignore reek for now [\#818](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/818) ([mhenrixon](https://github.com/mhenrixon))
91
+ - fix\(ci\): allow tests to run in docker [\#805](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/805) ([Earlopain](https://github.com/Earlopain))
92
+ - chore\(deps\): bump'n lint'n such [\#797](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/797) ([mhenrixon](https://github.com/mhenrixon))
93
+
94
+ **Fixed bugs:**
95
+
96
+ - Sidekiq 7: Unsupported command argument type: TrueClass [\#816](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/816)
97
+ - fix\(web\): don't show bogus lock digests [\#804](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/804) ([Earlopain](https://github.com/Earlopain))
98
+
99
+ **Closed issues:**
100
+
101
+ - register\_reaper\_process nx: true crash Sidekiq on startup [\#817](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/817)
102
+
103
+ **Merged pull requests:**
104
+
105
+ - fix: Sidekiq 7.2 throws TypeError: Unsupported command argument type: TrueClass when using byscore: true [\#815](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/815) ([Amnesthesia](https://github.com/Amnesthesia))
106
+ - Add sidekiq\_option for on-conflict-reschedule perform\_in time [\#813](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/813) ([mgmarlow](https://github.com/mgmarlow))
107
+ - Handle strategy fallbacks properly [\#809](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/809) ([matejrisek](https://github.com/matejrisek))
108
+ - Fix CI status badge [\#802](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/802) ([tagliala](https://github.com/tagliala))
109
+
110
+ ## [v7.1.31](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.31) (2023-08-28)
111
+
112
+ [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.30...v7.1.31)
113
+
114
+ **Closed issues:**
115
+
116
+ - Should client middleware also be added to the Sidekiq server config? [\#803](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/803)
117
+ - 8.0.1 Time on locks & changelog UI is incorrect/wrong [\#761](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/761)
118
+ - Job executed twice when reaper runs [\#738](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/738)
119
+
120
+ ## [v7.1.30](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.30) (2023-07-17)
121
+
122
+ [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v8.0.3...v7.1.30)
123
+
124
+ **Closed issues:**
125
+
126
+ - Release new gem version with "fix replace deprecated rpoplpush w/lmove" [\#795](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/795)
127
+ - Redis has deprecated the `rpoplpush`command [\#793](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/793)
128
+ - Deprecation Warning: Use of rpoplpush command in Redis [\#792](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/792)
129
+ - Logging::Middleware.context can include empty string as key when lock is not set in job hash [\#785](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/785)
130
+
131
+ ## [v8.0.3](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.3) (2023-05-27)
132
+
133
+ [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v8.0.2...v8.0.3)
134
+
135
+ **Implemented enhancements:**
136
+
137
+ - chore\(ci\): improve matrix [\#775](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/775) ([mhenrixon](https://github.com/mhenrixon))
138
+ - chore\(ci\): better ci job output [\#772](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/772) ([mhenrixon](https://github.com/mhenrixon))
139
+
140
+ **Fixed bugs:**
141
+
142
+ - Add unlocked reflect to locksmith unlock! [\#783](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/783) ([ttstarck](https://github.com/ttstarck))
143
+ - fix\(locksmith\): replace deprecated rpoplpush w/lmove [\#781](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/781) ([GabrielaGuedes](https://github.com/GabrielaGuedes))
144
+ - fix\(unlock\): ensure callback and unlock [\#771](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/771) ([mhenrixon](https://github.com/mhenrixon))
145
+
146
+ **Closed issues:**
147
+
148
+ - `unlocked` reflection is never called [\#782](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/782)
149
+ - Migrate away from deprecated Redis commands [\#780](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/780)
150
+ - `while_executing` with `on_conflict: :reschedule` reschedules job when unlock fails [\#770](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/770)
151
+ - Redis server v6.0 compatibility [\#769](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/769)
152
+ - Locks not removed after worker finishes [\#758](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/758)
153
+
154
+ **Merged pull requests:**
155
+
156
+ - Add lock to item when lock\_type is pulled from Job class sidekiq options [\#786](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/786) ([ttstarck](https://github.com/ttstarck))
157
+ - Fix rubocop failure by expecting array to be empty [\#784](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/784) ([GabrielaGuedes](https://github.com/GabrielaGuedes))
158
+ - fix\(doc\): improve information on unlock\_failed [\#776](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/776) ([mhenrixon](https://github.com/mhenrixon))
159
+
3
160
  ## [v8.0.2](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.2) (2023-03-13)
4
161
 
5
162
  [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v8.0.1...v8.0.2)
@@ -266,7 +423,6 @@
266
423
 
267
424
  - Job finished, but lock is not cleared [\#677](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/677)
268
425
  - sidekiq\_options lock conflicts with sidekiq-lock gem lock option [\#669](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/669)
269
- - Slow evalsha causing timeouts [\#668](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/668)
270
426
  - Inconsistent documentation for config validation [\#647](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/647)
271
427
 
272
428
  **Merged pull requests:**
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # SidekiqUniqueJobs
2
2
 
3
- [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](https://badges.gitter.im/mhenrixon/sidekiq-unique-jobs.svg)](https://gitter.im/mhenrixon/sidekiq-unique-jobs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) ![Build Status](https://github.com/mhenrixon/sidekiq-unique-jobs/actions/workflows/rspec.yml/badge.svg?branch=master) [![Code Climate](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs.svg)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs) [![Test Coverage](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/badges/coverage.svg)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/coverage)
3
+ [![Join the chat at https://gitter.im/mhenrixon/sidekiq-unique-jobs](https://badges.gitter.im/mhenrixon/sidekiq-unique-jobs.svg)](https://gitter.im/mhenrixon/sidekiq-unique-jobs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) ![Build Status](https://github.com/mhenrixon/sidekiq-unique-jobs/actions/workflows/rspec.yml/badge.svg?branch=main) [![Code Climate](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs.svg)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs) [![Test Coverage](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/badges/coverage.svg)](https://codeclimate.com/github/mhenrixon/sidekiq-unique-jobs/coverage)
4
4
 
5
5
  ## Support Me
6
6
 
@@ -582,8 +582,6 @@ end
582
582
  #spec/workers/bad_worker_spec.rb
583
583
 
584
584
  require "sidekiq_unique_jobs/testing"
585
- #OR
586
- require "sidekiq_unique_jobs/rspec/matchers"
587
585
 
588
586
  RSpec.describe BadWorker do
589
587
  specify { expect(described_class).to have_valid_sidekiq_options }
@@ -816,7 +814,7 @@ In my benchmarks deleting 1000 orphaned locks with lua performs around 65% faste
816
814
 
817
815
  On the other hand if I increase it to 10 000 orphaned locks per cleanup (`reaper_count: 10_0000`) then redis starts throwing:
818
816
 
819
- > BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE. (Redis::CommandError)
817
+ > BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE. (RedisClient::CommandError)
820
818
 
821
819
  If you want to disable the reaper set it to `:none`, `nil` or `false`. Actually, any value that isn't `:ruby` or `:lua` will disable the reaping.
822
820
 
data/lib/sidekiq_unique_jobs/batch_delete.rb CHANGED
@@ -9,7 +9,7 @@ module SidekiqUniqueJobs
9
9
  class BatchDelete
10
10
  #
11
11
  # @return [Integer] the default batch size
12
- BATCH_SIZE = 100
12
+ BATCH_SIZE = 500
13
13
 
14
14
  #
15
15
  # @return [Array<String>] Supported key suffixes
data/lib/sidekiq_unique_jobs/changelog.rb CHANGED
@@ -59,7 +59,7 @@ module SidekiqUniqueJobs
59
59
  # NOTE: When debugging, check the last item in the returned array.
60
60
  [
61
61
  total_size.to_i,
62
- result[0].to_i, # next_cursor
62
+ result[0].to_i, # next_cursor
63
63
  result[1].map { |entry| load_json(entry) }.select { |entry| entry.is_a?(Hash) },
64
64
  ]
65
65
  end
data/lib/sidekiq_unique_jobs/config.rb CHANGED
@@ -67,10 +67,10 @@ module SidekiqUniqueJobs
67
67
  # @return [Hash<Symbol, SidekiqUniqueJobs::Lock::BaseLock] all available default locks
68
68
  LOCKS =
69
69
  LOCKS_WHEN_BUSY.dup
70
- .merge(LOCKS_WHILE_ENQUEUED.dup)
71
- .merge(LOCKS_WITHOUT_UNLOCK.dup)
72
- .merge(LOCKS_FROM_PUSH_TO_PROCESSED.dup)
73
- .freeze
70
+ .merge(LOCKS_WHILE_ENQUEUED.dup)
71
+ .merge(LOCKS_WITHOUT_UNLOCK.dup)
72
+ .merge(LOCKS_FROM_PUSH_TO_PROCESSED.dup)
73
+ .freeze
74
74
 
75
75
  #
76
76
  # @return [Hash<Symbol, SidekiqUniqueJobs::OnConflict::Strategy] all available default strategies
data/lib/sidekiq_unique_jobs/constants.rb CHANGED
@@ -15,6 +15,7 @@ module SidekiqUniqueJobs
15
15
  DEAD_VERSION = "uniquejobs:dead"
16
16
  DIGESTS = "uniquejobs:digests"
17
17
  EXPIRING_DIGESTS = "uniquejobs:expiring_digests"
18
+ ORPHANED_DIGESTS = "uniquejobs:orphaned_digests"
18
19
  ERRORS = "errors"
19
20
  JID = "jid"
20
21
  LIMIT = "limit"
data/lib/sidekiq_unique_jobs/digests.rb CHANGED
@@ -7,6 +7,16 @@ module SidekiqUniqueJobs
7
7
  # @author Mikael Henriksson <mikael@mhenrixon.com>
8
8
  #
9
9
  class Digests < Redis::SortedSet
10
+ #
11
+ # @return [Integer] the number of matches to return by default
12
+ DEFAULT_COUNT = 1_000
13
+ #
14
+ # @return [String] the default pattern to use for matching
15
+ SCAN_PATTERN = "*"
16
+ #
17
+ # @return [Array(String, String, String, String)] The empty runtime or queuetime keys.
18
+ EMPTY_KEYS_SEGMENT = ["", "", "", ""].freeze
19
+
10
20
  def initialize(digests_key = DIGESTS)
11
21
  super(digests_key)
12
22
  end
@@ -41,19 +51,14 @@ module SidekiqUniqueJobs
41
51
  # Also deletes the :AVAILABLE, :EXPIRED etc keys
42
52
  #
43
53
  # @param [String] digest a unique digest to delete
44
- def delete_by_digest(digest) # rubocop:disable Metrics/MethodLength
54
+ # @param queuetime [Boolean] Whether to delete queue locks.
55
+ # @param runtime [Boolean] Whether to delete run locks.
56
+ def delete_by_digest(digest, queuetime: true, runtime: true)
45
57
  result, elapsed = timed do
46
- call_script(:delete_by_digest, [
47
- digest,
48
- "#{digest}:QUEUED",
49
- "#{digest}:PRIMED",
50
- "#{digest}:LOCKED",
51
- "#{digest}:RUN",
52
- "#{digest}:RUN:QUEUED",
53
- "#{digest}:RUN:PRIMED",
54
- "#{digest}:RUN:LOCKED",
55
- key,
56
- ])
58
+ call_script(
59
+ :delete_by_digest,
60
+ queuetime_keys(queuetime ? digest : nil) + runtime_keys(runtime ? digest : nil) + [key],
61
+ )
57
62
  end
58
63
 
59
64
  log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
@@ -93,9 +98,37 @@ module SidekiqUniqueJobs
93
98
  [
94
99
  total_size.to_i,
95
100
  digests[0].to_i, # next_cursor
96
- digests[1].map { |digest, score| Lock.new(digest, time: score) }, # entries
101
+ digests[1].each_slice(2).map { |digest, score| Lock.new(digest, time: score) }, # entries
97
102
  ]
98
103
  end
99
104
  end
105
+
106
+ private
107
+
108
+ # @param digest [String, nil] The digest to form runtime keys from.
109
+ # @return [Array(String, String, String, String)] The list of runtime keys or empty strings if +digest+ was +nil+.
110
+ def runtime_keys(digest)
111
+ return EMPTY_KEYS_SEGMENT unless digest
112
+
113
+ [
114
+ "#{digest}:RUN",
115
+ "#{digest}:RUN:QUEUED",
116
+ "#{digest}:RUN:PRIMED",
117
+ "#{digest}:RUN:LOCKED",
118
+ ]
119
+ end
120
+
121
+ # @param digest [String, nil] The digest to form queuetime keys from.
122
+ # @return [Array(String, String, String, String)] The list of queuetime keys or empty strings if +digest+ was +nil+.
123
+ def queuetime_keys(digest)
124
+ return EMPTY_KEYS_SEGMENT unless digest
125
+
126
+ [
127
+ digest,
128
+ "#{digest}:QUEUED",
129
+ "#{digest}:PRIMED",
130
+ "#{digest}:LOCKED",
131
+ ]
132
+ end
100
133
  end
101
134
  end
data/lib/sidekiq_unique_jobs/job.rb CHANGED
@@ -41,19 +41,19 @@ module SidekiqUniqueJobs
41
41
  end
42
42
 
43
43
  def add_lock_timeout(item)
44
- item[LOCK_TIMEOUT] ||= SidekiqUniqueJobs::LockTimeout.calculate(item)
44
+ item[LOCK_TIMEOUT] = SidekiqUniqueJobs::LockTimeout.calculate(item)
45
45
  end
46
46
 
47
47
  def add_lock_args(item)
48
- item[LOCK_ARGS] ||= SidekiqUniqueJobs::LockArgs.call(item)
48
+ item[LOCK_ARGS] = SidekiqUniqueJobs::LockArgs.call(item)
49
49
  end
50
50
 
51
51
  def add_lock_digest(item)
52
- item[LOCK_DIGEST] ||= SidekiqUniqueJobs::LockDigest.call(item)
52
+ item[LOCK_DIGEST] = SidekiqUniqueJobs::LockDigest.call(item)
53
53
  end
54
54
 
55
55
  def add_lock_prefix(item)
56
- item[LOCK_PREFIX] ||= SidekiqUniqueJobs.config.lock_prefix
56
+ item[LOCK_PREFIX] = SidekiqUniqueJobs.config.lock_prefix
57
57
  end
58
58
 
59
59
  def add_lock_type(item)
data/lib/sidekiq_unique_jobs/lock/base_lock.rb CHANGED
@@ -109,7 +109,7 @@ module SidekiqUniqueJobs
109
109
  #
110
110
  # Call whatever strategry that has been configured
111
111
  #
112
- # @param [Symbol] origin: the origin `:client` or `:server`
112
+ # @param [Symbol] origin the origin `:client` or `:server`
113
113
  #
114
114
  # @return [void] the return value is irrelevant
115
115
  #
data/lib/sidekiq_unique_jobs/lock/while_executing.rb CHANGED
@@ -11,6 +11,8 @@ module SidekiqUniqueJobs
11
11
  #
12
12
  # @author Mikael Henriksson <mikael@mhenrixon.com>
13
13
  class WhileExecuting < BaseLock
14
+ #
15
+ # @return [String] returns :RUN
14
16
  RUN_SUFFIX = ":RUN"
15
17
 
16
18
  include SidekiqUniqueJobs::OptionsWithFallback
@@ -42,6 +44,7 @@ module SidekiqUniqueJobs
42
44
  with_logging_context do
43
45
  executed = locksmith.execute do
44
46
  yield
47
+ item[JID]
45
48
  ensure
46
49
  unlock_and_callback
47
50
  end
data/lib/sidekiq_unique_jobs/lock.rb CHANGED
@@ -33,9 +33,9 @@ module SidekiqUniqueJobs
33
33
  #
34
34
  # @return [Lock] a newly lock that has been locked
35
35
  #
36
- def self.create(digest, job_id, lock_info = {})
37
- lock = new(digest, time: Timing.now_f)
38
- lock.lock(job_id, lock_info)
36
+ def self.create(digest, job_id, lock_info: {}, time: Timing.now_f, score: nil)
37
+ lock = new(digest, time: time)
38
+ lock.lock(job_id, lock_info, score)
39
39
  lock
40
40
  end
41
41
 
@@ -47,7 +47,7 @@ module SidekiqUniqueJobs
47
47
  #
48
48
  def initialize(key, time: nil)
49
49
  @key = get_key(key)
50
- time = time.is_a?(Float) ? time : time.to_f
50
+ time = time.to_f unless time.is_a?(Float)
51
51
  return unless time.nonzero?
52
52
 
53
53
  @created_at = time
@@ -63,15 +63,16 @@ module SidekiqUniqueJobs
63
63
  #
64
64
  # @return [void]
65
65
  #
66
- def lock(job_id, lock_info = {})
66
+ def lock(job_id, lock_info = {}, score = nil)
67
+ score ||= now_f
67
68
  redis do |conn|
68
69
  conn.multi do |pipeline|
69
70
  pipeline.set(key.digest, job_id)
70
71
  pipeline.hset(key.locked, job_id, now_f)
71
72
  info.set(lock_info, pipeline)
72
- add_digest_to_set(pipeline, lock_info)
73
- pipeline.zadd(key.changelog, now_f, changelog_json(job_id, "queue.lua", "Queued"))
74
- pipeline.zadd(key.changelog, now_f, changelog_json(job_id, "lock.lua", "Locked"))
73
+ add_digest_to_set(pipeline, lock_info, score)
74
+ pipeline.zadd(key.changelog, score, changelog_json(job_id, "queue.lua", "Queued"))
75
+ pipeline.zadd(key.changelog, score, changelog_json(job_id, "lock.lua", "Locked"))
75
76
  end
76
77
  end
77
78
  end
@@ -333,12 +334,14 @@ module SidekiqUniqueJobs
333
334
  #
334
335
  # @return [nil]
335
336
  #
336
- def add_digest_to_set(pipeline, lock_info)
337
+ def add_digest_to_set(pipeline, lock_info, score = nil)
338
+ score ||= now_f
337
339
  digest_string = key.digest
340
+
338
341
  if lock_info["lock"] == :until_expired
339
- pipeline.zadd(key.expiring_digests, now_f + lock_info["ttl"], digest_string)
342
+ pipeline.zadd(key.expiring_digests, score + lock_info["ttl"], digest_string)
340
343
  else
341
- pipeline.zadd(key.digests, now_f, digest_string)
344
+ pipeline.zadd(key.digests, score, digest_string)
342
345
  end
343
346
  end
344
347
  end
data/lib/sidekiq_unique_jobs/lock_config.rb CHANGED
@@ -113,14 +113,20 @@ module SidekiqUniqueJobs
113
113
 
114
114
  # the strategy to use as conflict resolution from sidekiq client
115
115
  def on_client_conflict
116
- @on_client_conflict ||= on_conflict["client"] || on_conflict[:client] if on_conflict.is_a?(Hash)
117
- @on_client_conflict ||= on_conflict
116
+ @on_client_conflict ||= if on_conflict.is_a?(Hash)
117
+ on_conflict["client"] || on_conflict[:client]
118
+ else
119
+ on_conflict
120
+ end
118
121
  end
119
122
 
120
123
  # the strategy to use as conflict resolution from sidekiq server
121
124
  def on_server_conflict
122
- @on_server_conflict ||= on_conflict["server"] || on_conflict[:server] if on_conflict.is_a?(Hash)
123
- @on_server_conflict ||= on_conflict
125
+ @on_server_conflict ||= if on_conflict.is_a?(Hash)
126
+ on_conflict["server"] || on_conflict[:server]
127
+ else
128
+ on_conflict
129
+ end
124
130
  end
125
131
  end
126
132
  end
data/lib/sidekiq_unique_jobs/locksmith.rb CHANGED
@@ -82,7 +82,7 @@ module SidekiqUniqueJobs
82
82
  # Deletes the lock regardless of if it has a pttl set
83
83
  #
84
84
  def delete!
85
- call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).to_i.positive?
85
+ call_script(:delete, key.to_a, argv).to_i.positive?
86
86
  end
87
87
 
88
88
  #
@@ -188,7 +188,7 @@ module SidekiqUniqueJobs
188
188
  #
189
189
  # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
190
190
  # @param [Method] primed_method reference to the method to use for getting a primed token
191
- # @param [nil, Integer, Float] time to wait before timeout
191
+ # @param [nil, Integer, Float] wait time to wait before timeout
192
192
  #
193
193
  # @yieldparam [string] job_id the sidekiq JID
194
194
  # @yieldreturn [void] whatever the calling block returns
@@ -258,8 +258,8 @@ module SidekiqUniqueJobs
258
258
 
259
259
  # NOTE: When debugging, change .value to .value!
260
260
  primed_jid = Concurrent::Promises
261
- .future(conn) { |red_con| pop_queued(red_con, timeout) }
262
- .value
261
+ .future(conn) { |red_con| pop_queued(red_con, timeout) }
262
+ .value
263
263
 
264
264
  handle_primed(primed_jid, &block)
265
265
  end
@@ -362,7 +362,11 @@ module SidekiqUniqueJobs
362
362
  end
363
363
 
364
364
  def argv
365
- [job_id, config.pttl, config.type, config.limit]
365
+ [job_id, config.pttl, config.type, config.limit, lock_score]
366
+ end
367
+
368
+ def lock_score
369
+ item[AT].to_s
366
370
  end
367
371
 
368
372
  def lock_info
@@ -375,6 +379,7 @@ module SidekiqUniqueJobs
375
379
  TYPE => config.type,
376
380
  LOCK_ARGS => item[LOCK_ARGS],
377
381
  TIME => now_f,
382
+ AT => item[AT],
378
383
  )
379
384
  end
380
385
 
data/lib/sidekiq_unique_jobs/lua/delete.lua CHANGED
@@ -13,14 +13,15 @@ local job_id = ARGV[1]
13
13
  local pttl = tonumber(ARGV[2])
14
14
  local lock_type = ARGV[3]
15
15
  local limit = tonumber(ARGV[4])
16
+ local lock_score = ARGV[5]
16
17
  -------- END lock arguments -----------
17
18
 
18
19
  -------- BEGIN injected arguments --------
19
- local current_time = tonumber(ARGV[5])
20
- local debug_lua = tostring(ARGV[6]) == "1"
21
- local max_history = tonumber(ARGV[7])
22
- local script_name = tostring(ARGV[8]) .. ".lua"
23
- local redisversion = tostring(ARGV[9])
20
+ local current_time = tonumber(ARGV[6])
21
+ local debug_lua = tostring(ARGV[7]) == "1"
22
+ local max_history = tonumber(ARGV[8])
23
+ local script_name = tostring(ARGV[9]) .. ".lua"
24
+ local redisversion = tostring(ARGV[10])
24
25
  --------- END injected arguments ---------
25
26
 
26
27
  -------- BEGIN local functions --------
data/lib/sidekiq_unique_jobs/lua/lock.lua CHANGED
@@ -15,15 +15,16 @@ local job_id = ARGV[1]
15
15
  local pttl = tonumber(ARGV[2])
16
16
  local lock_type = ARGV[3]
17
17
  local limit = tonumber(ARGV[4])
18
+ local lock_score = ARGV[5]
18
19
  -------- END lock arguments -----------
19
20
 
20
21
 
21
22
  -------- BEGIN injected arguments --------
22
- local current_time = tonumber(ARGV[5])
23
- local debug_lua = tostring(ARGV[6]) == "1"
24
- local max_history = tonumber(ARGV[7])
25
- local script_name = tostring(ARGV[8]) .. ".lua"
26
- local redisversion = ARGV[9]
23
+ local current_time = tonumber(ARGV[6])
24
+ local debug_lua = tostring(ARGV[7]) == "1"
25
+ local max_history = tonumber(ARGV[8])
26
+ local script_name = tostring(ARGV[9]) .. ".lua"
27
+ local redisversion = ARGV[10]
27
28
  --------- END injected arguments ---------
28
29
 
29
30
 
@@ -62,8 +63,16 @@ if lock_type == "until_expired" and pttl and pttl > 0 then
62
63
  log_debug("ZADD", expiring_digests, current_time + pttl, digest)
63
64
  redis.call("ZADD", expiring_digests, current_time + pttl, digest)
64
65
  else
65
- log_debug("ZADD", digests, current_time, digest)
66
- redis.call("ZADD", digests, current_time, digest)
66
+ local score
67
+
68
+ if #lock_score == 0 then
69
+ score = current_time
70
+ else
71
+ score = lock_score
72
+ end
73
+
74
+ log_debug("ZADD", digests, score, digest)
75
+ redis.call("ZADD", digests, score, digest)
67
76
  end
68
77
 
69
78
  log_debug("HSET", locked, job_id, current_time)
data/lib/sidekiq_unique_jobs/lua/queue.lua CHANGED
@@ -10,18 +10,19 @@ local digests = KEYS[7]
10
10
 
11
11
 
12
12
  -------- BEGIN lock arguments ---------
13
- local job_id = ARGV[1] -- The job_id that was previously primed
14
- local pttl = tonumber(ARGV[2])
15
- local lock_type = ARGV[3]
16
- local limit = tonumber(ARGV[4])
13
+ local job_id = ARGV[1] -- The job_id that was previously primed
14
+ local pttl = tonumber(ARGV[2])
15
+ local lock_type = ARGV[3]
16
+ local limit = tonumber(ARGV[4])
17
+ local lock_score = ARGV[5]
17
18
  -------- END lock arguments -----------
18
19
 
19
20
 
20
21
  -------- BEGIN injected arguments --------
21
- local current_time = tonumber(ARGV[5])
22
- local debug_lua = tostring(ARGV[6]) == "1"
23
- local max_history = tonumber(ARGV[7])
24
- local script_name = tostring(ARGV[8]) .. ".lua"
22
+ local current_time = tonumber(ARGV[6])
23
+ local debug_lua = tostring(ARGV[7]) == "1"
24
+ local max_history = tonumber(ARGV[8])
25
+ local script_name = tostring(ARGV[9]) .. ".lua"
25
26
  --------- END injected arguments ---------
26
27
 
27
28
 
data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua CHANGED
@@ -1,22 +1,22 @@
1
1
  local function delete_from_queue(queue, digest)
2
- local per = 50
3
- local total = redis.call("LLEN", queue)
4
- local index = 0
5
- local result = nil
2
+ local total = redis.call("LLEN", queue)
3
+ local per = 50
4
+
5
+ for index = 0, total, per do
6
+ local items = redis.call("LRANGE", queue, index, index + per - 1)
6
7
 
7
- while (index < total) do
8
- local items = redis.call("LRANGE", queue, index, index + per -1)
9
8
  if #items == 0 then
10
9
  break
11
10
  end
11
+
12
12
  for _, item in pairs(items) do
13
13
  if string.find(item, digest) then
14
14
  redis.call("LREM", queue, 1, item)
15
- result = item
16
- break
15
+
16
+ return item
17
17
  end
18
18
  end
19
- index = index + per
20
19
  end
21
- return result
20
+
21
+ return nil
22
22
  end