sidekiq-unique-jobs 6.0.25 → 7.1.33

Sign up to get free protection for your applications and to get access to all the features.
Files changed (130) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1157 -41
  3. data/README.md +825 -291
  4. data/lib/sidekiq_unique_jobs/batch_delete.rb +124 -0
  5. data/lib/sidekiq_unique_jobs/changelog.rb +78 -0
  6. data/lib/sidekiq_unique_jobs/cli.rb +57 -29
  7. data/lib/sidekiq_unique_jobs/config.rb +319 -0
  8. data/lib/sidekiq_unique_jobs/connection.rb +6 -5
  9. data/lib/sidekiq_unique_jobs/constants.rb +46 -25
  10. data/lib/sidekiq_unique_jobs/core_ext.rb +80 -0
  11. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  12. data/lib/sidekiq_unique_jobs/digests.rb +70 -102
  13. data/lib/sidekiq_unique_jobs/exceptions.rb +88 -12
  14. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  15. data/lib/sidekiq_unique_jobs/job.rb +46 -12
  16. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  17. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  18. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +111 -82
  19. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  20. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  21. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +40 -15
  22. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +30 -7
  23. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +26 -2
  24. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +27 -15
  25. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  26. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +26 -12
  27. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +3 -3
  28. data/lib/sidekiq_unique_jobs/lock.rb +342 -0
  29. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  30. data/lib/sidekiq_unique_jobs/lock_config.rb +126 -0
  31. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  32. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  33. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  34. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  35. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  36. data/lib/sidekiq_unique_jobs/locksmith.rb +305 -101
  37. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  38. data/lib/sidekiq_unique_jobs/logging.rb +202 -33
  39. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  40. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +42 -0
  41. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  42. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  43. data/lib/sidekiq_unique_jobs/lua/lock.lua +99 -0
  44. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  45. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  46. data/lib/sidekiq_unique_jobs/lua/queue.lua +87 -0
  47. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +122 -0
  48. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  49. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  57. data/lib/sidekiq_unique_jobs/lua/unlock.lua +107 -0
  58. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  59. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  60. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  61. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  62. data/lib/sidekiq_unique_jobs/middleware.rb +29 -43
  63. data/lib/sidekiq_unique_jobs/normalizer.rb +4 -4
  64. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +9 -5
  65. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +1 -1
  66. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
  67. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +63 -17
  68. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +54 -14
  69. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +16 -5
  70. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +25 -6
  71. data/lib/sidekiq_unique_jobs/on_conflict.rb +23 -10
  72. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +39 -36
  73. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  74. data/lib/sidekiq_unique_jobs/orphans/manager.rb +241 -0
  75. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  76. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  77. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +114 -0
  78. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  79. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +298 -0
  80. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  81. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  82. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  83. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  84. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +86 -0
  85. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  86. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  87. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  88. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  89. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  90. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/script/caller.rb +127 -0
  92. data/lib/sidekiq_unique_jobs/script.rb +15 -0
  93. data/lib/sidekiq_unique_jobs/server.rb +61 -0
  94. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +114 -65
  95. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +252 -36
  96. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +47 -32
  97. data/lib/sidekiq_unique_jobs/testing.rb +102 -29
  98. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  99. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  100. data/lib/sidekiq_unique_jobs/unlockable.rb +20 -4
  101. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  102. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +155 -0
  103. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  104. data/lib/sidekiq_unique_jobs/version_check.rb +23 -4
  105. data/lib/sidekiq_unique_jobs/web/helpers.rb +138 -13
  106. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +4 -4
  107. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +54 -0
  108. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  109. data/lib/sidekiq_unique_jobs/web/views/locks.erb +54 -0
  110. data/lib/sidekiq_unique_jobs/web.rb +82 -32
  111. data/lib/sidekiq_unique_jobs.rb +54 -7
  112. data/lib/tasks/changelog.rake +16 -16
  113. metadata +134 -177
  114. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  115. data/lib/sidekiq_unique_jobs/scripts.rb +0 -118
  116. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -46
  117. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  118. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  119. data/lib/sidekiq_unique_jobs/unique_args.rb +0 -150
  120. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  121. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  122. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -46
  123. data/redis/acquire_lock.lua +0 -21
  124. data/redis/convert_legacy_lock.lua +0 -13
  125. data/redis/delete.lua +0 -14
  126. data/redis/delete_by_digest.lua +0 -23
  127. data/redis/delete_job_by_digest.lua +0 -60
  128. data/redis/lock.lua +0 -62
  129. data/redis/release_stale_locks.lua +0 -90
  130. data/redis/unlock.lua +0 -35
@@ -1,60 +1,110 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- begin
4
- require "delegate"
5
- require "rack"
6
- require "sidekiq/web"
7
- rescue LoadError
8
- # client-only usage
9
- end
10
-
11
3
  require_relative "web/helpers"
12
4
 
13
5
  module SidekiqUniqueJobs
14
6
  # Utility module to help manage unique keys in redis.
15
7
  # Useful for deleting keys that for whatever reason wasn't deleted
16
8
  #
17
- # @author Mikael Henriksson <mikael@zoolutions.se>
9
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
18
10
  module Web
19
- def self.registered(app) # rubocop:disable Metrics/MethodLength
11
+ def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
20
12
  app.helpers do
21
13
  include Web::Helpers
22
14
  end
23
15
 
24
- app.get "/unique_digests" do
25
- @filter = params[:filter] || "*"
16
+ app.get "/changelogs" do
17
+ @filter = h(params[:filter] || "*")
18
+ @filter = "*" if @filter == ""
19
+ @count = h(params[:count] || 100).to_i
20
+ @current_cursor = h(params[:cursor])
21
+ @prev_cursor = h(params[:prev_cursor])
22
+ @total_size, @next_cursor, @changelogs = changelog.page(
23
+ cursor: @current_cursor,
24
+ pattern: @filter,
25
+ page_size: @count,
26
+ )
27
+
28
+ erb(unique_template(:changelogs))
29
+ end
30
+
31
+ app.get "/changelogs/delete_all" do
32
+ changelog.clear
33
+ redirect_to :changelogs
34
+ end
35
+
36
+ app.get "/locks" do
37
+ @filter = h(params[:filter] || "*")
38
+ @filter = "*" if @filter == ""
39
+ @count = h(params[:count] || 100).to_i
40
+ @current_cursor = h(params[:cursor])
41
+ @prev_cursor = h(params[:prev_cursor])
42
+
43
+ @total_size, @next_cursor, @locks = digests.page(
44
+ cursor: @current_cursor,
45
+ pattern: @filter,
46
+ page_size: @count,
47
+ )
48
+
49
+ erb(unique_template(:locks))
50
+ end
51
+
52
+ app.get "/expiring_locks" do
53
+ @filter = h(params[:filter] || "*")
26
54
  @filter = "*" if @filter == ""
27
- @count = (params[:count] || 100).to_i
28
- @current_cursor = params[:cursor]
29
- @prev_cursor = params[:prev_cursor]
30
- @total_size, @next_cursor, @unique_digests =
31
- SidekiqUniqueJobs::Digests.page(pattern: @filter, cursor: @current_cursor, page_size: @count)
55
+ @count = h(params[:count] || 100).to_i
56
+ @current_cursor = h(params[:cursor])
57
+ @prev_cursor = h(params[:prev_cursor])
58
+
59
+ @total_size, @next_cursor, @locks = expiring_digests.page(
60
+ cursor: @current_cursor,
61
+ pattern: @filter,
62
+ page_size: @count,
63
+ )
32
64
 
33
- erb(unique_template(:unique_digests))
65
+ erb(unique_template(:locks))
34
66
  end
35
67
 
36
- app.get "/unique_digests/delete_all" do
37
- SidekiqUniqueJobs::Digests.delete_by_pattern("*", count: SidekiqUniqueJobs::Digests.count)
38
- redirect_to :unique_digests
68
+ app.get "/locks/delete_all" do
69
+ digests.delete_by_pattern("*", count: digests.count)
70
+ expiring_digests.delete_by_pattern("*", count: digests.count)
71
+ redirect_to :locks
39
72
  end
40
73
 
41
- app.get "/unique_digests/:digest" do
42
- @digest = params[:digest]
43
- @unique_keys = SidekiqUniqueJobs::Util.keys("#{@digest}*", 1000)
74
+ app.get "/locks/:digest" do
75
+ @digest = h(params[:digest])
76
+ @lock = SidekiqUniqueJobs::Lock.new(@digest)
44
77
 
45
- erb(unique_template(:unique_digest))
78
+ erb(unique_template(:lock))
46
79
  end
47
80
 
48
- app.get "/unique_digests/:digest/delete" do
49
- SidekiqUniqueJobs::Digests.delete_by_digest(params[:digest])
50
- redirect_to :unique_digests
81
+ app.get "/locks/:digest/delete" do
82
+ digests.delete_by_digest(params[:digest])
83
+ expiring_digests.delete_by_digest(params[:digest])
84
+ redirect_to :locks
85
+ end
86
+
87
+ app.get "/locks/:digest/jobs/:job_id/delete" do
88
+ @digest = h(params[:digest])
89
+ @job_id = h(params[:job_id])
90
+ @lock = SidekiqUniqueJobs::Lock.new(@digest)
91
+ @lock.unlock(@job_id)
92
+
93
+ redirect_to "locks/#{@lock.key}"
51
94
  end
52
95
  end
53
96
  end
54
97
  end
55
98
 
56
- if defined?(Sidekiq::Web)
57
- Sidekiq::Web.register SidekiqUniqueJobs::Web
58
- Sidekiq::Web.tabs["Unique Digests"] = "unique_digests"
59
- # Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), 'locales')
99
+ begin
100
+ require "delegate" unless defined?(DelegateClass)
101
+ require "sidekiq/web" unless defined?(Sidekiq::Web)
102
+
103
+ Sidekiq::Web.register(SidekiqUniqueJobs::Web)
104
+ Sidekiq::Web.tabs["Locks"] = "locks"
105
+ Sidekiq::Web.tabs["Expiring Locks"] = "expiring_locks"
106
+ Sidekiq::Web.tabs["Changelogs"] = "changelogs"
107
+ Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
108
+ rescue NameError, LoadError => ex
109
+ SidekiqUniqueJobs.logger.error(ex)
60
110
  end
@@ -1,25 +1,64 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "forwardable"
3
+ require "brpoplpush/redis_script"
4
+ require "concurrent/future"
5
+ require "concurrent/promises"
6
+ require "concurrent/map"
4
7
  require "concurrent/mutable_struct"
8
+ require "concurrent/timer_task"
9
+ require "concurrent/executor/ruby_single_thread_executor"
10
+ require "digest"
11
+ require "digest/sha1"
12
+ require "erb"
13
+ require "forwardable"
14
+ require "json"
15
+ require "pathname"
16
+ require "sidekiq"
5
17
 
18
+ require "sidekiq_unique_jobs/deprecation"
19
+ require "sidekiq_unique_jobs/reflections"
20
+ require "sidekiq_unique_jobs/reflectable"
21
+ require "sidekiq_unique_jobs/timer_task"
6
22
  require "sidekiq_unique_jobs/version"
7
23
  require "sidekiq_unique_jobs/version_check"
8
24
  require "sidekiq_unique_jobs/constants"
25
+ require "sidekiq_unique_jobs/json"
9
26
  require "sidekiq_unique_jobs/logging"
27
+ require "sidekiq_unique_jobs/logging/middleware_context"
28
+ require "sidekiq_unique_jobs/timing"
10
29
  require "sidekiq_unique_jobs/sidekiq_worker_methods"
30
+ require "sidekiq_unique_jobs/lock_type"
11
31
  require "sidekiq_unique_jobs/connection"
12
32
  require "sidekiq_unique_jobs/exceptions"
33
+ require "sidekiq_unique_jobs/script"
34
+ require "sidekiq_unique_jobs/script/caller"
35
+ require "sidekiq_unique_jobs/normalizer"
13
36
  require "sidekiq_unique_jobs/job"
14
- require "sidekiq_unique_jobs/util"
15
- require "sidekiq_unique_jobs/digests"
37
+ require "sidekiq_unique_jobs/redis"
38
+ require "sidekiq_unique_jobs/redis/entity"
39
+ require "sidekiq_unique_jobs/redis/hash"
40
+ require "sidekiq_unique_jobs/redis/list"
41
+ require "sidekiq_unique_jobs/redis/set"
42
+ require "sidekiq_unique_jobs/redis/sorted_set"
43
+ require "sidekiq_unique_jobs/redis/string"
44
+ require "sidekiq_unique_jobs/batch_delete"
45
+ require "sidekiq_unique_jobs/orphans/reaper"
46
+ require "sidekiq_unique_jobs/orphans/observer"
47
+ require "sidekiq_unique_jobs/orphans/manager"
48
+ require "sidekiq_unique_jobs/orphans/reaper_resurrector"
16
49
  require "sidekiq_unique_jobs/cli"
17
50
  require "sidekiq_unique_jobs/core_ext"
18
- require "sidekiq_unique_jobs/timeout"
19
- require "sidekiq_unique_jobs/scripts"
20
- require "sidekiq_unique_jobs/unique_args"
51
+ require "sidekiq_unique_jobs/lock_timeout"
52
+ require "sidekiq_unique_jobs/lock_ttl"
53
+ require "sidekiq_unique_jobs/lock_args"
54
+ require "sidekiq_unique_jobs/lock_digest"
21
55
  require "sidekiq_unique_jobs/unlockable"
56
+ require "sidekiq_unique_jobs/key"
22
57
  require "sidekiq_unique_jobs/locksmith"
58
+ require "sidekiq_unique_jobs/options_with_fallback"
59
+ require "sidekiq_unique_jobs/lock"
60
+ require "sidekiq_unique_jobs/lock_config"
61
+ require "sidekiq_unique_jobs/lock_info"
23
62
  require "sidekiq_unique_jobs/lock/base_lock"
24
63
  require "sidekiq_unique_jobs/lock/until_executed"
25
64
  require "sidekiq_unique_jobs/lock/until_executing"
@@ -27,9 +66,17 @@ require "sidekiq_unique_jobs/lock/until_expired"
27
66
  require "sidekiq_unique_jobs/lock/while_executing"
28
67
  require "sidekiq_unique_jobs/lock/while_executing_reject"
29
68
  require "sidekiq_unique_jobs/lock/until_and_while_executing"
30
- require "sidekiq_unique_jobs/options_with_fallback"
31
69
  require "sidekiq_unique_jobs/middleware"
70
+ require "sidekiq_unique_jobs/middleware/client"
71
+ require "sidekiq_unique_jobs/middleware/server"
32
72
  require "sidekiq_unique_jobs/sidekiq_unique_ext"
33
73
  require "sidekiq_unique_jobs/on_conflict"
74
+ require "sidekiq_unique_jobs/changelog"
75
+ require "sidekiq_unique_jobs/digests"
76
+ require "sidekiq_unique_jobs/expiring_digests"
34
77
 
78
+ require "sidekiq_unique_jobs/config"
35
79
  require "sidekiq_unique_jobs/sidekiq_unique_jobs"
80
+ require "sidekiq_unique_jobs/update_version"
81
+ require "sidekiq_unique_jobs/upgrade_locks"
82
+ require "sidekiq_unique_jobs/server"
@@ -1,23 +1,23 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # rubocop:disable Style/MutableConstant
4
+ CHANGELOG_CMD = %w[
5
+ github_changelog_generator
6
+ --no-verbose
7
+ --user
8
+ mhenrixon
9
+ --project
10
+ sidekiq-unique-jobs
11
+ --token
12
+ ]
13
+ ADD_CHANGELOG_CMD = "git add --all"
14
+ COMMIT_CHANGELOG_CMD = "git commit -a -m 'Update changelog'"
15
+ # rubocop:enable Style/MutableConstant
16
+
3
17
  desc "Generate a Changelog"
4
18
  task :changelog do
5
- # rubocop:disable Style/MutableConstant
6
- CHANGELOG_CMD ||= %w[
7
- github_changelog_generator
8
- -u
9
- mhenrixon
10
- -p
11
- sidekiq-unique-jobs
12
- --no-verbose
13
- --token
14
- ]
15
- ADD_CHANGELOG_CMD ||= "git add --all"
16
- COMMIT_CHANGELOG_CMD ||= "git commit -a -m 'Update changelog'"
17
- # rubocop:enable Style/MutableConstant
18
-
19
- sh("git checkout master")
20
- sh(*CHANGELOG_CMD.push(ENV["CHANGELOG_GITHUB_TOKEN"]))
19
+ sh("git checkout main")
20
+ sh(*CHANGELOG_CMD.push(ENV.fetch("CHANGELOG_GITHUB_TOKEN", nil)))
21
21
  sh(ADD_CHANGELOG_CMD)
22
22
  sh(COMMIT_CHANGELOG_CMD)
23
23
  end