sidekiq-unique-jobs 6.0.25 → 7.1.29

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq-unique-jobs might be problematic. Click here for more details.

Files changed (129) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1157 -41
  3. data/README.md +825 -291
  4. data/lib/sidekiq_unique_jobs/batch_delete.rb +124 -0
  5. data/lib/sidekiq_unique_jobs/changelog.rb +78 -0
  6. data/lib/sidekiq_unique_jobs/cli.rb +57 -29
  7. data/lib/sidekiq_unique_jobs/config.rb +319 -0
  8. data/lib/sidekiq_unique_jobs/connection.rb +6 -5
  9. data/lib/sidekiq_unique_jobs/constants.rb +46 -25
  10. data/lib/sidekiq_unique_jobs/core_ext.rb +80 -0
  11. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  12. data/lib/sidekiq_unique_jobs/digests.rb +70 -102
  13. data/lib/sidekiq_unique_jobs/exceptions.rb +88 -12
  14. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  15. data/lib/sidekiq_unique_jobs/job.rb +41 -12
  16. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  17. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  18. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +111 -82
  19. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  20. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  21. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +40 -15
  22. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +29 -7
  23. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +26 -2
  24. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +27 -15
  25. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  26. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +26 -12
  27. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +3 -3
  28. data/lib/sidekiq_unique_jobs/lock.rb +342 -0
  29. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  30. data/lib/sidekiq_unique_jobs/lock_config.rb +126 -0
  31. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  32. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  33. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  34. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  35. data/lib/sidekiq_unique_jobs/locksmith.rb +295 -101
  36. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  37. data/lib/sidekiq_unique_jobs/logging.rb +202 -33
  38. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  39. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +42 -0
  40. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  41. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  42. data/lib/sidekiq_unique_jobs/lua/lock.lua +99 -0
  43. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  44. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  45. data/lib/sidekiq_unique_jobs/lua/queue.lua +87 -0
  46. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +122 -0
  47. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  48. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  49. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  56. data/lib/sidekiq_unique_jobs/lua/unlock.lua +107 -0
  57. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  58. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  59. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  60. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  61. data/lib/sidekiq_unique_jobs/middleware.rb +29 -43
  62. data/lib/sidekiq_unique_jobs/normalizer.rb +4 -4
  63. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +9 -5
  64. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +1 -1
  65. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
  66. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +63 -17
  67. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +54 -14
  68. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +16 -5
  69. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +25 -6
  70. data/lib/sidekiq_unique_jobs/on_conflict.rb +23 -10
  71. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +38 -35
  72. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  73. data/lib/sidekiq_unique_jobs/orphans/manager.rb +241 -0
  74. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  75. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  76. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +114 -0
  77. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  78. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +298 -0
  79. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  80. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  81. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  82. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  83. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +86 -0
  84. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  85. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  86. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  87. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  88. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  90. data/lib/sidekiq_unique_jobs/script/caller.rb +127 -0
  91. data/lib/sidekiq_unique_jobs/script.rb +15 -0
  92. data/lib/sidekiq_unique_jobs/server.rb +61 -0
  93. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +114 -65
  94. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +251 -35
  95. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +47 -32
  96. data/lib/sidekiq_unique_jobs/testing.rb +102 -29
  97. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  98. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  99. data/lib/sidekiq_unique_jobs/unlockable.rb +20 -4
  100. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  101. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +155 -0
  102. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  103. data/lib/sidekiq_unique_jobs/version_check.rb +23 -4
  104. data/lib/sidekiq_unique_jobs/web/helpers.rb +138 -13
  105. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +4 -4
  106. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +54 -0
  107. data/lib/sidekiq_unique_jobs/web/views/lock.erb +108 -0
  108. data/lib/sidekiq_unique_jobs/web/views/locks.erb +54 -0
  109. data/lib/sidekiq_unique_jobs/web.rb +76 -27
  110. data/lib/sidekiq_unique_jobs.rb +53 -7
  111. data/lib/tasks/changelog.rake +16 -16
  112. metadata +133 -177
  113. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  114. data/lib/sidekiq_unique_jobs/scripts.rb +0 -118
  115. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -46
  116. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  117. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  118. data/lib/sidekiq_unique_jobs/unique_args.rb +0 -150
  119. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  120. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  121. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -46
  122. data/redis/acquire_lock.lua +0 -21
  123. data/redis/convert_legacy_lock.lua +0 -13
  124. data/redis/delete.lua +0 -14
  125. data/redis/delete_by_digest.lua +0 -23
  126. data/redis/delete_job_by_digest.lua +0 -60
  127. data/redis/lock.lua +0 -62
  128. data/redis/release_stale_locks.lua +0 -90
  129. data/redis/unlock.lua +0 -35
@@ -1,63 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SidekiqUniqueJobs
4
- module Timeout
5
- # Calculates timeout and expiration
6
- #
7
- # @author Mikael Henriksson <mikael@zoolutions.se>
8
- class Calculator
9
- include SidekiqUniqueJobs::SidekiqWorkerMethods
10
-
11
- # @attr [Hash] item the Sidekiq job hash
12
- attr_reader :item
13
-
14
- # @param [Hash] item the Sidekiq job hash
15
- # @option item [Integer, nil] :lock_expiration the configured lock expiration
16
- # @option item [Integer, nil] :lock_timeout the configured lock timeout
17
- # @option item [String] :class the class of the sidekiq worker
18
- # @option item [Float] :at the unix time the job is scheduled at
19
- def initialize(item)
20
- @item = item
21
- @worker_class = item[CLASS_KEY]
22
- end
23
-
24
- # The time until a job is scheduled
25
- # @return [Integer] the number of seconds until job is scheduled
26
- def time_until_scheduled
27
- return 0 unless scheduled_at
28
-
29
- scheduled_at.to_i - Time.now.utc.to_i
30
- end
31
-
32
- # The time a job is scheduled
33
- # @return [Float] the exact unix time the job is scheduled at
34
- def scheduled_at
35
- @scheduled_at ||= item[AT_KEY]
36
- end
37
-
38
- # The configured lock_expiration
39
- def lock_expiration
40
- @lock_expiration ||= begin
41
- expiration = item[LOCK_EXPIRATION_KEY]
42
- expiration ||= worker_options[LOCK_EXPIRATION_KEY]
43
- expiration && expiration.to_i + time_until_scheduled
44
- end
45
- end
46
-
47
- # The configured lock_timeout
48
- def lock_timeout
49
- @lock_timeout = begin
50
- timeout = default_worker_options[LOCK_TIMEOUT_KEY]
51
- timeout = default_lock_timeout if default_lock_timeout
52
- timeout = worker_options[LOCK_TIMEOUT_KEY] if worker_options.key?(LOCK_TIMEOUT_KEY)
53
- timeout
54
- end
55
- end
56
-
57
- # The default lock_timeout of this gem
58
- def default_lock_timeout
59
- SidekiqUniqueJobs.config.default_lock_timeout
60
- end
61
- end
62
- end
63
- end
@@ -1,8 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SidekiqUniqueJobs
4
- module Timeout
5
- end
6
- end
7
-
8
- require "sidekiq_unique_jobs/timeout/calculator"
@@ -1,150 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "digest"
4
- require "openssl"
5
- require "sidekiq_unique_jobs/normalizer"
6
-
7
- module SidekiqUniqueJobs
8
- # Handles uniqueness of sidekiq arguments
9
- #
10
- # @author Mikael Henriksson <mikael@zoolutions.se>
11
- class UniqueArgs
12
- include SidekiqUniqueJobs::Logging
13
- include SidekiqUniqueJobs::SidekiqWorkerMethods
14
-
15
- # Convenience method for returning a digest
16
- # @param [Hash] item a Sidekiq job hash
17
- # @return [String] a unique digest
18
- def self.digest(item)
19
- new(item).unique_digest
20
- end
21
-
22
- # The sidekiq job hash
23
- # @return [Hash] the Sidekiq job hash
24
- attr_reader :item
25
-
26
- # @param [Hash] item a Sidekiq job hash
27
- def initialize(item)
28
- @item = item
29
- @worker_class = item[CLASS_KEY]
30
-
31
- add_uniqueness_to_item
32
- end
33
-
34
- # Appends the keys unique_prefix, unique_args and {#unique_digest} to the sidekiq job hash {#item}
35
- # @return [void]
36
- def add_uniqueness_to_item
37
- item[UNIQUE_PREFIX_KEY] ||= unique_prefix
38
- item[UNIQUE_ARGS_KEY] = unique_args(item[ARGS_KEY])
39
- item[UNIQUE_DIGEST_KEY] = unique_digest
40
- end
41
-
42
- # Memoized unique_digest
43
- # @return [String] a unique digest
44
- def unique_digest
45
- @unique_digest ||= create_digest
46
- end
47
-
48
- # Creates a namespaced unique digest based on the {#digestable_hash} and the {#unique_prefix}
49
- # @return [String] a unique digest
50
- def create_digest
51
- digest = OpenSSL::Digest::MD5.hexdigest(Sidekiq.dump_json(digestable_hash))
52
- "#{unique_prefix}:#{digest}"
53
- end
54
-
55
- # A prefix to use as namespace for the {#unique_digest}
56
- # @return [String] a unique digest
57
- def unique_prefix
58
- worker_options[UNIQUE_PREFIX_KEY] || SidekiqUniqueJobs.config.unique_prefix
59
- end
60
-
61
- # Filter a hash to use for digest
62
- # @return [Hash] to use for digest
63
- def digestable_hash
64
- @item.slice(CLASS_KEY, QUEUE_KEY, UNIQUE_ARGS_KEY, APARTMENT).tap do |hash|
65
- hash.delete(QUEUE_KEY) if unique_across_queues?
66
- hash.delete(CLASS_KEY) if unique_across_workers?
67
- end
68
- end
69
-
70
- # The unique arguments to use for creating a lock
71
- # @return [Array] the arguments filters by the {#filtered_args} method if {#unique_args_enabled?}
72
- def unique_args(args)
73
- return filtered_args(args) if unique_args_enabled?
74
-
75
- args
76
- end
77
-
78
- # Checks if we should disregard the queue when creating the unique digest
79
- # @return [true, false]
80
- def unique_across_queues?
81
- item[UNIQUE_ACROSS_QUEUES_KEY] || worker_options[UNIQUE_ACROSS_QUEUES_KEY] ||
82
- item[UNIQUE_ON_ALL_QUEUES_KEY] || worker_options[UNIQUE_ON_ALL_QUEUES_KEY] # TODO: Remove in v 6.1
83
- end
84
-
85
- # Checks if we should disregard the worker when creating the unique digest
86
- # @return [true, false]
87
- def unique_across_workers?
88
- item[UNIQUE_ACROSS_WORKERS_KEY] || worker_options[UNIQUE_ACROSS_WORKERS_KEY]
89
- end
90
-
91
- # Checks if the worker class has been enabled for unique_args?
92
- # @return [true, false]
93
- def unique_args_enabled?
94
- unique_args_method # && !unique_args_method.is_a?(Boolean)
95
- end
96
-
97
- # Filters unique arguments by proc or symbol
98
- # @param [Array] args the arguments passed to the sidekiq worker
99
- # @return [Array] {#filter_by_proc} when {#unique_args_method} is a Proc
100
- # @return [Array] {#filter_by_symbol} when {#unique_args_method} is a Symbol
101
- # @return [Array] args unfiltered when neither of the above
102
- def filtered_args(args)
103
- return args if args.empty?
104
-
105
- json_args = Normalizer.jsonify(args)
106
-
107
- case unique_args_method
108
- when Proc
109
- filter_by_proc(json_args)
110
- when Symbol
111
- filter_by_symbol(json_args)
112
- else
113
- log_debug("#{__method__} arguments not filtered (using all arguments for uniqueness)")
114
- json_args
115
- end
116
- end
117
-
118
- # Filters unique arguments by proc configured in the sidekiq worker
119
- # @param [Array] args the arguments passed to the sidekiq worker
120
- # @return [Array] with the filtered arguments
121
- def filter_by_proc(args)
122
- unique_args_method.call(args)
123
- end
124
-
125
- # Filters unique arguments by method configured in the sidekiq worker
126
- # @param [Array] args the arguments passed to the sidekiq worker
127
- # @return [Array] unfiltered unless {#worker_method_defined?}
128
- # @return [Array] with the filtered arguments
129
- def filter_by_symbol(args)
130
- return args unless worker_method_defined?(unique_args_method)
131
-
132
- worker_class.send(unique_args_method, args)
133
- rescue ArgumentError => ex
134
- log_fatal(ex)
135
- args
136
- end
137
-
138
- # The method to use for filtering unique arguments
139
- def unique_args_method
140
- @unique_args_method ||= worker_options[UNIQUE_ARGS_KEY]
141
- @unique_args_method ||= :unique_args if worker_method_defined?(:unique_args)
142
- @unique_args_method ||= default_unique_args_method
143
- end
144
-
145
- # The global worker options defined in Sidekiq directly
146
- def default_unique_args_method
147
- Sidekiq.default_worker_options.stringify_keys[UNIQUE_ARGS_KEY]
148
- end
149
- end
150
- end
@@ -1,103 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SidekiqUniqueJobs
4
- # Utility module to help manage unique keys in redis.
5
- # Useful for deleting keys that for whatever reason wasn't deleted
6
- #
7
- # @author Mikael Henriksson <mikael@zoolutions.se>
8
- module Util
9
- DEFAULT_COUNT = 1_000
10
- SCAN_PATTERN = "*"
11
-
12
- include SidekiqUniqueJobs::Logging
13
- include SidekiqUniqueJobs::Connection
14
- extend self
15
-
16
- # Find unique keys in redis
17
- #
18
- # @param [String] pattern a pattern to scan for in redis
19
- # @param [Integer] count the maximum number of keys to delete
20
- # @return [Array<String>] an array with active unique keys
21
- def keys(pattern = SCAN_PATTERN, count = DEFAULT_COUNT)
22
- return redis(&:keys) if pattern.nil?
23
-
24
- redis { |conn| conn.scan_each(match: prefix(pattern), count: count).to_a }
25
- end
26
-
27
- # Find unique keys with ttl
28
- # @param [String] pattern a pattern to scan for in redis
29
- # @param [Integer] count the maximum number of keys to delete
30
- # @return [Hash<String, Integer>] a hash with active unique keys and corresponding ttl
31
- def keys_with_ttl(pattern = SCAN_PATTERN, count = DEFAULT_COUNT)
32
- hash = {}
33
- redis do |conn|
34
- conn.scan_each(match: prefix(pattern), count: count).each do |key|
35
- hash[key] = conn.ttl(key)
36
- end
37
- end
38
- hash
39
- end
40
-
41
- # Deletes unique keys from redis
42
- #
43
- # @param [String] pattern a pattern to scan for in redis
44
- # @param [Integer] count the maximum number of keys to delete
45
- # @return [Integer] the number of keys deleted
46
- def del(pattern = SCAN_PATTERN, count = 0)
47
- raise ArgumentError, "Please provide a number of keys to delete greater than zero" if count.zero?
48
-
49
- pattern = suffix(pattern)
50
-
51
- log_debug { "Deleting keys by: #{pattern}" }
52
- keys, time = timed { keys(pattern, count) }
53
- key_size = keys.size
54
- log_debug { "#{key_size} keys found in #{time} sec." }
55
- _, time = timed { batch_delete(keys) }
56
- log_debug { "Deleted #{key_size} keys in #{time} sec." }
57
-
58
- key_size
59
- end
60
-
61
- private
62
-
63
- def batch_delete(keys)
64
- redis do |conn|
65
- keys.each_slice(500) do |chunk|
66
- conn.pipelined do
67
- chunk.each do |key|
68
- conn.del key
69
- end
70
- end
71
- end
72
- end
73
- end
74
-
75
- def timed
76
- start = current_time
77
- result = yield
78
- elapsed = (current_time - start).round(2)
79
- [result, elapsed]
80
- end
81
-
82
- def current_time
83
- Time.now
84
- end
85
-
86
- def prefix(key)
87
- return key if unique_prefix.nil?
88
- return key if key.start_with?("#{unique_prefix}:")
89
-
90
- "#{unique_prefix}:#{key}"
91
- end
92
-
93
- def suffix(key)
94
- return "#{key}*" unless key.end_with?(":*")
95
-
96
- key
97
- end
98
-
99
- def unique_prefix
100
- SidekiqUniqueJobs.config.unique_prefix
101
- end
102
- end
103
- end
@@ -1,28 +0,0 @@
1
- <header class="row">
2
- <div class="col-sm-5">
3
- <h3><%= t('Unique Digest') %> - <a class="btn btn-default btn-xs" href="<%= root_path %>unique_digests"><%= t('GoBack') %></a></h3>
4
- </div>
5
- </header>
6
-
7
- <% if @unique_keys.size.positive? %>
8
- <div class="table_container">
9
- <table class="table table-striped table-bordered table-hover">
10
- <thead>
11
- <tr>
12
- <th>
13
- <%= t('Keys') %>
14
- <span class="small text-muted">for (<%= @digest %>)</span>
15
- </th>
16
- </tr>
17
- </thead>
18
- <% @unique_keys.each do |key| %>
19
- <tr><td colspan="2"><%= key %></td></tr>
20
- <% end %>
21
- </table>
22
- </div>
23
- <form action="<%= root_path %>unique_digests/<%= @digest %>/delete" method="get">
24
- <%= csrf_tag %>
25
- <a class="btn btn-default btn-xs" href="<%= root_path %>unique_digests"><%= t('GoBack') %></a>
26
- <input class="btn btn-danger btn-xs flip" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
27
- </form>
28
- <% end %>
@@ -1,46 +0,0 @@
1
- <header class="row">
2
- <div class="col-sm-5">
3
- <h3><%= t('Unique Digests') %></h3>
4
- </div>
5
- <form action="<%= root_path %>unique_digests" class="form form-inline" method="get">
6
- <%= csrf_tag %>
7
- <input name="filter" class="form-control" type="text" value="<%= @filter %>" />
8
- <button class="btn btn-default" type="submit"><%= t('Filter') %></button>
9
- </form>
10
- <% if @unique_digests.size > 0 && @total_size > @count.to_i %>
11
- <div class="col-sm-4">
12
- <%= erb unique_template(:_paging), locals: { url: "#{root_path}unique_digests" } %>
13
- </div>
14
- <% end %>
15
- </header>
16
-
17
- <% if @unique_digests.size.positive? %>
18
- <div class="table_container">
19
- <table class="table table-striped table-bordered table-hover">
20
- <thead>
21
- <tr>
22
- <th><%= t('Delete') %></th>
23
- <th><%= t('Digest') %></th>
24
- </tr>
25
- </thead>
26
- <% @unique_digests.each do |digest| %>
27
- <tr>
28
- <td>
29
- <form action="<%= root_path %>unique_digests/<%= digest %>/delete" method="get">
30
- <%= csrf_tag %>
31
- <input name="digest" value="<%= h digest %>" type="hidden" />
32
- <input class="btn btn-danger btn-xs" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
33
- </form>
34
- </td>
35
- <td>
36
- <a href="<%= root_path %>unique_digests/<%= digest %>"><%= digest %></a>
37
- </td>
38
- </tr>
39
- <% end %>
40
- </table>
41
-
42
- <form action="<%= root_path %>unique_digests/delete_all" method="get">
43
- <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
44
- </form>
45
- </div>
46
- <% end %>
@@ -1,21 +0,0 @@
1
- local unique_key = KEYS[1]
2
- local job_id = ARGV[1]
3
- local expires = tonumber(ARGV[2])
4
- local stored_jid = redis.pcall('get', unique_key)
5
-
6
- if stored_jid then
7
- if stored_jid == job_id then
8
- return 1
9
- else
10
- return 0
11
- end
12
- end
13
-
14
- if redis.call('SET', unique_key, job_id, 'nx') then
15
- if expires then
16
- redis.call('EXPIRE', unique_key, expires)
17
- end
18
- return 1
19
- else
20
- return 0
21
- end
@@ -1,13 +0,0 @@
1
- local grabbed_key = KEYS[1]
2
- local unique_digest = KEYS[2]
3
-
4
- local job_id = ARGV[1]
5
- local current_time = tonumber(ARGV[2])
6
-
7
- local old_token = redis.call('GET', unique_digest)
8
- if old_token then
9
- if old_token == job_id or old_token == '2' then
10
- redis.call('DEL', unique_digest)
11
- redis.call('HSET', grabbed_key, job_id, current_time)
12
- end
13
- end
data/redis/delete.lua DELETED
@@ -1,14 +0,0 @@
1
- local exists_key = KEYS[1]
2
- local grabbed_key = KEYS[2]
3
- local available_key = KEYS[3]
4
- local version_key = KEYS[4]
5
- local unique_keys = KEYS[5]
6
- local unique_digest = KEYS[6] -- TODO: Legacy support (Remove in v6.1)
7
-
8
- redis.call('DEL', exists_key)
9
- redis.call('SREM', unique_keys, unique_digest)
10
- redis.call('DEL', grabbed_key)
11
- redis.call('DEL', available_key)
12
- redis.call('DEL', version_key)
13
- redis.call('DEL', 'uniquejobs') -- TODO: Old job hash, just drop the darn thing
14
- redis.call('DEL', unique_digest) -- TODO: Legacy support (Remove in v6.1)
@@ -1,23 +0,0 @@
1
- -- redis.replicate_commands();
2
- local unique_keys = KEYS[1]
3
- local unique_digest = KEYS[2]
4
- local exists_key = KEYS[3]
5
- local grabbed_key = KEYS[4]
6
- local available_key = KEYS[5]
7
- local version_key = KEYS[6]
8
- local run_exists_key = KEYS[7]
9
- local run_grabbed_key = KEYS[8]
10
- local run_available_key = KEYS[9]
11
- local run_version_key = KEYS[10]
12
-
13
- local count = redis.call('SREM', unique_keys, unique_digest)
14
- redis.call('DEL', exists_key)
15
- redis.call('DEL', grabbed_key)
16
- redis.call('DEL', available_key)
17
- redis.call('DEL', version_key)
18
- redis.call('DEL', run_exists_key)
19
- redis.call('DEL', run_grabbed_key)
20
- redis.call('DEL', run_available_key)
21
- redis.call('DEL', run_version_key)
22
-
23
- return count
@@ -1,60 +0,0 @@
1
- local queue = KEYS[1]
2
- local schedule_set = KEYS[2]
3
- local retry_set = KEYS[3]
4
- local unique_digest = ARGV[1]
5
-
6
- local function delete_from_sorted_set(name, digest)
7
- local per = 50
8
- local total = redis.call('zcard', name)
9
- local index = 0
10
- local result
11
- -- redis.log(redis.LOG_DEBUG, "delete_from_sorted_set("..name..","..digest..")")
12
- while (index < total) do
13
- -- redis.log(redis.LOG_DEBUG, "delete_from_sorted_set("..name..","..digest..") - "..index.."-"..per)
14
- local items = redis.call('ZRANGE', name, index, index + per -1)
15
- for _, item in pairs(items) do
16
- -- redis.log(redis.LOG_DEBUG, "delete_from_sorted_set("..name..","..digest..") - current item: " .. item)
17
- if string.find(item, digest) then
18
- -- redis.log(redis.LOG_DEBUG, "delete_from_sorted_set("..name..","..digest..") - deleting item")
19
- redis.call('ZREM', name, item)
20
- result = item
21
- break
22
- end
23
- end
24
- index = index + per
25
- end
26
- return result
27
- end
28
-
29
- local per = 50
30
- local total = redis.call('LLEN', queue)
31
- local index = 0
32
- local result = nil
33
-
34
- -- redis.log(redis.LOG_DEBUG, "delete_job_by_digest.lua - looping through: " .. queue)
35
- while (index < total) do
36
- -- redis.log(redis.LOG_DEBUG, "delete_job_by_digest.lua - " .. index .. "-" .. per)
37
- local items = redis.call('LRANGE', queue, index, index + per -1)
38
- for _, item in pairs(items) do
39
- -- redis.log(redis.LOG_DEBUG, "delete_job_by_digest.lua - item: " .. item)
40
- if string.find(item, unique_digest) then
41
- -- redis.log(redis.LOG_DEBUG, "delete_job_by_digest.lua - found item with digest: " .. unique_digest .. " in: " ..queue)
42
- redis.call('LREM', queue, 1, item)
43
- result = item
44
- break
45
- end
46
- end
47
- index = index + per
48
- end
49
-
50
- if result then
51
- return result
52
- end
53
-
54
- result = delete_from_sorted_set(schedule_set, unique_digest)
55
- if result then
56
- return result
57
- end
58
-
59
- result = delete_from_sorted_set(retry_set, unique_digest)
60
- return result
data/redis/lock.lua DELETED
@@ -1,62 +0,0 @@
1
- -- redis.replicate_commands();
2
-
3
- local exists_key = KEYS[1]
4
- local grabbed_key = KEYS[2]
5
- local available_key = KEYS[3]
6
- local unique_keys = KEYS[4]
7
- local unique_digest = KEYS[5]
8
-
9
- local job_id = ARGV[1]
10
- local ttl = tonumber(ARGV[2])
11
- local lock = ARGV[3]
12
-
13
- local function current_time()
14
- local time = redis.call('time')
15
- local s = time[1]
16
- local ms = time[2]
17
- local number = tonumber((s .. '.' .. ms))
18
-
19
- return number
20
- end
21
-
22
- local stored_token = redis.call('GET', exists_key)
23
- if stored_token and stored_token ~= job_id then
24
- return stored_token
25
- end
26
-
27
- redis.call('SET', exists_key, job_id)
28
-
29
- ----------------------------------------------------------------
30
- -- TODO: Legacy support (Remove in v6.1)
31
- local old_token = redis.call('GET', unique_digest)
32
- if old_token then
33
- if old_token == job_id or old_token == '2' then
34
- -- No need to return, we just delete the old key
35
- redis.call('DEL', unique_digest)
36
- else
37
- return old_token
38
- end
39
- end
40
- ----------------------------------------------------------------
41
-
42
- redis.call('SADD', unique_keys, unique_digest)
43
- redis.call('DEL', grabbed_key)
44
- -- TODO: Move this to LUA when redis 3.2 is the least supported
45
- -- redis.call('HSET', grabbed_key, job_id, current_time())
46
- ---------------------------------------------------------------
47
- redis.call('DEL', available_key)
48
- redis.call('RPUSH', available_key, job_id)
49
-
50
- -- The client should only set ttl for until_expired
51
- -- The server should set ttl for all other jobs
52
- -- redis.log(redis.LOG_WARNING, "lock: " .. lock .. ", ttl: " .. tostring(ttl))
53
- if lock == "until_expired" and ttl and ttl > 0 then
54
- -- We can't keep the key here because it will otherwise never be deleted
55
- redis.call('SREM', unique_keys, unique_digest)
56
-
57
- redis.call('EXPIRE', available_key, ttl)
58
- redis.call('EXPIRE', exists_key, ttl)
59
- redis.call('EXPIRE', unique_digest, ttl)
60
- end
61
-
62
- return job_id
@@ -1,90 +0,0 @@
1
- redis.replicate_commands();
2
-
3
- local exists_key = KEYS[1]
4
- local grabbed_key = KEYS[2]
5
- local available_key = KEYS[3]
6
- local release_key = KEYS[4]
7
- local version_key = KEYS[5]
8
-
9
- local expires_in = tonumber(ARGV[1])
10
- local stale_client_timeout = tonumber(ARGV[2])
11
- local expiration = tonumber(ARGV[3])
12
-
13
- local function current_time()
14
- local time = redis.call('time')
15
- local s = time[1]
16
- local ms = time[2]
17
- local number = tonumber((s .. '.' .. ms))
18
-
19
- return number
20
- end
21
-
22
- local hgetall = function (key)
23
- local bulk = redis.call('HGETALL', key)
24
- local result = {}
25
- local nextkey
26
- for i, v in ipairs(bulk) do
27
- if i % 2 == 1 then
28
- nextkey = v
29
- else
30
- result[nextkey] = v
31
- end
32
- end
33
- return result
34
- end
35
-
36
- local cached_current_time = current_time()
37
- redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - started at : " .. cached_current_time)
38
-
39
- local my_lock_expires_at = cached_current_time + expires_in + 1
40
- redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - my_lock_expires_at: " .. my_lock_expires_at)
41
-
42
- if not redis.call('SETNX', release_key, my_lock_expires_at) then
43
- -- Check if expired
44
- local other_lock_expires_at = tonumber(redis.call('GET', release_key))
45
- redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - other_lock_expires_at: " .. other_lock_expires_at)
46
-
47
- if other_lock_expires_at < cached_current_time then
48
- local old_expires_at = tonumber(redis.call('GETSET', release_key, my_lock_expires_at))
49
- redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - old_expires_at: " .. old_expires_at)
50
-
51
- -- Check if another client started cleanup yet. If not,
52
- -- then we now have the lock.
53
- if not old_expires_at == other_lock_expires_at then
54
- redis.log(redis.LOG_DEBUG, "release_stale_locks.lua -could not retrieve lock: exiting 0")
55
- return 0
56
- end
57
- end
58
- end
59
-
60
- local keys = hgetall(grabbed_key)
61
- for key, locked_at in pairs(keys) do
62
- local timed_out_at = tonumber(locked_at) + stale_client_timeout
63
- redis.log(redis.LOG_DEBUG,
64
- "release_stale_locks.lua - processing: " .. grabbed_key .. " key: " .. key .. " locked_at: " .. locked_at
65
- )
66
-
67
- if timed_out_at < current_time() then
68
- redis.log(redis.LOG_DEBUG, "HDEL " .. grabbed_key .. ":" .. key)
69
- redis.call('HDEL', grabbed_key, key)
70
- redis.log(redis.LOG_DEBUG, "LPUSH " .. available_key .. ":" .. key)
71
- redis.call('LPUSH', available_key, key)
72
-
73
- if expiration then
74
- redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - EXPIRE " .. available_key .. " with " .. expiration)
75
- redis.call('EXPIRE', available_key, expiration)
76
- redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - EXPIRE " .. exists_key .. " with " .. expiration)
77
- redis.call('EXPIRE', exists_key, expiration)
78
- end
79
- end
80
- end
81
-
82
- -- Make sure not to delete the lock in case someone else already expired
83
- -- our lock, with one second in between to account for some lag.
84
- if my_lock_expires_at > (current_time() - 1) then
85
- redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - DEL " .. release_key)
86
- redis.call('DEL', release_key)
87
- end
88
-
89
- redis.log(redis.LOG_DEBUG, "release_stale_locks.lua - comleted at : " .. current_time())
90
- return 1