sidekiq-unique-jobs 3.0.11 → 8.0.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (158) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +2163 -25
  3. data/LICENSE.txt +21 -0
  4. data/README.md +984 -47
  5. data/bin/uniquejobs +7 -0
  6. data/lib/sidekiq-unique-jobs.rb +2 -36
  7. data/lib/sidekiq_unique_jobs/batch_delete.rb +120 -0
  8. data/lib/sidekiq_unique_jobs/changelog.rb +68 -0
  9. data/lib/sidekiq_unique_jobs/cli.rb +95 -0
  10. data/lib/sidekiq_unique_jobs/config.rb +306 -33
  11. data/lib/sidekiq_unique_jobs/connection.rb +20 -0
  12. data/lib/sidekiq_unique_jobs/constants.rb +55 -0
  13. data/lib/sidekiq_unique_jobs/core_ext.rb +132 -0
  14. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  15. data/lib/sidekiq_unique_jobs/digests.rb +134 -0
  16. data/lib/sidekiq_unique_jobs/exceptions.rb +105 -0
  17. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  18. data/lib/sidekiq_unique_jobs/job.rb +63 -0
  19. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  20. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  21. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +165 -0
  22. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  23. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  24. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +71 -0
  25. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +48 -0
  26. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +43 -0
  27. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +42 -0
  28. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  29. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +70 -0
  30. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +21 -0
  31. data/lib/sidekiq_unique_jobs/lock.rb +348 -0
  32. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  33. data/lib/sidekiq_unique_jobs/lock_config.rb +132 -0
  34. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  35. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  36. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  37. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  38. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  39. data/lib/sidekiq_unique_jobs/locksmith.rb +390 -0
  40. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  41. data/lib/sidekiq_unique_jobs/logging.rb +236 -0
  42. data/lib/sidekiq_unique_jobs/lua/delete.lua +49 -0
  43. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +39 -0
  44. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  45. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  46. data/lib/sidekiq_unique_jobs/lua/lock.lua +108 -0
  47. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  48. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  49. data/lib/sidekiq_unique_jobs/lua/queue.lua +88 -0
  50. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +119 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +35 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +29 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  57. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  58. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  59. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  60. data/lib/sidekiq_unique_jobs/lua/unlock.lua +112 -0
  61. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  62. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +66 -0
  63. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  64. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  65. data/lib/sidekiq_unique_jobs/middleware.rb +41 -15
  66. data/lib/sidekiq_unique_jobs/normalizer.rb +17 -0
  67. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +24 -0
  68. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +16 -0
  69. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +17 -0
  70. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +75 -0
  71. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +82 -0
  72. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +39 -0
  73. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +51 -0
  74. data/lib/sidekiq_unique_jobs/on_conflict.rb +44 -0
  75. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +78 -0
  76. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  77. data/lib/sidekiq_unique_jobs/orphans/manager.rb +242 -0
  78. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  79. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  80. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +115 -0
  81. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  82. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +313 -0
  83. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  84. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  85. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  86. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  87. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +102 -0
  88. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  89. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  90. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  92. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  93. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  94. data/lib/sidekiq_unique_jobs/script/caller.rb +133 -0
  95. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  96. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  97. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  98. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  99. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  100. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  101. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  102. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  103. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  104. data/lib/sidekiq_unique_jobs/script.rb +46 -0
  105. data/lib/sidekiq_unique_jobs/server.rb +62 -0
  106. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +110 -37
  107. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +304 -0
  108. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +84 -0
  109. data/lib/sidekiq_unique_jobs/testing.rb +132 -9
  110. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  111. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  112. data/lib/sidekiq_unique_jobs/unlockable.rb +43 -0
  113. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  114. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +152 -0
  115. data/lib/sidekiq_unique_jobs/version.rb +5 -1
  116. data/lib/sidekiq_unique_jobs/version_check.rb +114 -0
  117. data/lib/sidekiq_unique_jobs/web/helpers.rb +175 -0
  118. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +10 -0
  119. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +60 -0
  120. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  121. data/lib/sidekiq_unique_jobs/web/views/locks.erb +59 -0
  122. data/lib/sidekiq_unique_jobs/web.rb +109 -0
  123. data/lib/sidekiq_unique_jobs.rb +83 -0
  124. data/lib/tasks/changelog.rake +23 -0
  125. metadata +157 -126
  126. data/.gitignore +0 -10
  127. data/.rspec +0 -3
  128. data/.rubocop.yml +0 -36
  129. data/.travis.yml +0 -25
  130. data/Appraisals +0 -20
  131. data/Gemfile +0 -5
  132. data/LICENSE +0 -22
  133. data/Rakefile +0 -11
  134. data/gemfiles/sidekiq_2.15.gemfile +0 -9
  135. data/gemfiles/sidekiq_2.16.gemfile +0 -9
  136. data/gemfiles/sidekiq_2.17.gemfile +0 -9
  137. data/gemfiles/sidekiq_3.0.gemfile +0 -9
  138. data/gemfiles/sidekiq_develop.gemfile +0 -9
  139. data/lib/sidekiq_unique_jobs/connectors/redis_pool.rb +0 -11
  140. data/lib/sidekiq_unique_jobs/connectors/sidekiq_redis.rb +0 -9
  141. data/lib/sidekiq_unique_jobs/connectors/testing.rb +0 -11
  142. data/lib/sidekiq_unique_jobs/connectors.rb +0 -16
  143. data/lib/sidekiq_unique_jobs/middleware/client/strategies/testing_inline.rb +0 -25
  144. data/lib/sidekiq_unique_jobs/middleware/client/strategies/unique.rb +0 -76
  145. data/lib/sidekiq_unique_jobs/middleware/client/unique_jobs.rb +0 -39
  146. data/lib/sidekiq_unique_jobs/middleware/server/unique_jobs.rb +0 -69
  147. data/lib/sidekiq_unique_jobs/payload_helper.rb +0 -42
  148. data/sidekiq-unique-jobs.gemspec +0 -27
  149. data/spec/lib/.sidekiq_testing_enabled_spec.rb.swp +0 -0
  150. data/spec/lib/client_spec.rb +0 -173
  151. data/spec/lib/middleware/server/unique_jobs_spec.rb +0 -81
  152. data/spec/lib/sidekiq_testing_enabled_spec.rb +0 -123
  153. data/spec/lib/sidekiq_unique_ext_spec.rb +0 -70
  154. data/spec/lib/unlock_order_spec.rb +0 -64
  155. data/spec/spec_helper.rb +0 -37
  156. data/spec/support/my_worker.rb +0 -13
  157. data/spec/support/sidekiq_meta.rb +0 -17
  158. data/spec/support/unique_worker.rb +0 -13
@@ -0,0 +1,236 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Utility module for reducing the number of uses of logger.
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ module Logging
8
+ def self.included(base)
9
+ base.send(:extend, self)
10
+ end
11
+
12
+ #
13
+ # A convenience method for using the configured gem logger
14
+ #
15
+ # @see SidekiqUniqueJobs#.logger
16
+ #
17
+ # @return [Logger]
18
+ #
19
+ def logger
20
+ SidekiqUniqueJobs.logger
21
+ end
22
+
23
+ #
24
+ # Logs a message at debug level
25
+ #
26
+ # @param [String, Exception] message_or_exception the message or exception to log
27
+ #
28
+ # @return [void]
29
+ #
30
+ # @yield [String, Exception] the message or exception to use for log message
31
+ #
32
+ def log_debug(message_or_exception = nil, item = nil, &block)
33
+ return unless logging?
34
+
35
+ message = build_message(message_or_exception, item)
36
+ logger.debug(message, &block)
37
+ nil
38
+ end
39
+
40
+ #
41
+ # Logs a message at info level
42
+ #
43
+ # @param [String, Exception] message_or_exception the message or exception to log
44
+ #
45
+ # @return [void]
46
+ #
47
+ # @yield [String, Exception] the message or exception to use for log message
48
+ #
49
+ def log_info(message_or_exception = nil, item = nil, &block)
50
+ return unless logging?
51
+
52
+ message = build_message(message_or_exception, item)
53
+ logger.info(message, &block)
54
+ nil
55
+ end
56
+
57
+ #
58
+ # Logs a message at warn level
59
+ #
60
+ # @param [String, Exception] message_or_exception the message or exception to log
61
+ #
62
+ # @return [void]
63
+ #
64
+ # @yield [String, Exception] the message or exception to use for log message
65
+ #
66
+ def log_warn(message_or_exception = nil, item = nil, &block)
67
+ return unless logging?
68
+
69
+ message = build_message(message_or_exception, item)
70
+ logger.warn(message, &block)
71
+ nil
72
+ end
73
+
74
+ #
75
+ # Logs a message at error level
76
+ #
77
+ # @param [String, Exception] message_or_exception the message or exception to log
78
+ #
79
+ # @return [void]
80
+ #
81
+ # @yield [String, Exception] the message or exception to use for log message
82
+ #
83
+ def log_error(message_or_exception = nil, item = nil, &block)
84
+ return unless logging?
85
+
86
+ message = build_message(message_or_exception, item)
87
+ logger.error(message, &block)
88
+ nil
89
+ end
90
+
91
+ #
92
+ # Logs a message at fatal level
93
+ #
94
+ # @param [String, Exception] message_or_exception the message or exception to log
95
+ #
96
+ # @return [void]
97
+ #
98
+ # @yield [String, Exception] the message or exception to use for log message
99
+ #
100
+ def log_fatal(message_or_exception = nil, item = nil, &block)
101
+ return unless logging?
102
+
103
+ message = build_message(message_or_exception, item)
104
+ logger.fatal(message, &block)
105
+
106
+ nil
107
+ end
108
+
109
+ #
110
+ # Build a log message
111
+ #
112
+ # @param [String, Exception] message_or_exception an entry to log
113
+ # @param [Hash] item the sidekiq job hash
114
+ #
115
+ # @return [String] a complete log entry
116
+ #
117
+ def build_message(message_or_exception, item = nil)
118
+ return nil if message_or_exception.nil?
119
+ return message_or_exception if item.nil?
120
+
121
+ message = message_or_exception.dup
122
+ details = item.slice(LOCK, QUEUE, CLASS, JID, LOCK_DIGEST).each_with_object([]) do |(key, value), memo|
123
+ memo << "#{key}=#{value}"
124
+ end
125
+ message << " ("
126
+ message << details.join(" ")
127
+ message << ")"
128
+
129
+ message
130
+ end
131
+
132
+ #
133
+ # Wraps the middleware logic with context aware logging
134
+ #
135
+ #
136
+ # @return [void]
137
+ #
138
+ # @yieldreturn [void] yield to the middleware instance
139
+ #
140
+ def with_logging_context
141
+ with_configured_loggers_context do
142
+ return yield
143
+ end
144
+
145
+ nil # Need to make sure we don't return anything here
146
+ end
147
+
148
+ #
149
+ # Attempt to setup context aware logging for the given logger
150
+ #
151
+ #
152
+ # @return [void]
153
+ #
154
+ # @yield
155
+ #
156
+ def with_configured_loggers_context(&block)
157
+ logger_method.call(logging_context, &block)
158
+ end
159
+
160
+ #
161
+ # Setup some variables to add to each log line
162
+ #
163
+ #
164
+ # @return [Hash] the context to use for each log line
165
+ #
166
+ def logging_context
167
+ raise NotImplementedError, "#{__method__} needs to be implemented in #{self.class}"
168
+ end
169
+
170
+ private
171
+
172
+ #
173
+ # A memoized method to use for setting up a logging context
174
+ #
175
+ #
176
+ # @return [proc] the method to call
177
+ #
178
+ def logger_method
179
+ @logger_method ||= sidekiq_context_method
180
+ @logger_method ||= sidekiq_logger_context_method
181
+ @logger_method ||= sidekiq_logging_context_method
182
+ @logger_method ||= no_sidekiq_context_method
183
+ end
184
+
185
+ #
186
+ # Checks if the logger respond to `with_context`.
187
+ #
188
+ # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
189
+ #
190
+ #
191
+ # @return [true,false]
192
+ #
193
+ def logger_respond_to_with_context?
194
+ logger.respond_to?(:with_context)
195
+ end
196
+
197
+ #
198
+ # Checks if the logger context takes a hash argument
199
+ #
200
+ # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
201
+ #
202
+ #
203
+ # @return [true,false]
204
+ #
205
+ def logger_context_hash?
206
+ defined?(Sidekiq::Context) || logger_respond_to_with_context?
207
+ end
208
+
209
+ def sidekiq_context_method
210
+ Sidekiq::Context.method(:with) if defined?(Sidekiq::Context)
211
+ end
212
+
213
+ def sidekiq_logger_context_method
214
+ logger.method(:with_context) if logger_respond_to_with_context?
215
+ end
216
+
217
+ def sidekiq_logging_context_method
218
+ Sidekiq::Logging.method(:with_context) if defined?(Sidekiq::Logging)
219
+ end
220
+
221
+ def no_sidekiq_context_method
222
+ method(:fake_logger_context)
223
+ end
224
+
225
+ def fake_logger_context(_context)
226
+ logger.warn "Don't know how to setup the logging context. Please open a feature request:" \
227
+ " https://github.com/mhenrixon/sidekiq-unique-jobs/issues/new?template=feature_request.md"
228
+
229
+ yield
230
+ end
231
+
232
+ def logging?
233
+ SidekiqUniqueJobs.logging?
234
+ end
235
+ end
236
+ end
@@ -0,0 +1,49 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local info = KEYS[5]
7
+ local changelog = KEYS[6]
8
+ local digests = KEYS[7]
9
+ -------- END keys ---------
10
+
11
+ -------- BEGIN lock arguments ---------
12
+ local job_id = ARGV[1]
13
+ local pttl = tonumber(ARGV[2])
14
+ local lock_type = ARGV[3]
15
+ local limit = tonumber(ARGV[4])
16
+ local lock_score = ARGV[5]
17
+ -------- END lock arguments -----------
18
+
19
+ -------- BEGIN injected arguments --------
20
+ local current_time = tonumber(ARGV[6])
21
+ local debug_lua = tostring(ARGV[7]) == "1"
22
+ local max_history = tonumber(ARGV[8])
23
+ local script_name = tostring(ARGV[9]) .. ".lua"
24
+ local redisversion = tostring(ARGV[10])
25
+ --------- END injected arguments ---------
26
+
27
+ -------- BEGIN local functions --------
28
+ <%= include_partial "shared/_common.lua" %>
29
+ ---------- END local functions ----------
30
+
31
+
32
+ -------- BEGIN delete.lua --------
33
+ log_debug("BEGIN delete", digest)
34
+
35
+ local redis_version = toversion(redisversion)
36
+ local count = 0
37
+
38
+ log_debug("ZREM", digests, digest)
39
+ count = count + redis.call("ZREM", digests, digest)
40
+
41
+ log_debug("UNLINK", digest, queued, primed, locked, info)
42
+ count = count + redis.call("UNLINK", digest, queued, primed, locked, info)
43
+
44
+
45
+ log("Deleted (" .. count .. ") keys")
46
+ log_debug("END delete (" .. count .. ") keys for:", digest)
47
+
48
+ return count
49
+ -------- END delete.lua --------
@@ -0,0 +1,39 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local run_digest = KEYS[5]
7
+ local run_queued = KEYS[6]
8
+ local run_primed = KEYS[7]
9
+ local run_locked = KEYS[8]
10
+ local digests = KEYS[9]
11
+ -------- END keys ---------
12
+
13
+ -------- BEGIN injected arguments --------
14
+ local current_time = tonumber(ARGV[1])
15
+ local debug_lua = tostring(ARGV[2]) == "1"
16
+ local max_history = tonumber(ARGV[3])
17
+ local script_name = tostring(ARGV[4]) .. ".lua"
18
+ local redisversion = tostring(ARGV[5])
19
+ --------- END injected arguments ---------
20
+
21
+ -------- BEGIN local functions --------
22
+ <%= include_partial "shared/_common.lua" %>
23
+ ---------- END local functions ----------
24
+
25
+ -------- BEGIN delete_by_digest.lua --------
26
+ local counter = 0
27
+ local redis_version = toversion(redisversion)
28
+
29
+ log_debug("BEGIN delete_by_digest:", digest)
30
+
31
+ log_debug("UNLINK", digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
32
+ counter = redis.call("UNLINK", digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
33
+
34
+ log_debug("ZREM", digests, digest)
35
+ redis.call("ZREM", digests, digest)
36
+
37
+ log_debug("END delete_by_digest:", digest, "(deleted " .. counter .. " keys)")
38
+ return counter
39
+ -------- END delete_by_digest.lua --------
@@ -0,0 +1,38 @@
1
+ -------- BEGIN keys ---------
2
+ local queue = KEYS[1]
3
+ local schedule_set = KEYS[2]
4
+ local retry_set = KEYS[3]
5
+ -------- END keys ---------
6
+
7
+ -------- BEGIN Arguments ---------
8
+ local digest = ARGV[1]
9
+ -------- END Arguments ---------
10
+
11
+ -------- BEGIN injected arguments --------
12
+ local current_time = tonumber(ARGV[2])
13
+ local debug_lua = tostring(ARGV[3]) == "1"
14
+ local max_history = tonumber(ARGV[4])
15
+ local script_name = tostring(ARGV[5]) .. ".lua"
16
+ --------- END injected arguments ---------
17
+
18
+ -------- BEGIN local functions --------
19
+ <%= include_partial "shared/_common.lua" %>
20
+ <%= include_partial "shared/_delete_from_queue.lua" %>
21
+ <%= include_partial "shared/_delete_from_sorted_set.lua" %>
22
+ ---------- END local functions ----------
23
+
24
+
25
+ -------- BEGIN delete_job_by_digest.lua --------
26
+ local result = delete_from_queue(queue, digest)
27
+ if result then
28
+ return result
29
+ end
30
+
31
+ result = delete_from_sorted_set(schedule_set, digest)
32
+ if result then
33
+ return result
34
+ end
35
+
36
+ result = delete_from_sorted_set(retry_set, digest)
37
+ return result
38
+ -------- END delete_job_by_digest.lua --------
@@ -0,0 +1,26 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ -------- END keys ---------
4
+
5
+ -------- BEGIN injected arguments --------
6
+ local current_time = tonumber(ARGV[2])
7
+ local debug_lua = tostring(ARGV[3]) == "1"
8
+ local max_history = tonumber(ARGV[4])
9
+ local script_name = tostring(ARGV[5]) .. ".lua"
10
+ --------- END injected arguments ---------
11
+
12
+
13
+ -------- BEGIN local functions --------
14
+ <%= include_partial "shared/_common.lua" %>
15
+ <%= include_partial "shared/_find_digest_in_queues.lua" %>
16
+ ---------- END local functions ----------
17
+
18
+
19
+ -------- BEGIN delete_orphaned.lua --------
20
+ log_debug("BEGIN")
21
+ local result = find_digest_in_queues(digest)
22
+ log_debug("END")
23
+ if result and result ~= nil then
24
+ return result
25
+ end
26
+ -------- END delete_orphaned.lua --------
@@ -0,0 +1,108 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local info = KEYS[5]
7
+ local changelog = KEYS[6]
8
+ local digests = KEYS[7]
9
+ local expiring_digests = KEYS[8]
10
+ -------- END keys ---------
11
+
12
+
13
+ -------- BEGIN lock arguments ---------
14
+ local job_id = ARGV[1]
15
+ local pttl = tonumber(ARGV[2])
16
+ local lock_type = ARGV[3]
17
+ local limit = tonumber(ARGV[4])
18
+ local lock_score = ARGV[5]
19
+ -------- END lock arguments -----------
20
+
21
+
22
+ -------- BEGIN injected arguments --------
23
+ local current_time = tonumber(ARGV[6])
24
+ local debug_lua = tostring(ARGV[7]) == "1"
25
+ local max_history = tonumber(ARGV[8])
26
+ local script_name = tostring(ARGV[9]) .. ".lua"
27
+ local redisversion = ARGV[10]
28
+ --------- END injected arguments ---------
29
+
30
+
31
+ -------- BEGIN local functions --------
32
+ <%= include_partial "shared/_common.lua" %>
33
+ ---------- END local functions ----------
34
+
35
+
36
+ --------- BEGIN lock.lua ---------
37
+ log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
38
+
39
+ if redis.call("HEXISTS", locked, job_id) == 1 then
40
+ log_debug(locked, "already locked with job_id:", job_id)
41
+ log("Duplicate")
42
+
43
+ log_debug("LREM", queued, -1, job_id)
44
+ redis.call("LREM", queued, -1, job_id)
45
+
46
+ log_debug("LREM", primed, 1, job_id)
47
+ redis.call("LREM", primed, 1, job_id)
48
+
49
+ return job_id
50
+ end
51
+
52
+ local locked_count = redis.call("HLEN", locked)
53
+ local within_limit = limit > locked_count
54
+ local limit_exceeded = not within_limit
55
+
56
+ if limit_exceeded then
57
+ log_debug("Limit exceeded:", digest, "(", locked_count, "of", limit, ")")
58
+ log("Limited")
59
+ return nil
60
+ end
61
+
62
+ if lock_type == "until_expired" and pttl and pttl > 0 then
63
+ log_debug("ZADD", expiring_digests, current_time + pttl, digest)
64
+ redis.call("ZADD", expiring_digests, current_time + pttl, digest)
65
+ else
66
+ local score
67
+
68
+ if #lock_score == 0 then
69
+ score = current_time
70
+ else
71
+ score = lock_score
72
+ end
73
+
74
+ log_debug("ZADD", digests, score, digest)
75
+ redis.call("ZADD", digests, score, digest)
76
+ end
77
+
78
+ log_debug("HSET", locked, job_id, current_time)
79
+ redis.call("HSET", locked, job_id, current_time)
80
+
81
+ log_debug("LREM", queued, -1, job_id)
82
+ redis.call("LREM", queued, -1, job_id)
83
+
84
+ log_debug("LREM", primed, 1, job_id)
85
+ redis.call("LREM", primed, 1, job_id)
86
+
87
+ -- The Sidekiq client sets pttl
88
+ if pttl and pttl > 0 then
89
+ log_debug("PEXPIRE", digest, pttl)
90
+ redis.call("PEXPIRE", digest, pttl)
91
+
92
+ log_debug("PEXPIRE", locked, pttl)
93
+ redis.call("PEXPIRE", locked, pttl)
94
+
95
+ log_debug("PEXPIRE", info, pttl)
96
+ redis.call("PEXPIRE", info, pttl)
97
+ end
98
+
99
+ log_debug("PEXPIRE", queued, 1000)
100
+ redis.call("PEXPIRE", queued, 1000)
101
+
102
+ log_debug("PEXPIRE", primed, 1000)
103
+ redis.call("PEXPIRE", primed, 1000)
104
+
105
+ log("Locked")
106
+ log_debug("END lock digest:", digest, "job_id:", job_id)
107
+ return job_id
108
+ ---------- END lock.lua ----------
@@ -0,0 +1,92 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local info = KEYS[5]
7
+ local changelog = KEYS[6]
8
+ local digests = KEYS[7]
9
+ local expiring_digests = KEYS[8]
10
+ -------- END keys ---------
11
+
12
+
13
+ -------- BEGIN lock arguments ---------
14
+ local job_id = ARGV[1]
15
+ local pttl = tonumber(ARGV[2])
16
+ local lock_type = ARGV[3]
17
+ local limit = tonumber(ARGV[4])
18
+ -------- END lock arguments -----------
19
+
20
+
21
+ -------- BEGIN injected arguments --------
22
+ local current_time = tonumber(ARGV[5])
23
+ local debug_lua = tostring(ARGV[6]) == "1"
24
+ local max_history = tonumber(ARGV[7])
25
+ local script_name = tostring(ARGV[8]) .. ".lua"
26
+ local redisversion = ARGV[9]
27
+ --------- END injected arguments ---------
28
+
29
+
30
+ -------- BEGIN local functions --------
31
+ <%= include_partial "shared/_common.lua" %>
32
+ ---------- END local functions ----------
33
+
34
+
35
+ --------- BEGIN lock.lua ---------
36
+ log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
37
+
38
+ if redis.call("HEXISTS", locked, job_id) == 1 then
39
+ log_debug(locked, "already locked with job_id:", job_id)
40
+ log("Duplicate")
41
+
42
+ log_debug("LREM", queued, -1, job_id)
43
+ redis.call("LREM", queued, -1, job_id)
44
+
45
+ log_debug("LREM", primed, 1, job_id)
46
+ redis.call("LREM", primed, 1, job_id)
47
+
48
+ return job_id
49
+ end
50
+
51
+ local locked_count = redis.call("HLEN", locked)
52
+ local within_limit = limit > locked_count
53
+ local limit_exceeded = not within_limit
54
+
55
+ if limit_exceeded then
56
+ log_debug("Limit exceeded:", digest, "(", locked_count, "of", limit, ")")
57
+ log("Limited")
58
+ return nil
59
+ end
60
+
61
+ log_debug("ZADD", expiring_digests, current_time + pttl, digest)
62
+ redis.call("ZADD", expiring_digests, current_time + pttl, digest)
63
+
64
+ log_debug("HSET", locked, job_id, current_time)
65
+ redis.call("HSET", locked, job_id, current_time)
66
+
67
+ log_debug("LREM", queued, -1, job_id)
68
+ redis.call("LREM", queued, -1, job_id)
69
+
70
+ log_debug("LREM", primed, 1, job_id)
71
+ redis.call("LREM", primed, 1, job_id)
72
+
73
+ -- The Sidekiq client sets pttl
74
+ log_debug("PEXPIRE", digest, pttl)
75
+ redis.call("PEXPIRE", digest, pttl)
76
+
77
+ log_debug("PEXPIRE", locked, pttl)
78
+ redis.call("PEXPIRE", locked, pttl)
79
+
80
+ log_debug("PEXPIRE", info, pttl)
81
+ redis.call("PEXPIRE", info, pttl)
82
+
83
+ log_debug("PEXPIRE", queued, 1000)
84
+ redis.call("PEXPIRE", queued, 1000)
85
+
86
+ log_debug("PEXPIRE", primed, 1000)
87
+ redis.call("PEXPIRE", primed, 1000)
88
+
89
+ log("Locked")
90
+ log_debug("END lock digest:", digest, "job_id:", job_id)
91
+ return job_id
92
+ ---------- END lock.lua ----------
@@ -0,0 +1,35 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local info = KEYS[5]
7
+ local changelog = KEYS[6]
8
+ local digests = KEYS[7]
9
+ -------- END keys ---------
10
+
11
+ -------- BEGIN lock arguments ---------
12
+ local job_id = ARGV[1]
13
+ -------- END lock arguments -----------
14
+
15
+ -------- BEGIN injected arguments --------
16
+ local current_time = tonumber(ARGV[2])
17
+ local debug_lua = tostring(ARGV[3]) == "1"
18
+ local max_history = tonumber(ARGV[4])
19
+ local script_name = tostring(ARGV[5]) .. ".lua"
20
+ --------- END injected arguments ---------
21
+
22
+ -------- BEGIN local functions --------
23
+ <%= include_partial "shared/_common.lua" %>
24
+ --------- END local functions ---------
25
+
26
+
27
+ -------- BEGIN locked.lua --------
28
+ if redis.call("HEXISTS", locked, job_id) == 1 then
29
+ log_debug("Locked", digest, "job_id:", job_id)
30
+ return 1
31
+ else
32
+ log_debug("NOT Locked", digest, "job_id:", job_id)
33
+ return -1
34
+ end
35
+ --------- END locked.lua ---------