sidekiq-unique-jobs 6.0.25 → 7.1.33

Sign up to get free protection for your applications and to get access to all the features.
Files changed (130) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1157 -41
  3. data/README.md +825 -291
  4. data/lib/sidekiq_unique_jobs/batch_delete.rb +124 -0
  5. data/lib/sidekiq_unique_jobs/changelog.rb +78 -0
  6. data/lib/sidekiq_unique_jobs/cli.rb +57 -29
  7. data/lib/sidekiq_unique_jobs/config.rb +319 -0
  8. data/lib/sidekiq_unique_jobs/connection.rb +6 -5
  9. data/lib/sidekiq_unique_jobs/constants.rb +46 -25
  10. data/lib/sidekiq_unique_jobs/core_ext.rb +80 -0
  11. data/lib/sidekiq_unique_jobs/deprecation.rb +65 -0
  12. data/lib/sidekiq_unique_jobs/digests.rb +70 -102
  13. data/lib/sidekiq_unique_jobs/exceptions.rb +88 -12
  14. data/lib/sidekiq_unique_jobs/expiring_digests.rb +14 -0
  15. data/lib/sidekiq_unique_jobs/job.rb +46 -12
  16. data/lib/sidekiq_unique_jobs/json.rb +47 -0
  17. data/lib/sidekiq_unique_jobs/key.rb +98 -0
  18. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +111 -82
  19. data/lib/sidekiq_unique_jobs/lock/client_validator.rb +28 -0
  20. data/lib/sidekiq_unique_jobs/lock/server_validator.rb +27 -0
  21. data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb +40 -15
  22. data/lib/sidekiq_unique_jobs/lock/until_executed.rb +30 -7
  23. data/lib/sidekiq_unique_jobs/lock/until_executing.rb +26 -2
  24. data/lib/sidekiq_unique_jobs/lock/until_expired.rb +27 -15
  25. data/lib/sidekiq_unique_jobs/lock/validator.rb +96 -0
  26. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +26 -12
  27. data/lib/sidekiq_unique_jobs/lock/while_executing_reject.rb +3 -3
  28. data/lib/sidekiq_unique_jobs/lock.rb +342 -0
  29. data/lib/sidekiq_unique_jobs/lock_args.rb +127 -0
  30. data/lib/sidekiq_unique_jobs/lock_config.rb +126 -0
  31. data/lib/sidekiq_unique_jobs/lock_digest.rb +79 -0
  32. data/lib/sidekiq_unique_jobs/lock_info.rb +68 -0
  33. data/lib/sidekiq_unique_jobs/lock_timeout.rb +62 -0
  34. data/lib/sidekiq_unique_jobs/lock_ttl.rb +77 -0
  35. data/lib/sidekiq_unique_jobs/lock_type.rb +37 -0
  36. data/lib/sidekiq_unique_jobs/locksmith.rb +305 -101
  37. data/lib/sidekiq_unique_jobs/logging/middleware_context.rb +44 -0
  38. data/lib/sidekiq_unique_jobs/logging.rb +202 -33
  39. data/lib/sidekiq_unique_jobs/lua/delete.lua +51 -0
  40. data/lib/sidekiq_unique_jobs/lua/delete_by_digest.lua +42 -0
  41. data/lib/sidekiq_unique_jobs/lua/delete_job_by_digest.lua +38 -0
  42. data/lib/sidekiq_unique_jobs/lua/find_digest_in_queues.lua +26 -0
  43. data/lib/sidekiq_unique_jobs/lua/lock.lua +99 -0
  44. data/lib/sidekiq_unique_jobs/lua/lock_until_expired.lua +92 -0
  45. data/lib/sidekiq_unique_jobs/lua/locked.lua +35 -0
  46. data/lib/sidekiq_unique_jobs/lua/queue.lua +87 -0
  47. data/lib/sidekiq_unique_jobs/lua/reap_orphans.lua +122 -0
  48. data/lib/sidekiq_unique_jobs/lua/shared/_common.lua +40 -0
  49. data/lib/sidekiq_unique_jobs/lua/shared/_current_time.lua +8 -0
  50. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_queue.lua +22 -0
  51. data/lib/sidekiq_unique_jobs/lua/shared/_delete_from_sorted_set.lua +18 -0
  52. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_process_set.lua +53 -0
  53. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_queues.lua +43 -0
  54. data/lib/sidekiq_unique_jobs/lua/shared/_find_digest_in_sorted_set.lua +24 -0
  55. data/lib/sidekiq_unique_jobs/lua/shared/_hgetall.lua +13 -0
  56. data/lib/sidekiq_unique_jobs/lua/shared/_upgrades.lua +3 -0
  57. data/lib/sidekiq_unique_jobs/lua/unlock.lua +107 -0
  58. data/lib/sidekiq_unique_jobs/lua/update_version.lua +40 -0
  59. data/lib/sidekiq_unique_jobs/lua/upgrade.lua +68 -0
  60. data/lib/sidekiq_unique_jobs/middleware/client.rb +42 -0
  61. data/lib/sidekiq_unique_jobs/middleware/server.rb +31 -0
  62. data/lib/sidekiq_unique_jobs/middleware.rb +29 -43
  63. data/lib/sidekiq_unique_jobs/normalizer.rb +4 -4
  64. data/lib/sidekiq_unique_jobs/on_conflict/log.rb +9 -5
  65. data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb +1 -1
  66. data/lib/sidekiq_unique_jobs/on_conflict/raise.rb +1 -1
  67. data/lib/sidekiq_unique_jobs/on_conflict/reject.rb +63 -17
  68. data/lib/sidekiq_unique_jobs/on_conflict/replace.rb +54 -14
  69. data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb +16 -5
  70. data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb +25 -6
  71. data/lib/sidekiq_unique_jobs/on_conflict.rb +23 -10
  72. data/lib/sidekiq_unique_jobs/options_with_fallback.rb +39 -36
  73. data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb +29 -0
  74. data/lib/sidekiq_unique_jobs/orphans/manager.rb +241 -0
  75. data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb +24 -0
  76. data/lib/sidekiq_unique_jobs/orphans/observer.rb +42 -0
  77. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +114 -0
  78. data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb +170 -0
  79. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +298 -0
  80. data/lib/sidekiq_unique_jobs/redis/entity.rb +112 -0
  81. data/lib/sidekiq_unique_jobs/redis/hash.rb +56 -0
  82. data/lib/sidekiq_unique_jobs/redis/list.rb +32 -0
  83. data/lib/sidekiq_unique_jobs/redis/set.rb +32 -0
  84. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +86 -0
  85. data/lib/sidekiq_unique_jobs/redis/string.rb +51 -0
  86. data/lib/sidekiq_unique_jobs/redis.rb +11 -0
  87. data/lib/sidekiq_unique_jobs/reflectable.rb +26 -0
  88. data/lib/sidekiq_unique_jobs/reflections.rb +79 -0
  89. data/lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb +51 -0
  90. data/lib/sidekiq_unique_jobs/rspec/matchers.rb +26 -0
  91. data/lib/sidekiq_unique_jobs/script/caller.rb +127 -0
  92. data/lib/sidekiq_unique_jobs/script.rb +15 -0
  93. data/lib/sidekiq_unique_jobs/server.rb +61 -0
  94. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +114 -65
  95. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +252 -36
  96. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +47 -32
  97. data/lib/sidekiq_unique_jobs/testing.rb +102 -29
  98. data/lib/sidekiq_unique_jobs/timer_task.rb +299 -0
  99. data/lib/sidekiq_unique_jobs/timing.rb +58 -0
  100. data/lib/sidekiq_unique_jobs/unlockable.rb +20 -4
  101. data/lib/sidekiq_unique_jobs/update_version.rb +25 -0
  102. data/lib/sidekiq_unique_jobs/upgrade_locks.rb +155 -0
  103. data/lib/sidekiq_unique_jobs/version.rb +3 -1
  104. data/lib/sidekiq_unique_jobs/version_check.rb +23 -4
  105. data/lib/sidekiq_unique_jobs/web/helpers.rb +138 -13
  106. data/lib/sidekiq_unique_jobs/web/views/_paging.erb +4 -4
  107. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +54 -0
  108. data/lib/sidekiq_unique_jobs/web/views/lock.erb +110 -0
  109. data/lib/sidekiq_unique_jobs/web/views/locks.erb +54 -0
  110. data/lib/sidekiq_unique_jobs/web.rb +82 -32
  111. data/lib/sidekiq_unique_jobs.rb +54 -7
  112. data/lib/tasks/changelog.rake +16 -16
  113. metadata +134 -177
  114. data/lib/sidekiq_unique_jobs/client/middleware.rb +0 -56
  115. data/lib/sidekiq_unique_jobs/scripts.rb +0 -118
  116. data/lib/sidekiq_unique_jobs/server/middleware.rb +0 -46
  117. data/lib/sidekiq_unique_jobs/timeout/calculator.rb +0 -63
  118. data/lib/sidekiq_unique_jobs/timeout.rb +0 -8
  119. data/lib/sidekiq_unique_jobs/unique_args.rb +0 -150
  120. data/lib/sidekiq_unique_jobs/util.rb +0 -103
  121. data/lib/sidekiq_unique_jobs/web/views/unique_digest.erb +0 -28
  122. data/lib/sidekiq_unique_jobs/web/views/unique_digests.erb +0 -46
  123. data/redis/acquire_lock.lua +0 -21
  124. data/redis/convert_legacy_lock.lua +0 -13
  125. data/redis/delete.lua +0 -14
  126. data/redis/delete_by_digest.lua +0 -23
  127. data/redis/delete_job_by_digest.lua +0 -60
  128. data/redis/lock.lua +0 -62
  129. data/redis/release_stale_locks.lua +0 -90
  130. data/redis/unlock.lua +0 -35
@@ -3,65 +3,234 @@
3
3
  module SidekiqUniqueJobs
4
4
  # Utility module for reducing the number of uses of logger.
5
5
  #
6
- # @author Mikael Henriksson <mikael@zoolutions.se>
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
7
  module Logging
8
- # A convenience method for using the configured logger
8
+ def self.included(base)
9
+ base.send(:extend, self)
10
+ end
11
+
12
+ #
13
+ # A convenience method for using the configured gem logger
14
+ #
15
+ # @see SidekiqUniqueJobs#.logger
16
+ #
17
+ # @return [Logger]
18
+ #
9
19
  def logger
10
20
  SidekiqUniqueJobs.logger
11
21
  end
12
22
 
23
+ #
13
24
  # Logs a message at debug level
14
- # @param message_or_exception [String, Exception] the message or exception to log
15
- # @yield the message or exception to use for log message
16
- # Used for compatibility with logger
17
- def log_debug(message_or_exception = nil, &block)
18
- logger.debug(message_or_exception, &block)
25
+ #
26
+ # @param [String, Exception] message_or_exception the message or exception to log
27
+ #
28
+ # @return [void]
29
+ #
30
+ # @yield [String, Exception] the message or exception to use for log message
31
+ #
32
+ def log_debug(message_or_exception = nil, item = nil, &block)
33
+ return unless logging?
34
+
35
+ message = build_message(message_or_exception, item)
36
+ logger.debug(message, &block)
19
37
  nil
20
38
  end
21
39
 
40
+ #
22
41
  # Logs a message at info level
23
- # @param message_or_exception [String, Exception] the message or exception to log
24
- # @yield the message or exception to use for log message
25
- # Used for compatibility with logger
26
- def log_info(message_or_exception = nil, &block)
27
- logger.info(message_or_exception, &block)
42
+ #
43
+ # @param [String, Exception] message_or_exception the message or exception to log
44
+ #
45
+ # @return [void]
46
+ #
47
+ # @yield [String, Exception] the message or exception to use for log message
48
+ #
49
+ def log_info(message_or_exception = nil, item = nil, &block)
50
+ return unless logging?
51
+
52
+ message = build_message(message_or_exception, item)
53
+ logger.info(message, &block)
28
54
  nil
29
55
  end
30
56
 
57
+ #
31
58
  # Logs a message at warn level
32
- # @param message_or_exception [String, Exception] the message or exception to log
33
- # @yield the message or exception to use for log message
34
- # Used for compatibility with logger
35
- def log_warn(message_or_exception = nil, &block)
36
- logger.warn(message_or_exception, &block)
59
+ #
60
+ # @param [String, Exception] message_or_exception the message or exception to log
61
+ #
62
+ # @return [void]
63
+ #
64
+ # @yield [String, Exception] the message or exception to use for log message
65
+ #
66
+ def log_warn(message_or_exception = nil, item = nil, &block)
67
+ return unless logging?
68
+
69
+ message = build_message(message_or_exception, item)
70
+ logger.warn(message, &block)
37
71
  nil
38
72
  end
39
73
 
74
+ #
40
75
  # Logs a message at error level
41
- # @param message_or_exception [String, Exception] the message or exception to log
42
- # @yield the message or exception to use for log message
43
- # Used for compatibility with logger
44
- def log_error(message_or_exception = nil, &block)
45
- logger.error(message_or_exception, &block)
76
+ #
77
+ # @param [String, Exception] message_or_exception the message or exception to log
78
+ #
79
+ # @return [void]
80
+ #
81
+ # @yield [String, Exception] the message or exception to use for log message
82
+ #
83
+ def log_error(message_or_exception = nil, item = nil, &block)
84
+ return unless logging?
85
+
86
+ message = build_message(message_or_exception, item)
87
+ logger.error(message, &block)
46
88
  nil
47
89
  end
48
90
 
91
+ #
49
92
  # Logs a message at fatal level
50
- # @param message_or_exception [String, Exception] the message or exception to log
51
- # @yield the message or exception to use for log message
52
- # Used for compatibility with logger
53
- def log_fatal(message_or_exception = nil, &block)
54
- logger.fatal(message_or_exception, &block)
93
+ #
94
+ # @param [String, Exception] message_or_exception the message or exception to log
95
+ #
96
+ # @return [void]
97
+ #
98
+ # @yield [String, Exception] the message or exception to use for log message
99
+ #
100
+ def log_fatal(message_or_exception = nil, item = nil, &block)
101
+ return unless logging?
102
+
103
+ message = build_message(message_or_exception, item)
104
+ logger.fatal(message, &block)
105
+
55
106
  nil
56
107
  end
57
108
 
58
- def logging_context(middleware_class, job_hash)
59
- digest = job_hash["unique_digest"]
60
- if defined?(Sidekiq::Logging)
61
- "#{middleware_class} #{"DIG-#{digest}" if digest}"
62
- else
63
- { middleware: middleware_class, unique_digest: digest }
109
+ #
110
+ # Build a log message
111
+ #
112
+ # @param [String, Exception] message_or_exception an entry to log
113
+ # @param [Hash] item the sidekiq job hash
114
+ #
115
+ # @return [String] a complete log entry
116
+ #
117
+ def build_message(message_or_exception, item = nil)
118
+ return nil if message_or_exception.nil?
119
+ return message_or_exception if item.nil?
120
+
121
+ message = message_or_exception.dup
122
+ details = item.slice(LOCK, QUEUE, CLASS, JID, LOCK_DIGEST).each_with_object([]) do |(key, value), memo|
123
+ memo << "#{key}=#{value}"
64
124
  end
125
+ message << " ("
126
+ message << details.join(" ")
127
+ message << ")"
128
+
129
+ message
130
+ end
131
+
132
+ #
133
+ # Wraps the middleware logic with context aware logging
134
+ #
135
+ #
136
+ # @return [void]
137
+ #
138
+ # @yieldreturn [void] yield to the middleware instance
139
+ #
140
+ def with_logging_context
141
+ with_configured_loggers_context do
142
+ return yield
143
+ end
144
+
145
+ nil # Need to make sure we don't return anything here
146
+ end
147
+
148
+ #
149
+ # Attempt to setup context aware logging for the given logger
150
+ #
151
+ #
152
+ # @return [void]
153
+ #
154
+ # @yield
155
+ #
156
+ def with_configured_loggers_context(&block)
157
+ logger_method.call(logging_context, &block)
158
+ end
159
+
160
+ #
161
+ # Setup some variables to add to each log line
162
+ #
163
+ #
164
+ # @return [Hash] the context to use for each log line
165
+ #
166
+ def logging_context
167
+ raise NotImplementedError, "#{__method__} needs to be implemented in #{self.class}"
168
+ end
169
+
170
+ private
171
+
172
+ #
173
+ # A memoized method to use for setting up a logging context
174
+ #
175
+ #
176
+ # @return [proc] the method to call
177
+ #
178
+ def logger_method
179
+ @logger_method ||= sidekiq_context_method
180
+ @logger_method ||= sidekiq_logger_context_method
181
+ @logger_method ||= sidekiq_logging_context_method
182
+ @logger_method ||= no_sidekiq_context_method
183
+ end
184
+
185
+ #
186
+ # Checks if the logger respond to `with_context`.
187
+ #
188
+ # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
189
+ #
190
+ #
191
+ # @return [true,false]
192
+ #
193
+ def logger_respond_to_with_context?
194
+ logger.respond_to?(:with_context)
195
+ end
196
+
197
+ #
198
+ # Checks if the logger context takes a hash argument
199
+ #
200
+ # @note only used to remove the need for explicitly ignoring manual dispatch in other places.
201
+ #
202
+ #
203
+ # @return [true,false]
204
+ #
205
+ def logger_context_hash?
206
+ defined?(Sidekiq::Context) || logger_respond_to_with_context?
207
+ end
208
+
209
+ def sidekiq_context_method
210
+ Sidekiq::Context.method(:with) if defined?(Sidekiq::Context)
211
+ end
212
+
213
+ def sidekiq_logger_context_method
214
+ logger.method(:with_context) if logger_respond_to_with_context?
215
+ end
216
+
217
+ def sidekiq_logging_context_method
218
+ Sidekiq::Logging.method(:with_context) if defined?(Sidekiq::Logging)
219
+ end
220
+
221
+ def no_sidekiq_context_method
222
+ method(:fake_logger_context)
223
+ end
224
+
225
+ def fake_logger_context(_context)
226
+ logger.warn "Don't know how to setup the logging context. Please open a feature request:" \
227
+ " https://github.com/mhenrixon/sidekiq-unique-jobs/issues/new?template=feature_request.md"
228
+
229
+ yield
230
+ end
231
+
232
+ def logging?
233
+ SidekiqUniqueJobs.logging?
65
234
  end
66
235
  end
67
236
  end
@@ -0,0 +1,51 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local info = KEYS[5]
7
+ local changelog = KEYS[6]
8
+ local digests = KEYS[7]
9
+ -------- END keys ---------
10
+
11
+ -------- BEGIN lock arguments ---------
12
+ local job_id = ARGV[1]
13
+ local pttl = tonumber(ARGV[2])
14
+ local lock_type = ARGV[3]
15
+ local limit = tonumber(ARGV[4])
16
+ -------- END lock arguments -----------
17
+
18
+ -------- BEGIN injected arguments --------
19
+ local current_time = tonumber(ARGV[5])
20
+ local debug_lua = tostring(ARGV[6]) == "true"
21
+ local max_history = tonumber(ARGV[7])
22
+ local script_name = tostring(ARGV[8]) .. ".lua"
23
+ local redisversion = tostring(ARGV[9])
24
+ --------- END injected arguments ---------
25
+
26
+ -------- BEGIN local functions --------
27
+ <%= include_partial "shared/_common.lua" %>
28
+ ---------- END local functions ----------
29
+
30
+
31
+ -------- BEGIN delete.lua --------
32
+ log_debug("BEGIN delete", digest)
33
+
34
+ local redis_version = toversion(redisversion)
35
+ local count = 0
36
+ local del_cmd = "DEL"
37
+
38
+ log_debug("ZREM", digests, digest)
39
+ count = count + redis.call("ZREM", digests, digest)
40
+
41
+ if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
42
+
43
+ log_debug(del_cmd, digest, queued, primed, locked, info)
44
+ count = count + redis.call(del_cmd, digest, queued, primed, locked, info)
45
+
46
+
47
+ log("Deleted (" .. count .. ") keys")
48
+ log_debug("END delete (" .. count .. ") keys for:", digest)
49
+
50
+ return count
51
+ -------- END delete.lua --------
@@ -0,0 +1,42 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local run_digest = KEYS[5]
7
+ local run_queued = KEYS[6]
8
+ local run_primed = KEYS[7]
9
+ local run_locked = KEYS[8]
10
+ local digests = KEYS[9]
11
+ -------- END keys ---------
12
+
13
+ -------- BEGIN injected arguments --------
14
+ local current_time = tonumber(ARGV[1])
15
+ local debug_lua = ARGV[2] == "true"
16
+ local max_history = tonumber(ARGV[3])
17
+ local script_name = tostring(ARGV[4]) .. ".lua"
18
+ local redisversion = tostring(ARGV[5])
19
+ --------- END injected arguments ---------
20
+
21
+ -------- BEGIN local functions --------
22
+ <%= include_partial "shared/_common.lua" %>
23
+ ---------- END local functions ----------
24
+
25
+ -------- BEGIN delete_by_digest.lua --------
26
+ local counter = 0
27
+ local redis_version = toversion(redisversion)
28
+ local del_cmd = "DEL"
29
+
30
+ log_debug("BEGIN delete_by_digest:", digest)
31
+
32
+ if redis_version["major"] >= 4 then del_cmd = "UNLINK"; end
33
+
34
+ log_debug(del_cmd, digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
35
+ counter = redis.call(del_cmd, digest, queued, primed, locked, run_digest, run_queued, run_primed, run_locked)
36
+
37
+ log_debug("ZREM", digests, digest)
38
+ redis.call("ZREM", digests, digest)
39
+
40
+ log_debug("END delete_by_digest:", digest, "(deleted " .. counter .. " keys)")
41
+ return counter
42
+ -------- END delete_by_digest.lua --------
@@ -0,0 +1,38 @@
1
+ -------- BEGIN keys ---------
2
+ local queue = KEYS[1]
3
+ local schedule_set = KEYS[2]
4
+ local retry_set = KEYS[3]
5
+ -------- END keys ---------
6
+
7
+ -------- BEGIN Arguments ---------
8
+ local digest = ARGV[1]
9
+ -------- END Arguments ---------
10
+
11
+ -------- BEGIN injected arguments --------
12
+ local current_time = tonumber(ARGV[2])
13
+ local debug_lua = ARGV[3] == "true"
14
+ local max_history = tonumber(ARGV[4])
15
+ local script_name = tostring(ARGV[5]) .. ".lua"
16
+ --------- END injected arguments ---------
17
+
18
+ -------- BEGIN local functions --------
19
+ <%= include_partial "shared/_common.lua" %>
20
+ <%= include_partial "shared/_delete_from_queue.lua" %>
21
+ <%= include_partial "shared/_delete_from_sorted_set.lua" %>
22
+ ---------- END local functions ----------
23
+
24
+
25
+ -------- BEGIN delete_job_by_digest.lua --------
26
+ local result = delete_from_queue(queue, digest)
27
+ if result then
28
+ return result
29
+ end
30
+
31
+ result = delete_from_sorted_set(schedule_set, digest)
32
+ if result then
33
+ return result
34
+ end
35
+
36
+ result = delete_from_sorted_set(retry_set, digest)
37
+ return result
38
+ -------- END delete_job_by_digest.lua --------
@@ -0,0 +1,26 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ -------- END keys ---------
4
+
5
+ -------- BEGIN injected arguments --------
6
+ local current_time = tonumber(ARGV[2])
7
+ local debug_lua = ARGV[3] == "true"
8
+ local max_history = tonumber(ARGV[4])
9
+ local script_name = tostring(ARGV[5]) .. ".lua"
10
+ --------- END injected arguments ---------
11
+
12
+
13
+ -------- BEGIN local functions --------
14
+ <%= include_partial "shared/_common.lua" %>
15
+ <%= include_partial "shared/_find_digest_in_queues.lua" %>
16
+ ---------- END local functions ----------
17
+
18
+
19
+ -------- BEGIN delete_orphaned.lua --------
20
+ log_debug("BEGIN")
21
+ local result = find_digest_in_queues(digest)
22
+ log_debug("END")
23
+ if result and result ~= nil then
24
+ return result
25
+ end
26
+ -------- END delete_orphaned.lua --------
@@ -0,0 +1,99 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local info = KEYS[5]
7
+ local changelog = KEYS[6]
8
+ local digests = KEYS[7]
9
+ local expiring_digests = KEYS[8]
10
+ -------- END keys ---------
11
+
12
+
13
+ -------- BEGIN lock arguments ---------
14
+ local job_id = ARGV[1]
15
+ local pttl = tonumber(ARGV[2])
16
+ local lock_type = ARGV[3]
17
+ local limit = tonumber(ARGV[4])
18
+ -------- END lock arguments -----------
19
+
20
+
21
+ -------- BEGIN injected arguments --------
22
+ local current_time = tonumber(ARGV[5])
23
+ local debug_lua = ARGV[6] == "true"
24
+ local max_history = tonumber(ARGV[7])
25
+ local script_name = tostring(ARGV[8]) .. ".lua"
26
+ local redisversion = ARGV[9]
27
+ --------- END injected arguments ---------
28
+
29
+
30
+ -------- BEGIN local functions --------
31
+ <%= include_partial "shared/_common.lua" %>
32
+ ---------- END local functions ----------
33
+
34
+
35
+ --------- BEGIN lock.lua ---------
36
+ log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
37
+
38
+ if redis.call("HEXISTS", locked, job_id) == 1 then
39
+ log_debug(locked, "already locked with job_id:", job_id)
40
+ log("Duplicate")
41
+
42
+ log_debug("LREM", queued, -1, job_id)
43
+ redis.call("LREM", queued, -1, job_id)
44
+
45
+ log_debug("LREM", primed, 1, job_id)
46
+ redis.call("LREM", primed, 1, job_id)
47
+
48
+ return job_id
49
+ end
50
+
51
+ local locked_count = redis.call("HLEN", locked)
52
+ local within_limit = limit > locked_count
53
+ local limit_exceeded = not within_limit
54
+
55
+ if limit_exceeded then
56
+ log_debug("Limit exceeded:", digest, "(", locked_count, "of", limit, ")")
57
+ log("Limited")
58
+ return nil
59
+ end
60
+
61
+ if lock_type == "until_expired" and pttl and pttl > 0 then
62
+ log_debug("ZADD", expiring_digests, current_time + pttl, digest)
63
+ redis.call("ZADD", expiring_digests, current_time + pttl, digest)
64
+ else
65
+ log_debug("ZADD", digests, current_time, digest)
66
+ redis.call("ZADD", digests, current_time, digest)
67
+ end
68
+
69
+ log_debug("HSET", locked, job_id, current_time)
70
+ redis.call("HSET", locked, job_id, current_time)
71
+
72
+ log_debug("LREM", queued, -1, job_id)
73
+ redis.call("LREM", queued, -1, job_id)
74
+
75
+ log_debug("LREM", primed, 1, job_id)
76
+ redis.call("LREM", primed, 1, job_id)
77
+
78
+ -- The Sidekiq client sets pttl
79
+ if pttl and pttl > 0 then
80
+ log_debug("PEXPIRE", digest, pttl)
81
+ redis.call("PEXPIRE", digest, pttl)
82
+
83
+ log_debug("PEXPIRE", locked, pttl)
84
+ redis.call("PEXPIRE", locked, pttl)
85
+
86
+ log_debug("PEXPIRE", info, pttl)
87
+ redis.call("PEXPIRE", info, pttl)
88
+ end
89
+
90
+ log_debug("PEXPIRE", queued, 1000)
91
+ redis.call("PEXPIRE", queued, 1000)
92
+
93
+ log_debug("PEXPIRE", primed, 1000)
94
+ redis.call("PEXPIRE", primed, 1000)
95
+
96
+ log("Locked")
97
+ log_debug("END lock digest:", digest, "job_id:", job_id)
98
+ return job_id
99
+ ---------- END lock.lua ----------
@@ -0,0 +1,92 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local info = KEYS[5]
7
+ local changelog = KEYS[6]
8
+ local digests = KEYS[7]
9
+ local expiring_digests = KEYS[8]
10
+ -------- END keys ---------
11
+
12
+
13
+ -------- BEGIN lock arguments ---------
14
+ local job_id = ARGV[1]
15
+ local pttl = tonumber(ARGV[2])
16
+ local lock_type = ARGV[3]
17
+ local limit = tonumber(ARGV[4])
18
+ -------- END lock arguments -----------
19
+
20
+
21
+ -------- BEGIN injected arguments --------
22
+ local current_time = tonumber(ARGV[5])
23
+ local debug_lua = ARGV[6] == "true"
24
+ local max_history = tonumber(ARGV[7])
25
+ local script_name = tostring(ARGV[8]) .. ".lua"
26
+ local redisversion = ARGV[9]
27
+ --------- END injected arguments ---------
28
+
29
+
30
+ -------- BEGIN local functions --------
31
+ <%= include_partial "shared/_common.lua" %>
32
+ ---------- END local functions ----------
33
+
34
+
35
+ --------- BEGIN lock.lua ---------
36
+ log_debug("BEGIN lock digest:", digest, "job_id:", job_id)
37
+
38
+ if redis.call("HEXISTS", locked, job_id) == 1 then
39
+ log_debug(locked, "already locked with job_id:", job_id)
40
+ log("Duplicate")
41
+
42
+ log_debug("LREM", queued, -1, job_id)
43
+ redis.call("LREM", queued, -1, job_id)
44
+
45
+ log_debug("LREM", primed, 1, job_id)
46
+ redis.call("LREM", primed, 1, job_id)
47
+
48
+ return job_id
49
+ end
50
+
51
+ local locked_count = redis.call("HLEN", locked)
52
+ local within_limit = limit > locked_count
53
+ local limit_exceeded = not within_limit
54
+
55
+ if limit_exceeded then
56
+ log_debug("Limit exceeded:", digest, "(", locked_count, "of", limit, ")")
57
+ log("Limited")
58
+ return nil
59
+ end
60
+
61
+ log_debug("ZADD", expiring_digests, current_time + pttl, digest)
62
+ redis.call("ZADD", expiring_digests, current_time + pttl, digest)
63
+
64
+ log_debug("HSET", locked, job_id, current_time)
65
+ redis.call("HSET", locked, job_id, current_time)
66
+
67
+ log_debug("LREM", queued, -1, job_id)
68
+ redis.call("LREM", queued, -1, job_id)
69
+
70
+ log_debug("LREM", primed, 1, job_id)
71
+ redis.call("LREM", primed, 1, job_id)
72
+
73
+ -- The Sidekiq client sets pttl
74
+ log_debug("PEXPIRE", digest, pttl)
75
+ redis.call("PEXPIRE", digest, pttl)
76
+
77
+ log_debug("PEXPIRE", locked, pttl)
78
+ redis.call("PEXPIRE", locked, pttl)
79
+
80
+ log_debug("PEXPIRE", info, pttl)
81
+ redis.call("PEXPIRE", info, pttl)
82
+
83
+ log_debug("PEXPIRE", queued, 1000)
84
+ redis.call("PEXPIRE", queued, 1000)
85
+
86
+ log_debug("PEXPIRE", primed, 1000)
87
+ redis.call("PEXPIRE", primed, 1000)
88
+
89
+ log("Locked")
90
+ log_debug("END lock digest:", digest, "job_id:", job_id)
91
+ return job_id
92
+ ---------- END lock.lua ----------
@@ -0,0 +1,35 @@
1
+ -------- BEGIN keys ---------
2
+ local digest = KEYS[1]
3
+ local queued = KEYS[2]
4
+ local primed = KEYS[3]
5
+ local locked = KEYS[4]
6
+ local info = KEYS[5]
7
+ local changelog = KEYS[6]
8
+ local digests = KEYS[7]
9
+ -------- END keys ---------
10
+
11
+ -------- BEGIN lock arguments ---------
12
+ local job_id = ARGV[1]
13
+ -------- END lock arguments -----------
14
+
15
+ -------- BEGIN injected arguments --------
16
+ local current_time = tonumber(ARGV[2])
17
+ local debug_lua = ARGV[3] == "true"
18
+ local max_history = tonumber(ARGV[4])
19
+ local script_name = tostring(ARGV[5]) .. ".lua"
20
+ --------- END injected arguments ---------
21
+
22
+ -------- BEGIN local functions --------
23
+ <%= include_partial "shared/_common.lua" %>
24
+ --------- END local functions ---------
25
+
26
+
27
+ -------- BEGIN locked.lua --------
28
+ if redis.call("HEXISTS", locked, job_id) == 1 then
29
+ log_debug("Locked", digest, "job_id:", job_id)
30
+ return 1
31
+ else
32
+ log_debug("NOT Locked", digest, "job_id:", job_id)
33
+ return -1
34
+ end
35
+ --------- END locked.lua ---------