sidekiq-unique-jobs 8.0.5 → 8.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sidekiq-unique-jobs might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/lib/sidekiq_unique_jobs/changelog.rb +1 -1
- data/lib/sidekiq_unique_jobs/orphans/manager.rb +6 -5
- data/lib/sidekiq_unique_jobs/script/caller.rb +1 -1
- data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
- data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
- data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
- data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
- data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
- data/lib/sidekiq_unique_jobs/script/script.rb +80 -0
- data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
- data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
- data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
- data/lib/sidekiq_unique_jobs/script.rb +32 -1
- data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +1 -1
- data/lib/sidekiq_unique_jobs/version.rb +1 -1
- data/lib/sidekiq_unique_jobs.rb +5 -4
- metadata +12 -23
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0f2baa17d73480d2a3af58b1202e5145935944abebfddbd94a199248df87e729
|
4
|
+
data.tar.gz: 11037fd5c6ed9a61452026541c874928ec0d465def211c7f34e91d7d1bb23bb2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 927072c34b1b7131352dbfe5ccd269bace168d601977dfcb63199183033202b6ec04824612620375b336de9e06501a2b39237370357494af66a376385260faa3
|
7
|
+
data.tar.gz: cb615cce8db1b353f0b12790ee35019c709586f94b497d5543ef43a50a782b1e09c47371ddd28b810243b15e7d983030c5a56999ef7535745f4596890fc983dd
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,18 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## [v8.0.5](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.5) (2023-11-11)
|
4
|
+
|
5
|
+
[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v8.0.4...v8.0.5)
|
6
|
+
|
7
|
+
**Merged pull requests:**
|
8
|
+
|
9
|
+
- Bump @babel/traverse from 7.22.8 to 7.23.3 in /myapp [\#819](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/819) ([dependabot[bot]](https://github.com/apps/dependabot))
|
10
|
+
- Bump postcss from 8.4.21 to 8.4.31 in /myapp [\#811](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/811) ([dependabot[bot]](https://github.com/apps/dependabot))
|
11
|
+
- fix: `while_executing` should not invoke conflict strategy when the job was successfully executed \[v8\] [\#810](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/810) ([cuzik](https://github.com/cuzik))
|
12
|
+
- Bump actions/checkout from 3 to 4 [\#808](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/808) ([dependabot[bot]](https://github.com/apps/dependabot))
|
13
|
+
- Bump semver from 6.3.0 to 6.3.1 in /myapp [\#798](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/798) ([dependabot[bot]](https://github.com/apps/dependabot))
|
14
|
+
- Because `replace` is a client strategy, it should only remove client locks aka queue locks. [\#778](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/778) ([bigzed](https://github.com/bigzed))
|
15
|
+
|
3
16
|
## [v8.0.4](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v8.0.4) (2023-11-11)
|
4
17
|
|
5
18
|
[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.30...v8.0.4)
|
@@ -59,7 +59,7 @@ module SidekiqUniqueJobs
|
|
59
59
|
# NOTE: When debugging, check the last item in the returned array.
|
60
60
|
[
|
61
61
|
total_size.to_i,
|
62
|
-
result[0].to_i,
|
62
|
+
result[0].to_i, # next_cursor
|
63
63
|
result[1].map { |entry| load_json(entry) }.select { |entry| entry.is_a?(Hash) },
|
64
64
|
]
|
65
65
|
end
|
@@ -39,12 +39,13 @@ module SidekiqUniqueJobs
|
|
39
39
|
self.task = test_task || default_task
|
40
40
|
|
41
41
|
with_logging_context do
|
42
|
-
register_reaper_process
|
43
|
-
|
42
|
+
if register_reaper_process
|
43
|
+
log_info("Starting Reaper")
|
44
44
|
|
45
|
-
|
46
|
-
|
47
|
-
|
45
|
+
task.add_observer(Observer.new)
|
46
|
+
task.execute
|
47
|
+
task
|
48
|
+
end
|
48
49
|
end
|
49
50
|
end
|
50
51
|
|
@@ -56,7 +56,7 @@ module SidekiqUniqueJobs
|
|
56
56
|
def do_call(file_name, conn, keys, argv)
|
57
57
|
argv = argv.dup.push(now_f, debug_lua, max_history, file_name, redis_version)
|
58
58
|
|
59
|
-
Script.execute(file_name, conn, keys: keys, argv: normalize_argv(argv))
|
59
|
+
SidekiqUniqueJobs::Script.execute(file_name, conn, keys: keys, argv: normalize_argv(argv))
|
60
60
|
end
|
61
61
|
|
62
62
|
#
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Script
|
5
|
+
# Interface to dealing with .lua files
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
class Client
|
9
|
+
include SidekiqUniqueJobs::Script::Timing
|
10
|
+
|
11
|
+
#
|
12
|
+
# @!attribute [r] logger
|
13
|
+
# @return [Logger] an instance of a logger
|
14
|
+
attr_reader :logger
|
15
|
+
#
|
16
|
+
# @!attribute [r] file_name
|
17
|
+
# @return [String] The name of the file to execute
|
18
|
+
attr_reader :config
|
19
|
+
#
|
20
|
+
# @!attribute [r] scripts
|
21
|
+
# @return [Scripts] the collection with loaded scripts
|
22
|
+
attr_reader :scripts
|
23
|
+
|
24
|
+
def initialize(config)
|
25
|
+
@config = config
|
26
|
+
@logger = config.logger
|
27
|
+
@scripts = Scripts.fetch(config.scripts_path)
|
28
|
+
end
|
29
|
+
|
30
|
+
#
|
31
|
+
# Execute a lua script with the provided script_name
|
32
|
+
#
|
33
|
+
# @note this method is recursive if we need to load a lua script
|
34
|
+
# that wasn't previously loaded.
|
35
|
+
#
|
36
|
+
# @param [Symbol] script_name the name of the script to execute
|
37
|
+
# @param [Redis] conn the redis connection to use for execution
|
38
|
+
# @param [Array<String>] keys script keys
|
39
|
+
# @param [Array<Object>] argv script arguments
|
40
|
+
#
|
41
|
+
# @return value from script
|
42
|
+
#
|
43
|
+
def execute(script_name, conn, keys: [], argv: [])
|
44
|
+
result, elapsed = timed do
|
45
|
+
scripts.execute(script_name, conn, keys: keys, argv: argv)
|
46
|
+
end
|
47
|
+
|
48
|
+
logger.debug("Executed #{script_name}.lua in #{elapsed}ms")
|
49
|
+
result
|
50
|
+
rescue ::RedisClient::CommandError => ex
|
51
|
+
handle_error(script_name, conn, ex) do
|
52
|
+
execute(script_name, conn, keys: keys, argv: argv)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
#
|
59
|
+
# Handle errors to allow retrying errors that need retrying
|
60
|
+
#
|
61
|
+
# @param [RedisClient::CommandError] ex exception to handle
|
62
|
+
#
|
63
|
+
# @return [void]
|
64
|
+
#
|
65
|
+
# @yieldreturn [void] yields back to the caller when NOSCRIPT is raised
|
66
|
+
def handle_error(script_name, conn, ex)
|
67
|
+
case ex.message
|
68
|
+
when /NOSCRIPT/
|
69
|
+
handle_noscript(script_name) { return yield }
|
70
|
+
when /BUSY/
|
71
|
+
handle_busy(conn) { return yield }
|
72
|
+
end
|
73
|
+
|
74
|
+
raise unless LuaError.intercepts?(ex)
|
75
|
+
|
76
|
+
script = scripts.fetch(script_name, conn)
|
77
|
+
raise LuaError.new(ex, script)
|
78
|
+
end
|
79
|
+
|
80
|
+
def handle_noscript(script_name)
|
81
|
+
scripts.delete(script_name)
|
82
|
+
yield
|
83
|
+
end
|
84
|
+
|
85
|
+
def handle_busy(conn)
|
86
|
+
scripts.kill(conn)
|
87
|
+
rescue ::RedisClient::CommandError => ex
|
88
|
+
logger.warn(ex)
|
89
|
+
ensure
|
90
|
+
yield
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Script
|
5
|
+
#
|
6
|
+
# Class holding gem configuration
|
7
|
+
#
|
8
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
9
|
+
class Config
|
10
|
+
#
|
11
|
+
# @!attribute [r] logger
|
12
|
+
# @return [Logger] a logger to use for debugging
|
13
|
+
attr_reader :logger
|
14
|
+
#
|
15
|
+
# @!attribute [r] scripts_path
|
16
|
+
# @return [Pathname] a directory with lua scripts
|
17
|
+
attr_reader :scripts_path
|
18
|
+
|
19
|
+
#
|
20
|
+
# Initialize a new instance of {Config}
|
21
|
+
#
|
22
|
+
#
|
23
|
+
def initialize
|
24
|
+
@conn = RedisClient.new
|
25
|
+
@logger = Logger.new($stdout)
|
26
|
+
@scripts_path = nil
|
27
|
+
end
|
28
|
+
|
29
|
+
#
|
30
|
+
# Sets a value for scripts_path
|
31
|
+
#
|
32
|
+
# @param [String, Pathname] obj <description>
|
33
|
+
#
|
34
|
+
# @raise [ArgumentError] when directory does not exist
|
35
|
+
# @raise [ArgumentError] when argument isn't supported
|
36
|
+
#
|
37
|
+
# @return [Pathname]
|
38
|
+
#
|
39
|
+
def scripts_path=(obj)
|
40
|
+
raise ArgumentError, "#{obj} should be a Pathname or String" unless obj.is_a?(Pathname) || obj.is_a?(String)
|
41
|
+
raise ArgumentError, "#{obj} does not exist" unless Dir.exist?(obj.to_s)
|
42
|
+
|
43
|
+
@scripts_path =
|
44
|
+
case obj
|
45
|
+
when String
|
46
|
+
Pathname.new(obj)
|
47
|
+
else
|
48
|
+
obj
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
#
|
53
|
+
# Sets a value for logger
|
54
|
+
#
|
55
|
+
# @param [Logger] obj a logger to use
|
56
|
+
#
|
57
|
+
# @raise [ArgumentError] when given argument isn't a Logger
|
58
|
+
#
|
59
|
+
# @return [Logger]
|
60
|
+
#
|
61
|
+
def logger=(obj)
|
62
|
+
raise ArgumentError, "#{obj} should be a Logger" unless obj.is_a?(Logger)
|
63
|
+
|
64
|
+
@logger = obj
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Script
|
5
|
+
# Interface to dealing with .lua files
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
module DSL
|
9
|
+
MUTEX = Mutex.new
|
10
|
+
|
11
|
+
def self.included(base)
|
12
|
+
base.class_eval do
|
13
|
+
extend ClassMethods
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
#
|
18
|
+
# Module ClassMethods extends the base class with necessary methods
|
19
|
+
#
|
20
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
21
|
+
#
|
22
|
+
module ClassMethods
|
23
|
+
def execute(file_name, conn, keys: [], argv: [])
|
24
|
+
SidekiqUniqueJobs::Script::Client
|
25
|
+
.new(config)
|
26
|
+
.execute(file_name, conn, keys: keys, argv: argv)
|
27
|
+
end
|
28
|
+
|
29
|
+
# Configure the gem
|
30
|
+
#
|
31
|
+
# This is usually called once at startup of an application
|
32
|
+
# @param [Hash] options global gem options
|
33
|
+
# @option options [String, Pathname] :path
|
34
|
+
# @option options [Logger] :logger (default is Logger.new(STDOUT))
|
35
|
+
# @yield control to the caller when given block
|
36
|
+
def configure(options = {})
|
37
|
+
if block_given?
|
38
|
+
yield config
|
39
|
+
else
|
40
|
+
options.each do |key, val|
|
41
|
+
config.send(:"#{key}=", val)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
#
|
47
|
+
# The current configuration (See: {.configure} on how to configure)
|
48
|
+
#
|
49
|
+
#
|
50
|
+
# @return [Script::Config] the gem configuration
|
51
|
+
#
|
52
|
+
def config
|
53
|
+
MUTEX.synchronize do
|
54
|
+
@config ||= Config.new
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Script
|
5
|
+
# Utility module for reducing the number of uses of logger.
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
module Logging
|
9
|
+
def self.included(base)
|
10
|
+
base.send(:extend, self)
|
11
|
+
end
|
12
|
+
|
13
|
+
#
|
14
|
+
# A convenience method for using the configured gem logger
|
15
|
+
#
|
16
|
+
# @see Script#.logger
|
17
|
+
#
|
18
|
+
# @return [Logger]
|
19
|
+
#
|
20
|
+
def logger
|
21
|
+
SidekiqUniqueJobs::Script.logger
|
22
|
+
end
|
23
|
+
|
24
|
+
#
|
25
|
+
# Logs a message at debug level
|
26
|
+
#
|
27
|
+
# @param [String, Exception] message_or_exception the message or exception to log
|
28
|
+
#
|
29
|
+
# @return [void]
|
30
|
+
#
|
31
|
+
# @yield [String, Exception] the message or exception to use for log message
|
32
|
+
#
|
33
|
+
def log_debug(message_or_exception = nil, &block)
|
34
|
+
logger.debug(message_or_exception, &block)
|
35
|
+
nil
|
36
|
+
end
|
37
|
+
|
38
|
+
#
|
39
|
+
# Logs a message at info level
|
40
|
+
#
|
41
|
+
# @param [String, Exception] message_or_exception the message or exception to log
|
42
|
+
#
|
43
|
+
# @return [void]
|
44
|
+
#
|
45
|
+
# @yield [String, Exception] the message or exception to use for log message
|
46
|
+
#
|
47
|
+
def log_info(message_or_exception = nil, &block)
|
48
|
+
logger.info(message_or_exception, &block)
|
49
|
+
nil
|
50
|
+
end
|
51
|
+
|
52
|
+
#
|
53
|
+
# Logs a message at warn level
|
54
|
+
#
|
55
|
+
# @param [String, Exception] message_or_exception the message or exception to log
|
56
|
+
#
|
57
|
+
# @return [void]
|
58
|
+
#
|
59
|
+
# @yield [String, Exception] the message or exception to use for log message
|
60
|
+
#
|
61
|
+
def log_warn(message_or_exception = nil, &block)
|
62
|
+
logger.warn(message_or_exception, &block)
|
63
|
+
nil
|
64
|
+
end
|
65
|
+
|
66
|
+
#
|
67
|
+
# Logs a message at error level
|
68
|
+
#
|
69
|
+
# @param [String, Exception] message_or_exception the message or exception to log
|
70
|
+
#
|
71
|
+
# @return [void]
|
72
|
+
#
|
73
|
+
# @yield [String, Exception] the message or exception to use for log message
|
74
|
+
#
|
75
|
+
def log_error(message_or_exception = nil, &block)
|
76
|
+
logger.error(message_or_exception, &block)
|
77
|
+
nil
|
78
|
+
end
|
79
|
+
|
80
|
+
#
|
81
|
+
# Logs a message at fatal level
|
82
|
+
#
|
83
|
+
# @param [String, Exception] message_or_exception the message or exception to log
|
84
|
+
#
|
85
|
+
# @return [void]
|
86
|
+
#
|
87
|
+
# @yield [String, Exception] the message or exception to use for log message
|
88
|
+
#
|
89
|
+
def log_fatal(message_or_exception = nil, &block)
|
90
|
+
logger.fatal(message_or_exception, &block)
|
91
|
+
nil
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Script
|
5
|
+
#
|
6
|
+
# Misconfiguration is raised when gem is misconfigured
|
7
|
+
#
|
8
|
+
# @author Mikael Henriksson <mikael@zoolutions.se>
|
9
|
+
#
|
10
|
+
class Misconfiguration < RuntimeError
|
11
|
+
end
|
12
|
+
|
13
|
+
# LuaError raised on errors in Lua scripts
|
14
|
+
#
|
15
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
16
|
+
class LuaError < RuntimeError
|
17
|
+
# Reformats errors raised by redis representing failures while executing
|
18
|
+
# a lua script. The default errors have confusing messages and backtraces,
|
19
|
+
# and a type of +RuntimeError+. This class improves the message and
|
20
|
+
# modifies the backtrace to include the lua script itself in a reasonable
|
21
|
+
# way.
|
22
|
+
|
23
|
+
PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/.freeze
|
24
|
+
LIB_PATH = File.expand_path("..", __dir__).freeze
|
25
|
+
CONTEXT_LINE_NUMBER = 2
|
26
|
+
|
27
|
+
attr_reader :error, :file, :content
|
28
|
+
|
29
|
+
# Is this error one that should be reformatted?
|
30
|
+
#
|
31
|
+
# @param error [StandardError] the original error raised by redis
|
32
|
+
# @return [Boolean] is this an error that should be reformatted?
|
33
|
+
def self.intercepts?(error)
|
34
|
+
PATTERN.match?(error.message)
|
35
|
+
end
|
36
|
+
|
37
|
+
# Initialize a new {LuaError} from an existing redis error, adjusting
|
38
|
+
# the message and backtrace in the process.
|
39
|
+
#
|
40
|
+
# @param error [StandardError] the original error raised by redis
|
41
|
+
# @param script [Script] a DTO with information about the script
|
42
|
+
#
|
43
|
+
def initialize(error, script)
|
44
|
+
@error = error
|
45
|
+
@file = script.path
|
46
|
+
@content = script.source
|
47
|
+
@backtrace = @error.backtrace
|
48
|
+
|
49
|
+
@error.message.match(PATTERN) do |regexp_match|
|
50
|
+
line_number = regexp_match[2].to_i
|
51
|
+
message = regexp_match[3]
|
52
|
+
error_context = generate_error_context(content, line_number)
|
53
|
+
|
54
|
+
super("#{message}\n\n#{error_context}\n\n")
|
55
|
+
set_backtrace(generate_backtrace(file, line_number))
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
# :nocov:
|
62
|
+
def generate_error_context(content, line_number)
|
63
|
+
lines = content.lines.to_a
|
64
|
+
beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
|
65
|
+
ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
|
66
|
+
line_number_width = ending_line_number.to_s.length
|
67
|
+
|
68
|
+
(beginning_line_number..ending_line_number).map do |number|
|
69
|
+
indicator = (number == line_number) ? "=>" : " "
|
70
|
+
formatted_number = format("%#{line_number_width}d", number)
|
71
|
+
" #{indicator} #{formatted_number}: #{lines[number - 1]}"
|
72
|
+
end.join.chomp
|
73
|
+
end
|
74
|
+
|
75
|
+
# :nocov:
|
76
|
+
def generate_backtrace(file, line_number)
|
77
|
+
pre_gem = backtrace_before_entering_gem(@backtrace)
|
78
|
+
index_of_first_gem_line = (@backtrace.size - pre_gem.size - 1)
|
79
|
+
|
80
|
+
pre_gem.unshift(@backtrace[index_of_first_gem_line])
|
81
|
+
pre_gem.unshift("#{file}:#{line_number}")
|
82
|
+
pre_gem
|
83
|
+
end
|
84
|
+
|
85
|
+
# :nocov:
|
86
|
+
def backtrace_before_entering_gem(backtrace)
|
87
|
+
backtrace.reverse.take_while { |line| !line_from_gem(line) }.reverse
|
88
|
+
end
|
89
|
+
|
90
|
+
# :nocov:
|
91
|
+
def line_from_gem(line)
|
92
|
+
line.split(":").first.include?(LIB_PATH)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Script
|
5
|
+
# Interface to dealing with .lua files
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
class Script
|
9
|
+
def self.load(name, root_path, conn)
|
10
|
+
script = new(name: name, root_path: root_path)
|
11
|
+
script.load(conn)
|
12
|
+
end
|
13
|
+
|
14
|
+
#
|
15
|
+
# @!attribute [r] script_name
|
16
|
+
# @return [Symbol, String] the name of the script without extension
|
17
|
+
attr_reader :name
|
18
|
+
#
|
19
|
+
# @!attribute [r] script_path
|
20
|
+
# @return [String] the path to the script on disk
|
21
|
+
attr_reader :path
|
22
|
+
#
|
23
|
+
# @!attribute [r] root_path
|
24
|
+
# @return [Pathname]
|
25
|
+
attr_reader :root_path
|
26
|
+
#
|
27
|
+
# @!attribute [r] source
|
28
|
+
# @return [String] the source code of the lua script
|
29
|
+
attr_reader :source
|
30
|
+
#
|
31
|
+
# @!attribute [rw] sha
|
32
|
+
# @return [String] the sha of the script
|
33
|
+
attr_reader :sha
|
34
|
+
#
|
35
|
+
# @!attribute [rw] call_count
|
36
|
+
# @return [Integer] the number of times the script was called/executed
|
37
|
+
attr_reader :call_count
|
38
|
+
|
39
|
+
def initialize(name:, root_path:)
|
40
|
+
@name = name
|
41
|
+
@root_path = root_path
|
42
|
+
@path = root_path.join("#{name}.lua").to_s
|
43
|
+
@source = render_file
|
44
|
+
@sha = compiled_sha
|
45
|
+
@call_count = 0
|
46
|
+
end
|
47
|
+
|
48
|
+
def ==(other)
|
49
|
+
sha == compiled_sha && compiled_sha == other.sha
|
50
|
+
end
|
51
|
+
|
52
|
+
def increment_call_count
|
53
|
+
@call_count += 1
|
54
|
+
end
|
55
|
+
|
56
|
+
def changed?
|
57
|
+
compiled_sha != sha
|
58
|
+
end
|
59
|
+
|
60
|
+
def render_file
|
61
|
+
Template.new(root_path).render(path)
|
62
|
+
end
|
63
|
+
|
64
|
+
def compiled_sha
|
65
|
+
Digest::SHA1.hexdigest(source)
|
66
|
+
end
|
67
|
+
|
68
|
+
def load(conn)
|
69
|
+
@sha =
|
70
|
+
if conn.respond_to?(:namespace)
|
71
|
+
conn.redis.script(:load, source)
|
72
|
+
else
|
73
|
+
conn.script(:load, source)
|
74
|
+
end
|
75
|
+
|
76
|
+
self
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Script
|
5
|
+
# Interface to dealing with .lua files
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
class Scripts
|
9
|
+
#
|
10
|
+
# @return [Concurrent::Map] a map with configured script paths
|
11
|
+
SCRIPT_PATHS = Concurrent::Map.new
|
12
|
+
|
13
|
+
#
|
14
|
+
# Fetch a scripts configuration for path
|
15
|
+
#
|
16
|
+
# @param [Pathname] root_path the path to scripts
|
17
|
+
#
|
18
|
+
# @return [Scripts] a collection of scripts
|
19
|
+
#
|
20
|
+
def self.fetch(root_path)
|
21
|
+
if (scripts = SCRIPT_PATHS.get(root_path))
|
22
|
+
return scripts
|
23
|
+
end
|
24
|
+
|
25
|
+
create(root_path)
|
26
|
+
end
|
27
|
+
|
28
|
+
#
|
29
|
+
# Create a new scripts collection based on path
|
30
|
+
#
|
31
|
+
# @param [Pathname] root_path the path to scripts
|
32
|
+
#
|
33
|
+
# @return [Scripts] a collection of scripts
|
34
|
+
#
|
35
|
+
def self.create(root_path)
|
36
|
+
scripts = new(root_path)
|
37
|
+
store(scripts)
|
38
|
+
end
|
39
|
+
|
40
|
+
#
|
41
|
+
# Store the scripts collection in memory
|
42
|
+
#
|
43
|
+
# @param [Scripts] scripts the path to scripts
|
44
|
+
#
|
45
|
+
# @return [Scripts] the scripts instance that was stored
|
46
|
+
#
|
47
|
+
def self.store(scripts)
|
48
|
+
SCRIPT_PATHS.put(scripts.root_path, scripts)
|
49
|
+
scripts
|
50
|
+
end
|
51
|
+
|
52
|
+
#
|
53
|
+
# @!attribute [r] scripts
|
54
|
+
# @return [Concurrent::Map] a collection of loaded scripts
|
55
|
+
attr_reader :scripts
|
56
|
+
|
57
|
+
#
|
58
|
+
# @!attribute [r] root_path
|
59
|
+
# @return [Pathname] the path to the directory with lua scripts
|
60
|
+
attr_reader :root_path
|
61
|
+
|
62
|
+
def initialize(path)
|
63
|
+
raise ArgumentError, "path needs to be a Pathname" unless path.is_a?(Pathname)
|
64
|
+
|
65
|
+
@scripts = Concurrent::Map.new
|
66
|
+
@root_path = path
|
67
|
+
end
|
68
|
+
|
69
|
+
def fetch(name, conn)
|
70
|
+
if (script = scripts.get(name.to_sym))
|
71
|
+
return script
|
72
|
+
end
|
73
|
+
|
74
|
+
load(name, conn)
|
75
|
+
end
|
76
|
+
|
77
|
+
def load(name, conn)
|
78
|
+
script = Script.load(name, root_path, conn)
|
79
|
+
scripts.put(name.to_sym, script)
|
80
|
+
|
81
|
+
script
|
82
|
+
end
|
83
|
+
|
84
|
+
def delete(script)
|
85
|
+
if script.is_a?(Script)
|
86
|
+
scripts.delete(script.name)
|
87
|
+
else
|
88
|
+
scripts.delete(script.to_sym)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def kill(conn)
|
93
|
+
if conn.respond_to?(:namespace)
|
94
|
+
conn.redis.script(:kill)
|
95
|
+
else
|
96
|
+
conn.script(:kill)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
#
|
101
|
+
# Execute a lua script with given name
|
102
|
+
#
|
103
|
+
# @note this method is recursive if we need to load a lua script
|
104
|
+
# that wasn't previously loaded.
|
105
|
+
#
|
106
|
+
# @param [Symbol] name the name of the script to execute
|
107
|
+
# @param [Redis] conn the redis connection to use for execution
|
108
|
+
# @param [Array<String>] keys script keys
|
109
|
+
# @param [Array<Object>] argv script arguments
|
110
|
+
#
|
111
|
+
# @return value from script
|
112
|
+
#
|
113
|
+
def execute(name, conn, keys: [], argv: [])
|
114
|
+
script = fetch(name, conn)
|
115
|
+
conn.evalsha(script.sha, keys, argv)
|
116
|
+
end
|
117
|
+
|
118
|
+
def count
|
119
|
+
scripts.keys.size
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
# Interface to dealing with .lua files
|
5
|
+
#
|
6
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
7
|
+
module Script
|
8
|
+
#
|
9
|
+
# Class Template provides LUA script partial template rendering
|
10
|
+
#
|
11
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
12
|
+
#
|
13
|
+
class Template
|
14
|
+
def initialize(script_path)
|
15
|
+
@script_path = script_path
|
16
|
+
end
|
17
|
+
|
18
|
+
#
|
19
|
+
# Renders a Lua script and includes any partials in that file
|
20
|
+
# all `<%= include_partial '' %>` replaced with the actual contents of the partial
|
21
|
+
#
|
22
|
+
# @param [Pathname] pathname the path to the
|
23
|
+
#
|
24
|
+
# @return [String] the rendered Luascript
|
25
|
+
#
|
26
|
+
def render(pathname)
|
27
|
+
@partial_templates ||= {}
|
28
|
+
::ERB.new(File.read(pathname)).result(binding)
|
29
|
+
end
|
30
|
+
|
31
|
+
# helper method to include a lua partial within another lua script
|
32
|
+
#
|
33
|
+
def include_partial(relative_path)
|
34
|
+
return if @partial_templates.key?(relative_path)
|
35
|
+
|
36
|
+
@partial_templates[relative_path] = nil
|
37
|
+
render(Pathname.new("#{@script_path}/#{relative_path}"))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SidekiqUniqueJobs
|
4
|
+
module Script
|
5
|
+
# Handles timing> of things
|
6
|
+
#
|
7
|
+
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
8
|
+
module Timing
|
9
|
+
module_function
|
10
|
+
|
11
|
+
#
|
12
|
+
# Used for timing method calls
|
13
|
+
#
|
14
|
+
#
|
15
|
+
# @return [yield return, Float]
|
16
|
+
#
|
17
|
+
def timed
|
18
|
+
start_time = now
|
19
|
+
|
20
|
+
[yield, now - start_time]
|
21
|
+
end
|
22
|
+
|
23
|
+
#
|
24
|
+
# Returns a float representation of the current time.
|
25
|
+
# Either from Process or Time
|
26
|
+
#
|
27
|
+
#
|
28
|
+
# @return [Float]
|
29
|
+
#
|
30
|
+
def now
|
31
|
+
(Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -1,15 +1,46 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "sidekiq_unique_jobs/script/template"
|
4
|
+
require "sidekiq_unique_jobs/script/lua_error"
|
5
|
+
require "sidekiq_unique_jobs/script/script"
|
6
|
+
require "sidekiq_unique_jobs/script/scripts"
|
7
|
+
require "sidekiq_unique_jobs/script/config"
|
8
|
+
require "sidekiq_unique_jobs/script/timing"
|
9
|
+
require "sidekiq_unique_jobs/script/logging"
|
10
|
+
require "sidekiq_unique_jobs/script/dsl"
|
11
|
+
require "sidekiq_unique_jobs/script/client"
|
12
|
+
|
3
13
|
module SidekiqUniqueJobs
|
4
14
|
# Interface to dealing with .lua files
|
5
15
|
#
|
6
16
|
# @author Mikael Henriksson <mikael@mhenrixon.com>
|
7
17
|
module Script
|
8
|
-
include
|
18
|
+
include SidekiqUniqueJobs::Script::DSL
|
9
19
|
|
10
20
|
configure do |config|
|
11
21
|
config.scripts_path = Pathname.new(__FILE__).dirname.join("lua")
|
12
22
|
config.logger = Sidekiq.logger # TODO: This becomes a little weird
|
13
23
|
end
|
24
|
+
|
25
|
+
#
|
26
|
+
# The current logger
|
27
|
+
#
|
28
|
+
#
|
29
|
+
# @return [Logger] the configured logger
|
30
|
+
#
|
31
|
+
def self.logger
|
32
|
+
config.logger
|
33
|
+
end
|
34
|
+
|
35
|
+
#
|
36
|
+
# Set a new logger
|
37
|
+
#
|
38
|
+
# @param [Logger] other another logger
|
39
|
+
#
|
40
|
+
# @return [Logger] the new logger
|
41
|
+
#
|
42
|
+
def self.logger=(other)
|
43
|
+
config.logger = other
|
44
|
+
end
|
14
45
|
end
|
15
46
|
end
|
data/lib/sidekiq_unique_jobs.rb
CHANGED
@@ -1,20 +1,22 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "
|
3
|
+
require "concurrent/executor/ruby_single_thread_executor"
|
4
4
|
require "concurrent/future"
|
5
|
-
require "concurrent/promises"
|
6
5
|
require "concurrent/map"
|
7
6
|
require "concurrent/mutable_struct"
|
7
|
+
require "concurrent/promises"
|
8
8
|
require "concurrent/timer_task"
|
9
|
-
require "concurrent/executor/ruby_single_thread_executor"
|
10
9
|
require "digest"
|
11
10
|
require "digest/sha1"
|
12
11
|
require "erb"
|
13
12
|
require "forwardable"
|
14
13
|
require "json"
|
15
14
|
require "pathname"
|
15
|
+
require "redis_client"
|
16
16
|
require "sidekiq"
|
17
17
|
|
18
|
+
require "sidekiq_unique_jobs/script"
|
19
|
+
|
18
20
|
require "sidekiq_unique_jobs/deprecation"
|
19
21
|
require "sidekiq_unique_jobs/reflections"
|
20
22
|
require "sidekiq_unique_jobs/reflectable"
|
@@ -29,7 +31,6 @@ require "sidekiq_unique_jobs/timing"
|
|
29
31
|
require "sidekiq_unique_jobs/sidekiq_worker_methods"
|
30
32
|
require "sidekiq_unique_jobs/connection"
|
31
33
|
require "sidekiq_unique_jobs/exceptions"
|
32
|
-
require "sidekiq_unique_jobs/script"
|
33
34
|
require "sidekiq_unique_jobs/script/caller"
|
34
35
|
require "sidekiq_unique_jobs/normalizer"
|
35
36
|
require "sidekiq_unique_jobs/job"
|
metadata
CHANGED
@@ -1,35 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-unique-jobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 8.0.
|
4
|
+
version: 8.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mikael Henriksson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-01-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: brpoplpush-redis_script
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ">"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 0.1.1
|
20
|
-
- - "<="
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 2.0.0
|
23
|
-
type: :runtime
|
24
|
-
prerelease: false
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
26
|
-
requirements:
|
27
|
-
- - ">"
|
28
|
-
- !ruby/object:Gem::Version
|
29
|
-
version: 0.1.1
|
30
|
-
- - "<="
|
31
|
-
- !ruby/object:Gem::Version
|
32
|
-
version: 2.0.0
|
33
13
|
- !ruby/object:Gem::Dependency
|
34
14
|
name: concurrent-ruby
|
35
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -195,6 +175,15 @@ files:
|
|
195
175
|
- lib/sidekiq_unique_jobs/rspec/matchers/have_valid_sidekiq_options.rb
|
196
176
|
- lib/sidekiq_unique_jobs/script.rb
|
197
177
|
- lib/sidekiq_unique_jobs/script/caller.rb
|
178
|
+
- lib/sidekiq_unique_jobs/script/client.rb
|
179
|
+
- lib/sidekiq_unique_jobs/script/config.rb
|
180
|
+
- lib/sidekiq_unique_jobs/script/dsl.rb
|
181
|
+
- lib/sidekiq_unique_jobs/script/logging.rb
|
182
|
+
- lib/sidekiq_unique_jobs/script/lua_error.rb
|
183
|
+
- lib/sidekiq_unique_jobs/script/script.rb
|
184
|
+
- lib/sidekiq_unique_jobs/script/scripts.rb
|
185
|
+
- lib/sidekiq_unique_jobs/script/template.rb
|
186
|
+
- lib/sidekiq_unique_jobs/script/timing.rb
|
198
187
|
- lib/sidekiq_unique_jobs/server.rb
|
199
188
|
- lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb
|
200
189
|
- lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb
|
@@ -234,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
234
223
|
- !ruby/object:Gem::Version
|
235
224
|
version: '0'
|
236
225
|
requirements: []
|
237
|
-
rubygems_version: 3.
|
226
|
+
rubygems_version: 3.5.5
|
238
227
|
signing_key:
|
239
228
|
specification_version: 4
|
240
229
|
summary: Sidekiq middleware that prevents duplicates jobs
|