sidekiq-unique-jobs 8.0.5 → 8.0.9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +83 -5
  3. data/README.md +1 -3
  4. data/lib/sidekiq_unique_jobs/batch_delete.rb +1 -1
  5. data/lib/sidekiq_unique_jobs/changelog.rb +1 -1
  6. data/lib/sidekiq_unique_jobs/config.rb +4 -4
  7. data/lib/sidekiq_unique_jobs/constants.rb +1 -0
  8. data/lib/sidekiq_unique_jobs/job.rb +4 -4
  9. data/lib/sidekiq_unique_jobs/lock/base_lock.rb +1 -1
  10. data/lib/sidekiq_unique_jobs/lock/while_executing.rb +2 -0
  11. data/lib/sidekiq_unique_jobs/lock.rb +13 -10
  12. data/lib/sidekiq_unique_jobs/locksmith.rb +3 -3
  13. data/lib/sidekiq_unique_jobs/lua/unlock.lua +16 -9
  14. data/lib/sidekiq_unique_jobs/middleware/client.rb +1 -1
  15. data/lib/sidekiq_unique_jobs/middleware/server.rb +1 -1
  16. data/lib/sidekiq_unique_jobs/middleware.rb +5 -5
  17. data/lib/sidekiq_unique_jobs/orphans/manager.rb +6 -5
  18. data/lib/sidekiq_unique_jobs/orphans/reaper.rb +2 -1
  19. data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb +26 -7
  20. data/lib/sidekiq_unique_jobs/redis/sorted_set.rb +8 -0
  21. data/lib/sidekiq_unique_jobs/script/caller.rb +1 -1
  22. data/lib/sidekiq_unique_jobs/script/client.rb +94 -0
  23. data/lib/sidekiq_unique_jobs/script/config.rb +68 -0
  24. data/lib/sidekiq_unique_jobs/script/dsl.rb +60 -0
  25. data/lib/sidekiq_unique_jobs/script/logging.rb +95 -0
  26. data/lib/sidekiq_unique_jobs/script/lua_error.rb +96 -0
  27. data/lib/sidekiq_unique_jobs/script/script.rb +75 -0
  28. data/lib/sidekiq_unique_jobs/script/scripts.rb +123 -0
  29. data/lib/sidekiq_unique_jobs/script/template.rb +41 -0
  30. data/lib/sidekiq_unique_jobs/script/timing.rb +35 -0
  31. data/lib/sidekiq_unique_jobs/script.rb +32 -1
  32. data/lib/sidekiq_unique_jobs/server.rb +2 -0
  33. data/lib/sidekiq_unique_jobs/sidekiq_unique_ext.rb +13 -35
  34. data/lib/sidekiq_unique_jobs/sidekiq_unique_jobs.rb +4 -4
  35. data/lib/sidekiq_unique_jobs/sidekiq_worker_methods.rb +3 -3
  36. data/lib/sidekiq_unique_jobs/testing.rb +4 -4
  37. data/lib/sidekiq_unique_jobs/version.rb +1 -1
  38. data/lib/sidekiq_unique_jobs/web/helpers.rb +3 -3
  39. data/lib/sidekiq_unique_jobs/web/views/changelogs.erb +1 -1
  40. data/lib/sidekiq_unique_jobs/web/views/lock.erb +2 -2
  41. data/lib/sidekiq_unique_jobs/web/views/locks.erb +1 -1
  42. data/lib/sidekiq_unique_jobs/web.rb +18 -17
  43. data/lib/sidekiq_unique_jobs.rb +5 -4
  44. metadata +12 -23
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Script
5
+ #
6
+ # Class holding gem configuration
7
+ #
8
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
9
+ class Config
10
+ #
11
+ # @!attribute [r] logger
12
+ # @return [Logger] a logger to use for debugging
13
+ attr_reader :logger
14
+ #
15
+ # @!attribute [r] scripts_path
16
+ # @return [Pathname] a directory with lua scripts
17
+ attr_reader :scripts_path
18
+
19
+ #
20
+ # Initialize a new instance of {Config}
21
+ #
22
+ #
23
+ def initialize
24
+ @conn = RedisClient.new
25
+ @logger = Logger.new($stdout)
26
+ @scripts_path = nil
27
+ end
28
+
29
+ #
30
+ # Sets a value for scripts_path
31
+ #
32
+ # @param [String, Pathname] obj <description>
33
+ #
34
+ # @raise [ArgumentError] when directory does not exist
35
+ # @raise [ArgumentError] when argument isn't supported
36
+ #
37
+ # @return [Pathname]
38
+ #
39
+ def scripts_path=(obj)
40
+ raise ArgumentError, "#{obj} should be a Pathname or String" unless obj.is_a?(Pathname) || obj.is_a?(String)
41
+ raise ArgumentError, "#{obj} does not exist" unless Dir.exist?(obj.to_s)
42
+
43
+ @scripts_path =
44
+ case obj
45
+ when String
46
+ Pathname.new(obj)
47
+ else
48
+ obj
49
+ end
50
+ end
51
+
52
+ #
53
+ # Sets a value for logger
54
+ #
55
+ # @param [Logger] obj a logger to use
56
+ #
57
+ # @raise [ArgumentError] when given argument isn't a Logger
58
+ #
59
+ # @return [Logger]
60
+ #
61
+ def logger=(obj)
62
+ raise ArgumentError, "#{obj} should be a Logger" unless obj.is_a?(Logger)
63
+
64
+ @logger = obj
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Script
5
+ # Interface to dealing with .lua files
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ module DSL
9
+ MUTEX = Mutex.new
10
+
11
+ def self.included(base)
12
+ base.class_eval do
13
+ extend ClassMethods
14
+ end
15
+ end
16
+
17
+ #
18
+ # Module ClassMethods extends the base class with necessary methods
19
+ #
20
+ # @author Mikael Henriksson <mikael@zoolutions.se>
21
+ #
22
+ module ClassMethods
23
+ def execute(file_name, conn, keys: [], argv: [])
24
+ SidekiqUniqueJobs::Script::Client
25
+ .new(config)
26
+ .execute(file_name, conn, keys: keys, argv: argv)
27
+ end
28
+
29
+ # Configure the gem
30
+ #
31
+ # This is usually called once at startup of an application
32
+ # @param [Hash] options global gem options
33
+ # @option options [String, Pathname] :path
34
+ # @option options [Logger] :logger (default is Logger.new(STDOUT))
35
+ # @yield control to the caller when given block
36
+ def configure(options = {})
37
+ if block_given?
38
+ yield config
39
+ else
40
+ options.each do |key, val|
41
+ config.send(:"#{key}=", val)
42
+ end
43
+ end
44
+ end
45
+
46
+ #
47
+ # The current configuration (See: {.configure} on how to configure)
48
+ #
49
+ #
50
+ # @return [Script::Config] the gem configuration
51
+ #
52
+ def config
53
+ MUTEX.synchronize do
54
+ @config ||= Config.new
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Script
5
+ # Utility module for reducing the number of uses of logger.
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ module Logging
9
+ def self.included(base)
10
+ base.send(:extend, self)
11
+ end
12
+
13
+ #
14
+ # A convenience method for using the configured gem logger
15
+ #
16
+ # @see Script#.logger
17
+ #
18
+ # @return [Logger]
19
+ #
20
+ def logger
21
+ SidekiqUniqueJobs::Script.logger
22
+ end
23
+
24
+ #
25
+ # Logs a message at debug level
26
+ #
27
+ # @param [String, Exception] message_or_exception the message or exception to log
28
+ #
29
+ # @return [void]
30
+ #
31
+ # @yield [String, Exception] the message or exception to use for log message
32
+ #
33
+ def log_debug(message_or_exception = nil, &block)
34
+ logger.debug(message_or_exception, &block)
35
+ nil
36
+ end
37
+
38
+ #
39
+ # Logs a message at info level
40
+ #
41
+ # @param [String, Exception] message_or_exception the message or exception to log
42
+ #
43
+ # @return [void]
44
+ #
45
+ # @yield [String, Exception] the message or exception to use for log message
46
+ #
47
+ def log_info(message_or_exception = nil, &block)
48
+ logger.info(message_or_exception, &block)
49
+ nil
50
+ end
51
+
52
+ #
53
+ # Logs a message at warn level
54
+ #
55
+ # @param [String, Exception] message_or_exception the message or exception to log
56
+ #
57
+ # @return [void]
58
+ #
59
+ # @yield [String, Exception] the message or exception to use for log message
60
+ #
61
+ def log_warn(message_or_exception = nil, &block)
62
+ logger.warn(message_or_exception, &block)
63
+ nil
64
+ end
65
+
66
+ #
67
+ # Logs a message at error level
68
+ #
69
+ # @param [String, Exception] message_or_exception the message or exception to log
70
+ #
71
+ # @return [void]
72
+ #
73
+ # @yield [String, Exception] the message or exception to use for log message
74
+ #
75
+ def log_error(message_or_exception = nil, &block)
76
+ logger.error(message_or_exception, &block)
77
+ nil
78
+ end
79
+
80
+ #
81
+ # Logs a message at fatal level
82
+ #
83
+ # @param [String, Exception] message_or_exception the message or exception to log
84
+ #
85
+ # @return [void]
86
+ #
87
+ # @yield [String, Exception] the message or exception to use for log message
88
+ #
89
+ def log_fatal(message_or_exception = nil, &block)
90
+ logger.fatal(message_or_exception, &block)
91
+ nil
92
+ end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Script
5
+ #
6
+ # Misconfiguration is raised when gem is misconfigured
7
+ #
8
+ # @author Mikael Henriksson <mikael@zoolutions.se>
9
+ #
10
+ class Misconfiguration < RuntimeError
11
+ end
12
+
13
+ # LuaError raised on errors in Lua scripts
14
+ #
15
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
16
+ class LuaError < RuntimeError
17
+ # Reformats errors raised by redis representing failures while executing
18
+ # a lua script. The default errors have confusing messages and backtraces,
19
+ # and a type of +RuntimeError+. This class improves the message and
20
+ # modifies the backtrace to include the lua script itself in a reasonable
21
+ # way.
22
+
23
+ PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/.freeze
24
+ LIB_PATH = File.expand_path("..", __dir__).freeze
25
+ CONTEXT_LINE_NUMBER = 2
26
+
27
+ attr_reader :error, :file, :content
28
+
29
+ # Is this error one that should be reformatted?
30
+ #
31
+ # @param error [StandardError] the original error raised by redis
32
+ # @return [Boolean] is this an error that should be reformatted?
33
+ def self.intercepts?(error)
34
+ PATTERN.match?(error.message)
35
+ end
36
+
37
+ # Initialize a new {LuaError} from an existing redis error, adjusting
38
+ # the message and backtrace in the process.
39
+ #
40
+ # @param error [StandardError] the original error raised by redis
41
+ # @param script [Script] a DTO with information about the script
42
+ #
43
+ def initialize(error, script)
44
+ @error = error
45
+ @file = script.path
46
+ @content = script.source
47
+ @backtrace = @error.backtrace
48
+
49
+ @error.message.match(PATTERN) do |regexp_match|
50
+ line_number = regexp_match[2].to_i
51
+ message = regexp_match[3]
52
+ error_context = generate_error_context(content, line_number)
53
+
54
+ super("#{message}\n\n#{error_context}\n\n")
55
+ set_backtrace(generate_backtrace(file, line_number))
56
+ end
57
+ end
58
+
59
+ private
60
+
61
+ # :nocov:
62
+ def generate_error_context(content, line_number)
63
+ lines = content.lines.to_a
64
+ beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
65
+ ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
66
+ line_number_width = ending_line_number.to_s.length
67
+
68
+ (beginning_line_number..ending_line_number).map do |number|
69
+ indicator = (number == line_number) ? "=>" : " "
70
+ formatted_number = format("%#{line_number_width}d", number)
71
+ " #{indicator} #{formatted_number}: #{lines[number - 1]}"
72
+ end.join.chomp
73
+ end
74
+
75
+ # :nocov:
76
+ def generate_backtrace(file, line_number)
77
+ pre_gem = backtrace_before_entering_gem(@backtrace)
78
+ index_of_first_gem_line = (@backtrace.size - pre_gem.size - 1)
79
+
80
+ pre_gem.unshift(@backtrace[index_of_first_gem_line])
81
+ pre_gem.unshift("#{file}:#{line_number}")
82
+ pre_gem
83
+ end
84
+
85
+ # :nocov:
86
+ def backtrace_before_entering_gem(backtrace)
87
+ backtrace.reverse.take_while { |line| !line_from_gem(line) }.reverse
88
+ end
89
+
90
+ # :nocov:
91
+ def line_from_gem(line)
92
+ line.split(":").first.include?(LIB_PATH)
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Script
5
+ # Interface to dealing with .lua files
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class Script
9
+ def self.load(name, root_path, conn)
10
+ script = new(name: name, root_path: root_path)
11
+ script.load(conn)
12
+ end
13
+
14
+ #
15
+ # @!attribute [r] script_name
16
+ # @return [Symbol, String] the name of the script without extension
17
+ attr_reader :name
18
+ #
19
+ # @!attribute [r] script_path
20
+ # @return [String] the path to the script on disk
21
+ attr_reader :path
22
+ #
23
+ # @!attribute [r] root_path
24
+ # @return [Pathname]
25
+ attr_reader :root_path
26
+ #
27
+ # @!attribute [r] source
28
+ # @return [String] the source code of the lua script
29
+ attr_reader :source
30
+ #
31
+ # @!attribute [rw] sha
32
+ # @return [String] the sha of the script
33
+ attr_reader :sha
34
+ #
35
+ # @!attribute [rw] call_count
36
+ # @return [Integer] the number of times the script was called/executed
37
+ attr_reader :call_count
38
+
39
+ def initialize(name:, root_path:)
40
+ @name = name
41
+ @root_path = root_path
42
+ @path = root_path.join("#{name}.lua").to_s
43
+ @source = render_file
44
+ @sha = compiled_sha
45
+ @call_count = 0
46
+ end
47
+
48
+ def ==(other)
49
+ sha == compiled_sha && compiled_sha == other.sha
50
+ end
51
+
52
+ def increment_call_count
53
+ @call_count += 1
54
+ end
55
+
56
+ def changed?
57
+ compiled_sha != sha
58
+ end
59
+
60
+ def render_file
61
+ Template.new(root_path).render(path)
62
+ end
63
+
64
+ def compiled_sha
65
+ Digest::SHA1.hexdigest(source)
66
+ end
67
+
68
+ def load(conn)
69
+ @sha = conn.script(:load, source)
70
+
71
+ self
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,123 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Script
5
+ # Interface to dealing with .lua files
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ class Scripts
9
+ #
10
+ # @return [Concurrent::Map] a map with configured script paths
11
+ SCRIPT_PATHS = Concurrent::Map.new
12
+
13
+ #
14
+ # Fetch a scripts configuration for path
15
+ #
16
+ # @param [Pathname] root_path the path to scripts
17
+ #
18
+ # @return [Scripts] a collection of scripts
19
+ #
20
+ def self.fetch(root_path)
21
+ if (scripts = SCRIPT_PATHS.get(root_path))
22
+ return scripts
23
+ end
24
+
25
+ create(root_path)
26
+ end
27
+
28
+ #
29
+ # Create a new scripts collection based on path
30
+ #
31
+ # @param [Pathname] root_path the path to scripts
32
+ #
33
+ # @return [Scripts] a collection of scripts
34
+ #
35
+ def self.create(root_path)
36
+ scripts = new(root_path)
37
+ store(scripts)
38
+ end
39
+
40
+ #
41
+ # Store the scripts collection in memory
42
+ #
43
+ # @param [Scripts] scripts the path to scripts
44
+ #
45
+ # @return [Scripts] the scripts instance that was stored
46
+ #
47
+ def self.store(scripts)
48
+ SCRIPT_PATHS.put(scripts.root_path, scripts)
49
+ scripts
50
+ end
51
+
52
+ #
53
+ # @!attribute [r] scripts
54
+ # @return [Concurrent::Map] a collection of loaded scripts
55
+ attr_reader :scripts
56
+
57
+ #
58
+ # @!attribute [r] root_path
59
+ # @return [Pathname] the path to the directory with lua scripts
60
+ attr_reader :root_path
61
+
62
+ def initialize(path)
63
+ raise ArgumentError, "path needs to be a Pathname" unless path.is_a?(Pathname)
64
+
65
+ @scripts = Concurrent::Map.new
66
+ @root_path = path
67
+ end
68
+
69
+ def fetch(name, conn)
70
+ if (script = scripts.get(name.to_sym))
71
+ return script
72
+ end
73
+
74
+ load(name, conn)
75
+ end
76
+
77
+ def load(name, conn)
78
+ script = Script.load(name, root_path, conn)
79
+ scripts.put(name.to_sym, script)
80
+
81
+ script
82
+ end
83
+
84
+ def delete(script)
85
+ if script.is_a?(Script)
86
+ scripts.delete(script.name)
87
+ else
88
+ scripts.delete(script.to_sym)
89
+ end
90
+ end
91
+
92
+ def kill(conn)
93
+ if conn.respond_to?(:namespace)
94
+ conn.redis.script(:kill)
95
+ else
96
+ conn.script(:kill)
97
+ end
98
+ end
99
+
100
+ #
101
+ # Execute a lua script with given name
102
+ #
103
+ # @note this method is recursive if we need to load a lua script
104
+ # that wasn't previously loaded.
105
+ #
106
+ # @param [Symbol] name the name of the script to execute
107
+ # @param [Redis] conn the redis connection to use for execution
108
+ # @param [Array<String>] keys script keys
109
+ # @param [Array<Object>] argv script arguments
110
+ #
111
+ # @return value from script
112
+ #
113
+ def execute(name, conn, keys: [], argv: [])
114
+ script = fetch(name, conn)
115
+ conn.evalsha(script.sha, keys, argv)
116
+ end
117
+
118
+ def count
119
+ scripts.keys.size
120
+ end
121
+ end
122
+ end
123
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ # Interface to dealing with .lua files
5
+ #
6
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
7
+ module Script
8
+ #
9
+ # Class Template provides LUA script partial template rendering
10
+ #
11
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
12
+ #
13
+ class Template
14
+ def initialize(script_path)
15
+ @script_path = script_path
16
+ end
17
+
18
+ #
19
+ # Renders a Lua script and includes any partials in that file
20
+ # all `<%= include_partial '' %>` replaced with the actual contents of the partial
21
+ #
22
+ # @param [Pathname] pathname the path to the
23
+ #
24
+ # @return [String] the rendered Luascript
25
+ #
26
+ def render(pathname)
27
+ @partial_templates ||= {}
28
+ ::ERB.new(File.read(pathname)).result(binding)
29
+ end
30
+
31
+ # helper method to include a lua partial within another lua script
32
+ #
33
+ def include_partial(relative_path)
34
+ return if @partial_templates.key?(relative_path)
35
+
36
+ @partial_templates[relative_path] = nil
37
+ render(Pathname.new("#{@script_path}/#{relative_path}"))
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SidekiqUniqueJobs
4
+ module Script
5
+ # Handles timing> of things
6
+ #
7
+ # @author Mikael Henriksson <mikael@mhenrixon.com>
8
+ module Timing
9
+ module_function
10
+
11
+ #
12
+ # Used for timing method calls
13
+ #
14
+ #
15
+ # @return [yield return, Float]
16
+ #
17
+ def timed
18
+ start_time = now
19
+
20
+ [yield, now - start_time]
21
+ end
22
+
23
+ #
24
+ # Returns a float representation of the current time.
25
+ # Either from Process or Time
26
+ #
27
+ #
28
+ # @return [Float]
29
+ #
30
+ def now
31
+ (Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i
32
+ end
33
+ end
34
+ end
35
+ end
@@ -1,15 +1,46 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "sidekiq_unique_jobs/script/template"
4
+ require "sidekiq_unique_jobs/script/lua_error"
5
+ require "sidekiq_unique_jobs/script/script"
6
+ require "sidekiq_unique_jobs/script/scripts"
7
+ require "sidekiq_unique_jobs/script/config"
8
+ require "sidekiq_unique_jobs/script/timing"
9
+ require "sidekiq_unique_jobs/script/logging"
10
+ require "sidekiq_unique_jobs/script/dsl"
11
+ require "sidekiq_unique_jobs/script/client"
12
+
3
13
  module SidekiqUniqueJobs
4
14
  # Interface to dealing with .lua files
5
15
  #
6
16
  # @author Mikael Henriksson <mikael@mhenrixon.com>
7
17
  module Script
8
- include Brpoplpush::RedisScript::DSL
18
+ include SidekiqUniqueJobs::Script::DSL
9
19
 
10
20
  configure do |config|
11
21
  config.scripts_path = Pathname.new(__FILE__).dirname.join("lua")
12
22
  config.logger = Sidekiq.logger # TODO: This becomes a little weird
13
23
  end
24
+
25
+ #
26
+ # The current logger
27
+ #
28
+ #
29
+ # @return [Logger] the configured logger
30
+ #
31
+ def self.logger
32
+ config.logger
33
+ end
34
+
35
+ #
36
+ # Set a new logger
37
+ #
38
+ # @param [Logger] other another logger
39
+ #
40
+ # @return [Logger] the new logger
41
+ #
42
+ def self.logger=(other)
43
+ config.logger = other
44
+ end
14
45
  end
15
46
  end
@@ -5,6 +5,8 @@ module SidekiqUniqueJobs
5
5
  #
6
6
  # @author Mikael Henriksson <mikael@mhenrixon.com>
7
7
  class Server
8
+ #
9
+ # @return [Proc] returns a default death handler for the gem to cleanup dead locks
8
10
  DEATH_HANDLER = (lambda do |job, _ex|
9
11
  return unless (digest = job["lock_digest"])
10
12