sidekiq-amigo 1.10.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/amigo/audit_logger.rb +1 -1
- data/lib/amigo/autoscaler.rb +20 -4
- data/lib/amigo/job.rb +1 -1
- data/lib/amigo/memory_pressure.rb +16 -3
- data/lib/amigo/retry.rb +5 -3
- data/lib/amigo/router.rb +1 -1
- data/lib/amigo/scheduled_job.rb +1 -1
- data/lib/amigo/semaphore_backoff_job.rb +4 -4
- data/lib/amigo/spec_helpers.rb +7 -6
- data/lib/amigo/version.rb +1 -1
- data/lib/amigo.rb +4 -5
- metadata +12 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5e1aa001f6060b1e13decb95357627151007e8f17066cef3401021103a3cd7a2
|
4
|
+
data.tar.gz: 955cb548d0739f5fa51bc0fe887a4e0df5d15c154ee5470e4bdba4883a308af4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9fb081204d9465257ac48df3193925bb547961971fc0db39ea235584e9f4b9acef7ab2f93eaed60c1dcd4d1de759d09daadc5a94ec89473e600eb72b67c4afc7
|
7
|
+
data.tar.gz: bea97ecddf27912029035a941e6ee3aff653c0f8ba958e27f21a539f984a987c017311aa21b5f306832bb2240a0f487522e0cd63bf5a87988e5e568ebf503c7b
|
data/lib/amigo/audit_logger.rb
CHANGED
data/lib/amigo/autoscaler.rb
CHANGED
@@ -35,6 +35,13 @@ module Amigo
|
|
35
35
|
class Autoscaler
|
36
36
|
class InvalidHandler < StandardError; end
|
37
37
|
|
38
|
+
# Struct representing data serialized to Redis.
|
39
|
+
# Useful for diagnostics. Can be retried with +fetch_persisted+.
|
40
|
+
# @!attribute last_alerted_at [Time] 0-time if there is no recent alert.
|
41
|
+
# @!attribute depth [Integer] 0 if not in a latency event.
|
42
|
+
# @!attribute latency_event_started_at [Time] 0-time if not in a latency event.
|
43
|
+
Persisted = Struct.new(:last_alerted_at, :depth, :latency_event_started_at)
|
44
|
+
|
38
45
|
# How often should Autoscaler check for latency?
|
39
46
|
# @return [Integer]
|
40
47
|
attr_reader :poll_interval
|
@@ -139,10 +146,19 @@ module Amigo
|
|
139
146
|
@alert_methods = self.handlers.map { |a| _handler_to_method("alert_", a) }
|
140
147
|
@restored_methods = self.latency_restored_handlers.map { |a| _handler_to_method("alert_restored_", a) }
|
141
148
|
@stop = false
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
149
|
+
persisted = self.fetch_persisted
|
150
|
+
@last_alerted = persisted.last_alerted_at
|
151
|
+
@depth = persisted.depth
|
152
|
+
@latency_event_started = persisted.latency_event_started_at
|
153
|
+
end
|
154
|
+
|
155
|
+
def fetch_persisted
|
156
|
+
return Sidekiq.redis do |r|
|
157
|
+
Persisted.new(
|
158
|
+
Time.at((r.get("#{namespace}/last_alerted") || 0).to_f),
|
159
|
+
(r.get("#{namespace}/depth") || 0).to_i,
|
160
|
+
Time.at((r.get("#{namespace}/latency_event_started") || 0).to_f),
|
161
|
+
)
|
146
162
|
end
|
147
163
|
end
|
148
164
|
|
data/lib/amigo/job.rb
CHANGED
@@ -74,10 +74,23 @@ module Amigo
|
|
74
74
|
return percentage > self.threshold
|
75
75
|
end
|
76
76
|
|
77
|
-
|
78
|
-
|
79
|
-
|
77
|
+
def get_memory_info
|
78
|
+
s = self.get_memory_info_string
|
79
|
+
return self.parse_memory_string(s)
|
80
|
+
end
|
81
|
+
|
82
|
+
protected def get_memory_info_string
|
83
|
+
s = Sidekiq.redis do |c|
|
84
|
+
c.call("INFO", "MEMORY")
|
80
85
|
end
|
86
|
+
return s
|
87
|
+
end
|
88
|
+
|
89
|
+
protected def parse_memory_string(s)
|
90
|
+
# See bottom of https://redis.io/docs/latest/commands/info/ for format.
|
91
|
+
pairs = s.split("\r\n").reject { |line| line.start_with?("#") }.map { |pair| pair.split(":", 2) }
|
92
|
+
h = pairs.to_h
|
93
|
+
return h
|
81
94
|
end
|
82
95
|
end
|
83
96
|
end
|
data/lib/amigo/retry.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
require "sidekiq"
|
4
4
|
require "sidekiq/api"
|
5
5
|
|
6
|
-
# Middleware so Sidekiq
|
6
|
+
# Middleware so Sidekiq jobs can use a custom retry logic.
|
7
7
|
# See +Amigo::Retry::Retry+, +Amigo::Retry::Die+,
|
8
8
|
# and +Amigo::Retry::OrDie+ for more details
|
9
9
|
# on how these should be used.
|
@@ -83,6 +83,8 @@ module Amigo
|
|
83
83
|
end
|
84
84
|
|
85
85
|
class ServerMiddleware
|
86
|
+
include Sidekiq::ServerMiddleware
|
87
|
+
|
86
88
|
def call(worker, job, _queue)
|
87
89
|
yield
|
88
90
|
rescue Amigo::Retry::Retry => e
|
@@ -120,14 +122,14 @@ module Amigo
|
|
120
122
|
end
|
121
123
|
end
|
122
124
|
|
123
|
-
def amigo_retry_in(
|
125
|
+
def amigo_retry_in(job_class, item, interval)
|
124
126
|
# pulled from perform_in
|
125
127
|
int = interval.to_f
|
126
128
|
now = Time.now.to_f
|
127
129
|
ts = (int < 1_000_000_000 ? now + int : int)
|
128
130
|
item["at"] = ts if ts > now
|
129
131
|
item["retry_count"] = item.fetch("retry_count", 0) + 1
|
130
|
-
|
132
|
+
job_class.client_push(item)
|
131
133
|
end
|
132
134
|
end
|
133
135
|
end
|
data/lib/amigo/router.rb
CHANGED
data/lib/amigo/scheduled_job.rb
CHANGED
@@ -33,7 +33,7 @@ require "amigo/memory_pressure"
|
|
33
33
|
# - `semaphore_expiry` should return the TTL of the semaphore key.
|
34
34
|
# Defaults to 30 seconds. See below for key expiry and negative semaphore value details.
|
35
35
|
# - `before_perform` is called before calling the `perform` method.
|
36
|
-
# This is required so that implementers can set
|
36
|
+
# This is required so that implementers can set job state, based on job arguments,
|
37
37
|
# that can be used for calculating the semaphore key.
|
38
38
|
#
|
39
39
|
# Note that we give the semaphore key an expiry. This is to avoid situation where
|
@@ -41,7 +41,7 @@ require "amigo/memory_pressure"
|
|
41
41
|
# have fewer than the expected number of jobs running.
|
42
42
|
#
|
43
43
|
# This does mean that, when a job runs longer than the semaphore expiry,
|
44
|
-
# another
|
44
|
+
# another job can be started, which would increment the counter back to 1.
|
45
45
|
# When the original job ends, the counter would be 0; then when the new job ends,
|
46
46
|
# the counter would be -1. To avoid negative counters (which create the same issue
|
47
47
|
# around missing decrements), if we ever detect a negative 'jobs running',
|
@@ -78,11 +78,11 @@ module Amigo
|
|
78
78
|
|
79
79
|
module InstanceMethods
|
80
80
|
def semaphore_key
|
81
|
-
raise NotImplementedError, "must be implemented on
|
81
|
+
raise NotImplementedError, "must be implemented on job"
|
82
82
|
end
|
83
83
|
|
84
84
|
def semaphore_size
|
85
|
-
raise NotImplementedError, "must be implemented on
|
85
|
+
raise NotImplementedError, "must be implemented on job"
|
86
86
|
end
|
87
87
|
|
88
88
|
def semaphore_backoff
|
data/lib/amigo/spec_helpers.rb
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "amigo"
|
4
|
-
require "sidekiq/worker"
|
5
4
|
|
6
5
|
module Amigo
|
7
6
|
module SpecHelpers
|
@@ -248,7 +247,7 @@ module Amigo
|
|
248
247
|
return PerformAsyncJobMatcher.new(job)
|
249
248
|
end
|
250
249
|
|
251
|
-
# Like a Sidekiq
|
250
|
+
# Like a Sidekiq job's perform_inline,
|
252
251
|
# but allows an arbitrary item to be used, rather than just the
|
253
252
|
# given class and args. For example, when testing,
|
254
253
|
# you may need to assume something like 'retry_count' is in the job payload,
|
@@ -256,18 +255,18 @@ module Amigo
|
|
256
255
|
# This allows those arbitrary job payload fields
|
257
256
|
# to be included when the job is run.
|
258
257
|
module_function def sidekiq_perform_inline(klass, args, item=nil)
|
259
|
-
Sidekiq::
|
258
|
+
Sidekiq::Job::Setter.override_item = item
|
260
259
|
begin
|
261
260
|
klass.perform_inline(*args)
|
262
261
|
ensure
|
263
|
-
Sidekiq::
|
262
|
+
Sidekiq::Job::Setter.override_item = nil
|
264
263
|
end
|
265
264
|
end
|
266
265
|
|
267
266
|
module_function def drain_sidekiq_jobs(q)
|
268
267
|
all_sidekiq_jobs(q).each do |job|
|
269
268
|
klass = job.item.fetch("class")
|
270
|
-
klass =
|
269
|
+
klass = Object.const_get(klass) if klass.is_a?(String)
|
271
270
|
sidekiq_perform_inline(klass, job.item["args"], job.item)
|
272
271
|
job.delete
|
273
272
|
end
|
@@ -282,6 +281,8 @@ module Amigo
|
|
282
281
|
# Use this middleware to pass an arbitrary callback evaluated before a job runs.
|
283
282
|
# Make sure to call +reset+ after the test.
|
284
283
|
class ServerCallbackMiddleware
|
284
|
+
include Sidekiq::ServerMiddleware
|
285
|
+
|
285
286
|
class << self
|
286
287
|
attr_accessor :callback
|
287
288
|
end
|
@@ -304,7 +305,7 @@ module Amigo
|
|
304
305
|
end
|
305
306
|
|
306
307
|
module ::Sidekiq
|
307
|
-
module
|
308
|
+
module Job
|
308
309
|
class Setter
|
309
310
|
class << self
|
310
311
|
attr_accessor :override_item
|
data/lib/amigo/version.rb
CHANGED
data/lib/amigo.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "redis"
|
4
3
|
require "sidekiq"
|
5
4
|
require "sidekiq-cron"
|
6
5
|
|
@@ -61,18 +60,18 @@ require "sidekiq-cron"
|
|
61
60
|
# to control the matching rules more closely than File.fnmatch can provide.
|
62
61
|
#
|
63
62
|
# Jobs must implement a `_perform` method, which takes a Amigo::Event.
|
64
|
-
# Note that normal Sidekiq
|
63
|
+
# Note that normal Sidekiq jobs use a 'perform' method that takes a variable number of arguments;
|
65
64
|
# the base Async::Job class has this method and delegates its business logic to the subclass _perform method.
|
66
65
|
#
|
67
66
|
# Routing
|
68
67
|
#
|
69
|
-
# There are two special
|
70
|
-
# (and do not inherit from Job but rather than Sidekiq::
|
68
|
+
# There are two special jobs that are important for the overall functioning of the system
|
69
|
+
# (and do not inherit from Job but rather than Sidekiq::Job so they are not classified and treated as 'Jobs').
|
71
70
|
#
|
72
71
|
# The first is the AuditLogger, which is a basic job that logs all async events.
|
73
72
|
# This acts as a useful change log for the state of the database.
|
74
73
|
#
|
75
|
-
# The second special
|
74
|
+
# The second special job is the Router, which calls `perform` on the event Jobs
|
76
75
|
# that match the routing information, as explained in Jobs.
|
77
76
|
# It does this by filtering through all event-based jobs and performing the ones with a route match.
|
78
77
|
#
|
metadata
CHANGED
@@ -1,43 +1,42 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-amigo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lithic Technology
|
8
|
-
autorequire:
|
9
8
|
bindir: bin
|
10
9
|
cert_chain: []
|
11
|
-
date:
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
12
11
|
dependencies:
|
13
12
|
- !ruby/object:Gem::Dependency
|
14
13
|
name: sidekiq
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
|
-
- - "
|
16
|
+
- - ">="
|
18
17
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
18
|
+
version: '7'
|
20
19
|
type: :runtime
|
21
20
|
prerelease: false
|
22
21
|
version_requirements: !ruby/object:Gem::Requirement
|
23
22
|
requirements:
|
24
|
-
- - "
|
23
|
+
- - ">="
|
25
24
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
25
|
+
version: '7'
|
27
26
|
- !ruby/object:Gem::Dependency
|
28
27
|
name: sidekiq-cron
|
29
28
|
requirement: !ruby/object:Gem::Requirement
|
30
29
|
requirements:
|
31
30
|
- - "~>"
|
32
31
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
32
|
+
version: '2'
|
34
33
|
type: :runtime
|
35
34
|
prerelease: false
|
36
35
|
version_requirements: !ruby/object:Gem::Requirement
|
37
36
|
requirements:
|
38
37
|
- - "~>"
|
39
38
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
39
|
+
version: '2'
|
41
40
|
- !ruby/object:Gem::Dependency
|
42
41
|
name: platform-api
|
43
42
|
requirement: !ruby/object:Gem::Requirement
|
@@ -58,14 +57,14 @@ dependencies:
|
|
58
57
|
requirements:
|
59
58
|
- - "~>"
|
60
59
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
60
|
+
version: '3.1'
|
62
61
|
type: :development
|
63
62
|
prerelease: false
|
64
63
|
version_requirements: !ruby/object:Gem::Requirement
|
65
64
|
requirements:
|
66
65
|
- - "~>"
|
67
66
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
67
|
+
version: '3.1'
|
69
68
|
- !ruby/object:Gem::Dependency
|
70
69
|
name: rspec
|
71
70
|
requirement: !ruby/object:Gem::Requirement
|
@@ -193,7 +192,6 @@ licenses:
|
|
193
192
|
- MIT
|
194
193
|
metadata:
|
195
194
|
rubygems_mfa_required: 'true'
|
196
|
-
post_install_message:
|
197
195
|
rdoc_options: []
|
198
196
|
require_paths:
|
199
197
|
- lib
|
@@ -201,15 +199,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
201
199
|
requirements:
|
202
200
|
- - ">="
|
203
201
|
- !ruby/object:Gem::Version
|
204
|
-
version: 3.
|
202
|
+
version: 3.2.0
|
205
203
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
206
204
|
requirements:
|
207
205
|
- - ">="
|
208
206
|
- !ruby/object:Gem::Version
|
209
207
|
version: '0'
|
210
208
|
requirements: []
|
211
|
-
rubygems_version: 3.
|
212
|
-
signing_key:
|
209
|
+
rubygems_version: 3.6.7
|
213
210
|
specification_version: 4
|
214
211
|
summary: Pubsub system and other enhancements around Sidekiq.
|
215
212
|
test_files: []
|