sidekiq-amigo 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: b5204accb28834a6d5cb0baf274aa92e96f8ab2480316d6578c1e35e31075f44
4
+ data.tar.gz: c397ff5a38053be56726310f8b5e4e1b934c85c84f17fa60bb636e6d4b477252
5
+ SHA512:
6
+ metadata.gz: 04ce08ea274cf3132db350ecf826787e594faa47e1905fbc61307156d6b87f6a33b5301c05acf2b88b21aff52d600e52ae7cc31dd83f53e72ac972c5c2b56e0e
7
+ data.tar.gz: e46d7aab0d27953b1d16903cb98d77e8b8ddcbe56ea091950c76bdf60c18e3207cc085dc2c87e3f0e78c2f7e310325d4de8492c3d73f04babdcbf0a53be2a3f9
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "amigo/job"
4
+
5
+ class Amigo
6
+ class AuditLogger
7
+ include Sidekiq::Worker
8
+
9
+ def perform(event_json)
10
+ Amigo.log(self, :info, "async_job_audit",
11
+ event_id: event_json["id"],
12
+ event_name: event_json["name"],
13
+ event_payload: event_json["payload"],)
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "amigo/job"
4
+
5
+ # Put jobs here to die. If you just remove a job in Sidekiq, it may be queued up
6
+ # (like if it's scheduled or retrying),
7
+ # and will fail if the class does not exist.
8
+ #
9
+ # So, make the class exist, but noop so it won't be scheduled and won't be retried.
10
+ # Then it can be deleted later.
11
+ #
12
+ class Amigo
13
+ module DeprecatedJobs
14
+ def self.install(const_base, *names)
15
+ cls = self.noop_class
16
+ names.each { |n| self.__install_one(const_base, n, cls) }
17
+ end
18
+
19
+ def self.__install_one(const_base, cls_name, cls)
20
+ name_parts = cls_name.split("::").map(&:to_sym)
21
+ name_parts[0..-2].each do |part|
22
+ const_base = if const_base.const_defined?(part)
23
+ const_base.const_get(part)
24
+ else
25
+ const_base.const_set(part, Module.new)
26
+ end
27
+ end
28
+ const_base.const_set(name_parts.last, cls)
29
+ end
30
+
31
+ def self.noop_class
32
+ cls = Class.new do
33
+ def _perform(*)
34
+ Amigo.log(self, :warn, "deprecated_job_invoked", nil)
35
+ end
36
+ end
37
+ cls.extend(Amigo::Job)
38
+ return cls
39
+ end
40
+ end
41
+ end
data/lib/amigo/job.rb ADDED
@@ -0,0 +1,242 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sidekiq"
4
+
5
+ class Amigo
6
+ module Job
7
+ def self.extended(cls)
8
+ Amigo.registered_jobs << cls
9
+
10
+ cls.include(Sidekiq::Worker)
11
+ cls.extend(ClassMethods)
12
+ cls.pattern = ""
13
+ cls.include(InstanceMethods)
14
+ end
15
+
16
+ module InstanceMethods
17
+ def initialize(*)
18
+ super
19
+ @change_matchers = {}
20
+ end
21
+
22
+ def logger
23
+ return Sidekiq.logger
24
+ end
25
+
26
+ def perform(*args)
27
+ if args.empty?
28
+ event = nil
29
+ elsif args.size == 1
30
+ event = Amigo::Event.from_json(args[0])
31
+ else
32
+ raise "perform should always be called with no args or [Amigo::Event#as_json], got %p" % [args]
33
+ end
34
+ self._perform(event)
35
+ end
36
+
37
+ # Return +klass[payload_or_id]+ if it exists, or raise an error if it does not.
38
+ # +payload_or_id+ can be an integer,
39
+ # or the async job payload if the id is the first argument.
40
+ #
41
+ # Examples:
42
+ # - `customer = self.lookup_model( MyApp::Customer, event )`
43
+ # - `customer = self.lookup_model( MyApp::Customer, event.payload )`
44
+ # - `customer = self.lookup_model( MyApp::Customer, customer_id )`
45
+ def lookup_model(klass, payload_or_id)
46
+ if payload_or_id.is_a?(Integer)
47
+ id = payload_or_id
48
+ elsif payload_or_id.respond_to?(:payload)
49
+ id = payload_or_id.payload.first
50
+ elsif payload_or_id.respond_to?(:first)
51
+ id = payload_or_id.first
52
+ else
53
+ raise "Don't know how to handle #{payload_or_id}"
54
+ end
55
+ result = klass[id] or raise "%s[%p] does not exist" % [klass.name, id]
56
+ return result
57
+ end
58
+
59
+ # Create a matcher against the 'changed' hash of an update event.
60
+ #
61
+ # Example:
62
+ # on 'myapp.customer.update' do |payload, _|
63
+ # customerid, changes, _ = *payload
64
+ # customer = MyApp::Customer[ customerid ] or return false
65
+ #
66
+ # case changes
67
+ # when changed( :password )
68
+ # send_password_change_email( customer )
69
+ # when changed( :activation_code, to: nil )
70
+ # send_welcome_email( customer )
71
+ # when changed( :type, from: 'seeder', to: 'eater' )
72
+ # send_becoming_an_eater_email( customer )
73
+ # end
74
+ # end
75
+ def changed(field, values={})
76
+ unless @change_matchers[[field, values]]
77
+ match_proc = self.make_match_proc_for(field, values)
78
+ @change_matchers[[field, values]] = match_proc
79
+ end
80
+
81
+ return @change_matchers[[field, values]]
82
+ end
83
+
84
+ # Make a curried Proc for calling a match method with the given +field+ and +values+.
85
+ def make_match_proc_for(field, values)
86
+ return self.method(:match_any_change).to_proc.curry[field] if values.empty?
87
+
88
+ if values.key?(:from)
89
+ if values.key?(:to)
90
+ return self.method(:match_change_from_to).to_proc.
91
+ curry[ field, values[:from], values[:to] ]
92
+ else
93
+ return self.method(:match_change_from).to_proc.
94
+ curry[ field, values[:from] ]
95
+ end
96
+ elsif values.key?(:to)
97
+ return self.method(:match_change_to).to_proc.
98
+ curry[ field, values[:to] ]
99
+ else
100
+ raise ScriptError,
101
+ "Unhandled change option/s: %p; expected :to and/or :from" % [values.keys]
102
+ end
103
+ end
104
+
105
+ # Returns +true+ if the given +field+ is listed at all in the specified
106
+ # +changes+.
107
+ def match_any_change(field, changes)
108
+ self.logger.debug "Checking for existance of field %p in %p" % [field, changes]
109
+ return changes&.key?(field.to_s)
110
+ end
111
+
112
+ # Returns +true+ if the given +field+ is listed in the specified +changes+,
113
+ # and the value it changed to matches +to+. The +to+ value can be:
114
+ #
115
+ # a Regexp::
116
+ # The new value is stringified, and matched against the +to+ Regexp.
117
+ # an immediate value (String, Numeric, NilClass, etc.)::
118
+ # The new value is matched using Object#==.
119
+ #
120
+ def match_change_to(field, to, changes)
121
+ self.logger.debug "Checking for change to %p of field %p in %p" % [to, field, changes]
122
+ return false unless changes&.key?(field.to_s)
123
+
124
+ newval = changes[field.to_s][1]
125
+
126
+ case to
127
+ when NilClass, Numeric, String, TrueClass, FalseClass
128
+ return newval == to
129
+ when Regexp
130
+ return to.match(newval)
131
+ when Proc
132
+ return to[newval]
133
+ else
134
+ raise TypeError, "Unhandled type of 'to' criteria %p (a %p)" % [to, to.class]
135
+ end
136
+ end
137
+
138
+ # Returns +true+ if the given +field+ is listed in the specified +changes+,
139
+ # and the value it changed from matches +from+. The +from+ value can be:
140
+ #
141
+ # a Regexp::
142
+ # The old value is stringified, and matched against the +from+ Regexp.
143
+ # an immediate value (String, Numeric, NilClass, etc.)::
144
+ # The old value is matched using Object#==.
145
+ #
146
+ def match_change_from(field, from, changes)
147
+ self.logger.debug "Checking for change from %p of field %p in %p" % [from, field, changes]
148
+ return false unless changes&.key?(field.to_s)
149
+
150
+ oldval = changes[field.to_s][0]
151
+
152
+ case from
153
+ when NilClass, Numeric, String, TrueClass, FalseClass
154
+ return oldval == from
155
+ when Regexp
156
+ return from.match(oldval)
157
+ when Proc
158
+ return from[oldval]
159
+ else
160
+ raise TypeError, "Unhandled type of 'from' criteria %p (a %p)" % [from, from.class]
161
+ end
162
+ end
163
+
164
+ # Returns +true+ if the given +field+ is listed in the specified +changes+,
165
+ # and the value it changed from matches +from+ and the value it changed to
166
+ # matches +to+. The +from+ and +to+ values can be:
167
+ #
168
+ # a Regexp::
169
+ # The corresponding value is stringified, and matched against the Regexp.
170
+ # an immediate value (String, Numeric, NilClass, etc.)::
171
+ # The corresponding value is matched using Object#==.
172
+ #
173
+ def match_change_from_to(field, from, to, changes)
174
+ self.logger.debug "Checking for change from %p to %p of field %p in %p" %
175
+ [from, to, field, changes]
176
+ return false unless changes&.key?(field.to_s)
177
+ return self.match_change_to(field, to, changes) &&
178
+ self.match_change_from(field, from, changes)
179
+ end
180
+
181
+ # Create a matcher against a changed Hash JSON column of an update event.
182
+ #
183
+ # Example:
184
+ # on 'myapp.customer.update' do |payload, _|
185
+ # customerid, changes, _ = *payload
186
+ #
187
+ # case changes
188
+ # when changed_at( :flags, :trustworthy, to: true )
189
+ # mark_customer_safe_in_external_service( customerid )
190
+ # end
191
+ # end
192
+ def changed_at(field, index, values={})
193
+ return self.method(:match_change_at).to_proc.curry[field, index, values]
194
+ end
195
+
196
+ # Return +true+ if `field[index]` has changed in the specified +changes+,
197
+ # configured by +values+ (contains +from+, +to+, or whatever supported kwargs).
198
+ # Unlike other matches, +changes+ is expected to be an entire +Hash+
199
+ # and may contain a value at +index+ in either or both sets of changes.
200
+ # This method does not attempt to do the matching itself.
201
+ # Rather, it sets up the data to defer to the other matcher methods.
202
+ def match_change_at(field, index, values, changes)
203
+ field = field.to_s
204
+ return false unless changes&.key?(field)
205
+ index = index.to_s
206
+ old_values, new_values = changes[field]
207
+ unrolled_changes = self.unroll_hash_changes(old_values, new_values)
208
+ return self.changed(index, values)[unrolled_changes]
209
+ end
210
+
211
+ # Given two hashes that represent the before and after column hash values,
212
+ # return a hash in the usual "changes" format,
213
+ # where keys are the hash keys with changed values,
214
+ # and values are a tuple of the before and after values.
215
+ def unroll_hash_changes(old_hash, new_hash)
216
+ old_hash ||= {}
217
+ new_hash ||= {}
218
+ return old_hash.keys.concat(new_hash.keys).uniq.each_with_object({}) do |key, h|
219
+ old_val = old_hash[key]
220
+ new_val = new_hash[key]
221
+ h[key] = [old_val, new_val] if old_val != new_val
222
+ end
223
+ end
224
+ end
225
+
226
+ module ClassMethods
227
+ attr_accessor :pattern
228
+
229
+ def on(pattern)
230
+ self.pattern = pattern
231
+ end
232
+
233
+ def scheduled_job?
234
+ return false
235
+ end
236
+
237
+ def event_job?
238
+ return true
239
+ end
240
+ end
241
+ end
242
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "amigo/job"
4
+
5
+ class Amigo
6
+ class Router
7
+ include Sidekiq::Worker
8
+
9
+ def perform(event_json)
10
+ event_name = event_json["name"]
11
+ matches = Amigo.registered_event_jobs.
12
+ select { |job| File.fnmatch(job.pattern, event_name, File::FNM_EXTGLOB) }
13
+ matches.each do |job|
14
+ Amigo.synchronous_mode ? job.new.perform(event_json) : job.perform_async(event_json)
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sidekiq"
4
+ require "sidekiq-cron"
5
+
6
+ require "amigo"
7
+
8
+ class Amigo
9
+ module ScheduledJob
10
+ def self.extended(cls)
11
+ Amigo.registered_jobs << cls
12
+
13
+ cls.include(Sidekiq::Worker)
14
+ cls.sidekiq_options(retry: false)
15
+ cls.extend(ClassMethods)
16
+ cls.splay_duration = 30
17
+ cls.include(InstanceMethods)
18
+ end
19
+
20
+ module InstanceMethods
21
+ def logger
22
+ return Sidekiq.logger
23
+ end
24
+
25
+ def perform(*args)
26
+ if args.empty?
27
+ jitter = rand(0..self.class.splay_duration.to_i)
28
+ self.class.perform_in(jitter, true)
29
+ elsif args == [true]
30
+ self._perform
31
+ else
32
+ raise "ScheduledJob#perform must be called with no arguments, or [true]"
33
+ end
34
+ end
35
+ end
36
+
37
+ module ClassMethods
38
+ attr_accessor :cron_expr, :splay_duration
39
+
40
+ def scheduled_job?
41
+ return true
42
+ end
43
+
44
+ def event_job?
45
+ return false
46
+ end
47
+
48
+ # Return the UTC hour for the given hour and timezone.
49
+ # For example, during DST, `utc_hour(6, 'US/Pacific')` returns 13 (or, 6 + 7),
50
+ # while in standard time (not DST) it returns 8 (or, 6 + 8).
51
+ # This is useful in crontab notation, when we want something to happen at
52
+ # a certain local time and don't want it to shift with DST.
53
+ def utc_hour(hour, tzstr)
54
+ local = TZInfo::Timezone.get(tzstr)
55
+ utc = TZInfo::Timezone.get("UTC")
56
+ n = Time.now
57
+ intz = Time.new(n.year, n.month, n.day, hour, n.min, n.sec, local)
58
+ inutc = utc.to_local(intz)
59
+ return inutc.hour
60
+ end
61
+
62
+ def cron(expr)
63
+ self.cron_expr = expr
64
+ end
65
+
66
+ def splay(duration)
67
+ self.splay_duration = duration
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,234 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sidekiq/testing"
4
+
5
+ class Amigo
6
+ module SpecHelpers
7
+ def self.included(context)
8
+ Sidekiq::Testing.inline!
9
+ context.before(:each) do |example|
10
+ Amigo.synchronous_mode = true if example.metadata[:async]
11
+ end
12
+ context.after(:each) do |example|
13
+ Amigo.synchronous_mode = false if example.metadata[:async]
14
+ end
15
+ super
16
+ end
17
+
18
+ module_function def snapshot_async_state(opts={})
19
+ old_hooks = Amigo.subscribers.to_a
20
+ old_jobs = Amigo.registered_jobs.to_a
21
+ old_failure = Amigo.on_publish_error
22
+
23
+ begin
24
+ Amigo.on_publish_error = opts[:on_error] if opts.key?(:on_error)
25
+ Amigo.subscribers.replace(opts[:subscribers]) if opts.key?(:subscribers)
26
+ Amigo.registered_jobs.replace(opts[:jobs]) if opts.key?(:jobs)
27
+ yield
28
+ ensure
29
+ Amigo.on_publish_error = old_failure
30
+ Amigo.subscribers.replace(old_hooks)
31
+ Amigo.registered_jobs.replace(old_jobs)
32
+ end
33
+ end
34
+
35
+ class EventPublishedMatcher
36
+ attr_reader :recorded_events
37
+
38
+ def initialize(eventname, expected_payload=[])
39
+ @expected_events = [[eventname, expected_payload]]
40
+ @recorded_events = []
41
+ @missing = []
42
+ @matched = []
43
+ end
44
+
45
+ def and(another_eventname, *expected_payload)
46
+ @expected_events << [another_eventname, expected_payload]
47
+ return self
48
+ end
49
+
50
+ def with_payload(expected_payload)
51
+ raise ArgumentError, "expected payload must be an array or matcher" unless
52
+ expected_payload.is_a?(Array) || expected_payload.respond_to?(:matches?)
53
+ @expected_events.last[1] = expected_payload
54
+ return self
55
+ end
56
+
57
+ def record_event(event)
58
+ Amigo.log nil, :debug, "recording_event", event: event
59
+ @recorded_events << event
60
+ end
61
+
62
+ def supports_block_expectations?
63
+ true
64
+ end
65
+
66
+ def matches?(given_proc)
67
+ unless given_proc.respond_to?(:call)
68
+ warn "publish matcher used with non-proc object #{given_proc.inspect}"
69
+ return false
70
+ end
71
+
72
+ unless Amigo.synchronous_mode
73
+ warn "publish matcher used without synchronous_mode (use :async test metadata)"
74
+ return false
75
+ end
76
+
77
+ state = {on_error: self.method(:on_publish_error), subscribers: [self.method(:record_event)]}
78
+ Amigo::SpecHelpers.snapshot_async_state(state) do
79
+ given_proc.call
80
+ end
81
+
82
+ self.match_expected_events
83
+
84
+ return @error.nil? && @missing.empty?
85
+ end
86
+
87
+ def on_publish_error(err)
88
+ @error = err
89
+ end
90
+
91
+ def match_expected_events
92
+ @expected_events.each do |expected_event, expected_payload|
93
+ match = @recorded_events.find do |recorded|
94
+ recorded.name == expected_event && self.payloads_match?(expected_payload, recorded.payload)
95
+ end
96
+
97
+ if match
98
+ self.add_matched(expected_event, expected_payload)
99
+ else
100
+ self.add_missing(expected_event, expected_payload)
101
+ end
102
+ end
103
+ end
104
+
105
+ def payloads_match?(expected, recorded)
106
+ return expected.matches?(recorded) if expected.respond_to?(:matches?)
107
+ return expected.nil? || expected.empty? || expected == recorded
108
+ end
109
+
110
+ def add_matched(event, payload)
111
+ @matched << [event, payload]
112
+ end
113
+
114
+ def add_missing(event, payload)
115
+ @missing << [event, payload]
116
+ end
117
+
118
+ def failure_message
119
+ return "Error while publishing: %p" % [@error] if @error
120
+
121
+ messages = []
122
+
123
+ @missing.each do |event, payload|
124
+ message = "expected a '%s' event to be fired" % [event]
125
+ message << " with a payload of %p" % [payload] unless payload.nil?
126
+ message << " but none was."
127
+
128
+ messages << message
129
+ end
130
+
131
+ if @recorded_events.empty?
132
+ messages << "No events were sent."
133
+ else
134
+ parts = @recorded_events.map(&:inspect)
135
+ messages << "The following events were recorded: %s" % [parts.join(", ")]
136
+ end
137
+
138
+ return messages.join("\n")
139
+ end
140
+
141
+ def failure_message_when_negated
142
+ messages = []
143
+ @matched.each do |event, _payload|
144
+ message = "expected a '%s' event not to be fired" % [event]
145
+ message << " with a payload of %p" % [@expected_payload] if @expected_payload
146
+ message << " but one was."
147
+ messages << message
148
+ end
149
+
150
+ return messages.join("\n")
151
+ end
152
+ end
153
+
154
+ # RSpec matcher -- set up an expectation that an event will be fired
155
+ # with the specified +eventname+ and optional +expected_payload+.
156
+ #
157
+ # expect {
158
+ # Myapp::Customer.create( attributes )
159
+ # }.to publish( 'myapp.customer.create' )
160
+ #
161
+ # expect {
162
+ # Myapp::Customer.create( attributes )
163
+ # }.to publish( 'myapp.customer.create', [1] )
164
+ #
165
+ # expect { enter_hatch() }.
166
+ # to publish( 'myapp.hatch.entered' ).
167
+ # with_payload( [4, 8, 15, 16, 23, 42] )
168
+ #
169
+ # expect { cook_potatoes() }.
170
+ # to publish( 'myapp.potatoes.cook' ).
171
+ # with_payload( including( a_hash_containing( taste: 'good' ) ) )
172
+ #
173
+ def publish(eventname=nil, expected_payload=nil)
174
+ return EventPublishedMatcher.new(eventname, expected_payload)
175
+ end
176
+
177
+ class PerformAsyncJobMatcher
178
+ include RSpec::Matchers::Composable
179
+
180
+ def initialize(job)
181
+ @job = job
182
+ end
183
+
184
+ # RSpec matcher API -- specify that this matcher supports expect with a block.
185
+ def supports_block_expectations?
186
+ true
187
+ end
188
+
189
+ # Return +true+ if the +given_proc+ is a valid callable.
190
+ def valid_proc?(given_proc)
191
+ return true if given_proc.respond_to?(:call)
192
+
193
+ warn "`perform_async_job` was called with non-proc object #{given_proc.inspect}"
194
+ return false
195
+ end
196
+
197
+ # RSpec matcher API -- return +true+ if the specified job ran successfully.
198
+ def matches?(given_proc)
199
+ return false unless self.valid_proc?(given_proc)
200
+ return self.run_isolated_job(given_proc)
201
+ end
202
+
203
+ # Run +given_proc+ in a 'clean' async environment, where 'clean' means:
204
+ # - Async jobs are subscribed to events
205
+ # - The only registered job is the matcher's job
206
+ def run_isolated_job(given_proc)
207
+ unless Amigo.synchronous_mode
208
+ warn "publish matcher used without synchronous_mode (use :async test metadata)"
209
+ return false
210
+ end
211
+
212
+ state = {on_error: self.method(:on_publish_error), subscribers: [], jobs: [@job]}
213
+ Amigo::SpecHelpers.snapshot_async_state(state) do
214
+ Amigo.install_amigo_jobs
215
+ given_proc.call
216
+ end
217
+
218
+ return @error.nil?
219
+ end
220
+
221
+ def on_publish_error(err)
222
+ @error = err
223
+ end
224
+
225
+ def failure_message
226
+ return "Job errored: %p" % [@error]
227
+ end
228
+ end
229
+
230
+ def perform_async_job(job)
231
+ return PerformAsyncJobMatcher.new(job)
232
+ end
233
+ end
234
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Amigo
4
+ VERSION = "1.0.0"
5
+ end
data/lib/amigo.rb ADDED
@@ -0,0 +1,272 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "redis"
4
+ require "sidekiq"
5
+ require "sidekiq-cron"
6
+
7
+ # Host module and namespace for the Amigo async jobs system.
8
+ #
9
+ # The async job system is mostly decoupled into a few parts,
10
+ # so we can understand them in pieces.
11
+ # Those pieces are: Publish, Subscribe, Event Jobs, Routing, and Scheduled Jobs.
12
+ #
13
+ # Under the hood, the async job system uses Sidekiq.
14
+ # Sidekiq is a background job system that persists its data in Redis.
15
+ # Worker processes process the jobs off the queue.
16
+ #
17
+ # Publish
18
+ #
19
+ # The Amigo module has a very basic pub/sub system.
20
+ # You can use `Amigo.publish` to broadcast an event (event name and payload),
21
+ # and register subscribers to listen to the event.
22
+ # The actual event exchanged is a Amigo::Event, which is a simple wrapper object.
23
+ #
24
+ # Amigo.publish('myapp.auth.failed', email: params[:email])
25
+ #
26
+ # Subscribe
27
+ #
28
+ # Calling Amigo.register_subscriber registers a hook that listens for events published
29
+ # via Amigo.publish. All the subscriber does is send the events to the Router job.
30
+ #
31
+ # The subscriber should be enabled on clients that should emit events (so all web processes,
32
+ # console work that should have side effects like sending emails, and worker processes).
33
+ #
34
+ # Note that enabling the subscriber on worker processes means that it would be possible
35
+ # for a job to end up in an infinite loop
36
+ # (imagine if the audit logger, which records all published events, published an event).
37
+ # This is expected; be careful of infinite loops!
38
+ #
39
+ # Event Jobs
40
+ #
41
+ # Jobs must `include Amigo::Job`.
42
+ # As per best-practices when writing works, keep them as simple as possible,
43
+ # and put the business logic elsewhere.
44
+ #
45
+ # Standard jobs, which we call event-based jobs, generally respond to published events.
46
+ # Use the `on` method to define a glob pattern that is matched against event names:
47
+ #
48
+ # class Amigo::CustomerMailer
49
+ # include Amigo::Job
50
+ # on 'myapp.customer.created'
51
+ # def _perform(event)
52
+ # customer_id = event.payload.first
53
+ # # Send welcome email
54
+ # end
55
+ # end
56
+ #
57
+ # The 'on' pattern can be 'myapp.customer.*' to match all customer events for example,
58
+ # or '*' to match all events. The rules of matching follow File.fnmatch.
59
+ #
60
+ # Jobs must implement a `_perform` method, which takes a Amigo::Event.
61
+ # Note that normal Sidekiq workers use a 'perform' method that takes a variable number of arguments;
62
+ # the base Async::Job class has this method and delegates its business logic to the subclass _perform method.
63
+ #
64
+ # Routing
65
+ #
66
+ # There are two special workers that are important for the overall functioning of the system
67
+ # (and do not inherit from Job but rather than Sidekiq::Worker so they are not classified and treated as 'Jobs').
68
+ #
69
+ # The first is the AuditLogger, which is a basic job that logs all async events.
70
+ # This acts as a useful change log for the state of the database.
71
+ #
72
+ # The second special worker is the Router, which calls `perform` on the event Jobs
73
+ # that match the routing information, as explained in Jobs.
74
+ # It does this by filtering through all event-based jobs and performing the ones with a route match.
75
+ #
76
+ # Scheduled Jobs
77
+ #
78
+ # Scheduled jobs use the sidekiq-cron package: https://github.com/ondrejbartas/sidekiq-cron
79
+ # There is a separate base class, Amigo::ScheduledJob, that takes care of some standard job setup.
80
+ #
81
+ # To implement a scheduled job, `include Amigo::ScheduledJob`,
82
+ # call the `cron` method, and provide a `_perform` method.
83
+ # You can also use an optional `splay` method:
84
+ #
85
+ # class Amigo::CacheBuster
86
+ # include Amigo::ScheduledJob
87
+ # cron '*/10 * * * *'
88
+ # splay 60.seconds
89
+ # def _perform
90
+ # # Bust the cache
91
+ # end
92
+ # end
93
+ #
94
+ # This code will run once every 10 minutes or so (check out https://crontab.guru/ for testing cron expressions).
95
+ # The "or so" refers to the _splay_, which is a 'fuzz factor' of how close to the target interval
96
+ # the job may run. So in reality, this job will run every 9 to 11 minutes, due to the 60 second splay.
97
+ # Splay exists to avoid a "thundering herd" issue.
98
+ # Splay defaults to 30s; you may wish to always provide splay, whatever you think for your job.
99
+ #
100
+ class Amigo
101
+ require "amigo/job"
102
+ require "amigo/scheduled_job"
103
+ require "amigo/audit_logger"
104
+ require "amigo/router"
105
+
106
+ class << self
107
+ attr_accessor :structured_logging
108
+
109
+ # Proc called with [job, level, message, params].
110
+ # By default, logs to the job's logger (or Sidekiq's if job is nil).
111
+ # If structured_logging is true, the message will be an 'event' without any dynamic info,
112
+ # if false, the params will be rendered into the message so are suitable for unstructured logging.
113
+ attr_accessor :log_callback
114
+
115
+ def reset_logging
116
+ self.log_callback = ->(job, level, msg, _params) { (job || Sidekiq).logger.send(level, msg) }
117
+ self.structured_logging = false
118
+ end
119
+
120
+ def log(job, level, message, params)
121
+ if self.structured_logging
122
+ paramstr = params.map { |k, v| "#{k}=#{v}" }.join(" ")
123
+ message = "#{message} #{paramstr}"
124
+ end
125
+ self.log_callback[job, level, message, params]
126
+ end
127
+
128
+ # If true, perform event work synchronously rather than asynchronously.
129
+ # Only useful for testing.
130
+ attr_accessor :synchronous_mode
131
+
132
+ # Every subclass of Amigo::Job and Amigo::ScheduledJob goes here.
133
+ # It is used for routing and testing isolated jobs.
134
+ attr_accessor :registered_jobs
135
+
136
+ # An Array of callbacks to be run when an event is published.
137
+ attr_accessor :subscribers
138
+
139
+ # A single callback to be run when an event publication errors.
140
+ attr_accessor :on_publish_error
141
+
142
+ # Publish an event with the specified +eventname+ and +payload+
143
+ # to any configured publishers.
144
+ def publish(eventname, *payload)
145
+ ev = Event.new(SecureRandom.uuid, eventname, payload)
146
+
147
+ self.subscribers.to_a.each do |hook|
148
+ hook.call(ev)
149
+ rescue StandardError => e
150
+ self.log(nil, :error, "amigo_subscriber_hook_error", error: e, hook: hook, event: ev)
151
+ self.on_publish_error.call(e)
152
+ end
153
+ end
154
+
155
+ # Register a hook to be called when an event is sent.
156
+ def register_subscriber(&block)
157
+ raise LocalJumpError, "no block given" unless block
158
+ self.log nil, :info, "amigo_installed_subscriber", block: block
159
+ self.subscribers << block
160
+ return block
161
+ end
162
+
163
+ def unregister_subscriber(block_ref)
164
+ self.subscribers.delete(block_ref)
165
+ end
166
+
167
+ # Return an array of all Job subclasses that respond to event publishing (have patterns).
168
+ def registered_event_jobs
169
+ return self.registered_jobs.select(&:event_job?)
170
+ end
171
+
172
+ # Return an array of all Job subclasses that are scheduled (have intervals).
173
+ def registered_scheduled_jobs
174
+ return self.registered_jobs.select(&:scheduled_job?)
175
+ end
176
+ #
177
+ # Register a Amigo subscriber that will publish events to Sidekiq/Redis,
178
+ # for future routing.
179
+
180
+ # Install Amigo so that every publish will be sent to the AuditLogger job
181
+ # and will invoke the relevant jobs in registered_jobs via the Router job.
182
+ def install_amigo_jobs
183
+ return self.register_subscriber do |ev|
184
+ self._subscriber(ev)
185
+ end
186
+ end
187
+
188
+ def _subscriber(event)
189
+ event_json = event.as_json
190
+ Amigo::AuditLogger.perform_async(event_json)
191
+ Amigo::Router.perform_async(event_json)
192
+ end
193
+ #
194
+ # # Start the scheduler.
195
+ # # This should generally be run in the Sidekiq worker process,
196
+ # # not a webserver process.
197
+ # def self.start_scheduler
198
+ # hash = self.scheduled_jobs.each_with_object({}) do |job, memo|
199
+ # self.logger.info "Scheduling %s every %p" % [job.name, job.cron_expr]
200
+ # memo[job.name] = {
201
+ # "class" => job.name,
202
+ # "cron" => job.cron_expr,
203
+ # }
204
+ # end
205
+ # load_errs = Sidekiq::Cron::Job.load_from_hash hash
206
+ # raise "Errors loading sidekiq-cron jobs: %p" % [load_errs] if load_errs.present?
207
+ # end
208
+ end
209
+
210
+ class Event
211
+ def self.from_json(o)
212
+ return self.new(o["id"], o["name"], o["payload"])
213
+ end
214
+
215
+ attr_reader :id, :name, :payload
216
+
217
+ def initialize(id, name, payload)
218
+ @id = id
219
+ @name = name
220
+ @payload = payload.map { |p| self.safe_stringify(p) }
221
+ end
222
+
223
+ def inspect
224
+ return "#<%p:%#0x [%s] %s %p>" % [
225
+ self.class,
226
+ self.object_id * 2,
227
+ self.id,
228
+ self.name,
229
+ self.payload,
230
+ ]
231
+ end
232
+
233
+ def as_json(_opts={})
234
+ return {
235
+ "id" => self.id,
236
+ "name" => self.name,
237
+ "payload" => self.payload,
238
+ }
239
+ end
240
+
241
+ def to_json(opts={})
242
+ return JSON.dump(self.as_json(opts))
243
+ end
244
+
245
+ protected def safe_stringify(o)
246
+ return self.deep_stringify_keys(o) if o.is_a?(Hash)
247
+ return o
248
+ end
249
+
250
+ def deep_stringify_keys(hash)
251
+ stringified_hash = {}
252
+ hash.each do |k, v|
253
+ stringified_hash[k.to_s] =
254
+ case v
255
+ when Hash
256
+ self.deep_stringify_keys(v)
257
+ when Array
258
+ v.map { |i| i.is_a?(Hash) ? self.deep_stringify_keys(i) : i }
259
+ else
260
+ v
261
+ end
262
+ end
263
+ stringified_hash
264
+ end
265
+ end
266
+ end
267
+
268
+ Amigo.reset_logging
269
+ Amigo.synchronous_mode = false
270
+ Amigo.registered_jobs = []
271
+ Amigo.subscribers = Set.new
272
+ Amigo.on_publish_error = proc {}
metadata ADDED
@@ -0,0 +1,165 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sidekiq-amigo
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Lithic Technology
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2022-03-04 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: sidekiq
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '6'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '6'
27
+ - !ruby/object:Gem::Dependency
28
+ name: sidekiq-cron
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rack
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '2.2'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '2.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rspec
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '3.10'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '3.10'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rspec-core
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '3.10'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '3.10'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rubocop
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1.11'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.11'
97
+ - !ruby/object:Gem::Dependency
98
+ name: rubocop-performance
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '1.10'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '1.10'
111
+ - !ruby/object:Gem::Dependency
112
+ name: timecop
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ description: 'sidekiq-amigo provides a pubsub system and other enhancements around
126
+ Sidekiq.
127
+
128
+ '
129
+ email: hello@lithic.tech
130
+ executables: []
131
+ extensions: []
132
+ extra_rdoc_files: []
133
+ files:
134
+ - lib/amigo.rb
135
+ - lib/amigo/audit_logger.rb
136
+ - lib/amigo/deprecated_jobs.rb
137
+ - lib/amigo/job.rb
138
+ - lib/amigo/router.rb
139
+ - lib/amigo/scheduled_job.rb
140
+ - lib/amigo/spec_helpers.rb
141
+ - lib/amigo/version.rb
142
+ homepage: https://github.com/lithictech/sidekiq-amigo
143
+ licenses:
144
+ - MIT
145
+ metadata: {}
146
+ post_install_message:
147
+ rdoc_options: []
148
+ require_paths:
149
+ - lib
150
+ required_ruby_version: !ruby/object:Gem::Requirement
151
+ requirements:
152
+ - - ">="
153
+ - !ruby/object:Gem::Version
154
+ version: 2.7.0
155
+ required_rubygems_version: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: '0'
160
+ requirements: []
161
+ rubygems_version: 3.1.6
162
+ signing_key:
163
+ specification_version: 4
164
+ summary: Pubsub system and other enhancements around Sidekiq.
165
+ test_files: []