eventboss 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/.rspec +2 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +57 -0
- data/Guardfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +132 -0
- data/Rakefile +6 -0
- data/bin/eventboss +47 -0
- data/eventboss.gemspec +31 -0
- data/lib/eventboss.rb +79 -0
- data/lib/eventboss/configuration.rb +104 -0
- data/lib/eventboss/error_handlers/airbrake.rb +13 -0
- data/lib/eventboss/error_handlers/logger.rb +12 -0
- data/lib/eventboss/extensions.rb +2 -0
- data/lib/eventboss/fetcher.rb +33 -0
- data/lib/eventboss/instrumentation.rb +26 -0
- data/lib/eventboss/launcher.rb +96 -0
- data/lib/eventboss/listener.rb +28 -0
- data/lib/eventboss/logger.rb +34 -0
- data/lib/eventboss/logging.rb +8 -0
- data/lib/eventboss/long_poller.rb +75 -0
- data/lib/eventboss/manager.rb +116 -0
- data/lib/eventboss/polling/basic.rb +68 -0
- data/lib/eventboss/polling/timed_round_robin.rb +42 -0
- data/lib/eventboss/publisher.rb +32 -0
- data/lib/eventboss/queue.rb +42 -0
- data/lib/eventboss/queue_listener.rb +11 -0
- data/lib/eventboss/railtie.rb +5 -0
- data/lib/eventboss/runner.rb +60 -0
- data/lib/eventboss/safe_thread.rb +23 -0
- data/lib/eventboss/scripts.rb +1 -0
- data/lib/eventboss/sender.rb +25 -0
- data/lib/eventboss/sns_client.rb +55 -0
- data/lib/eventboss/unit_of_work.rb +33 -0
- data/lib/eventboss/version.rb +3 -0
- data/lib/eventboss/worker.rb +55 -0
- data/lib/tasks/eventboss.rake +47 -0
- metadata +192 -0
@@ -0,0 +1,68 @@
|
|
1
|
+
module Eventboss
|
2
|
+
module Polling
|
3
|
+
class Basic
|
4
|
+
PAUSE_AFTER_EMPTY = 2 # seconds
|
5
|
+
|
6
|
+
def initialize(queues, timer = Time)
|
7
|
+
@queues = queues.to_a
|
8
|
+
@timer = timer
|
9
|
+
@paused_until = @queues.each_with_object(Hash.new) do |queue, hash|
|
10
|
+
hash[queue] = @timer.at(0)
|
11
|
+
end
|
12
|
+
|
13
|
+
reset_next_queue
|
14
|
+
end
|
15
|
+
|
16
|
+
def next_queue
|
17
|
+
next_active_queue
|
18
|
+
end
|
19
|
+
|
20
|
+
def messages_found(queue, messages_count)
|
21
|
+
if messages_count == 0
|
22
|
+
pause(queue)
|
23
|
+
else
|
24
|
+
reset_next_queue
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def active_queues
|
29
|
+
@queues.reject { |q, _| queue_paused?(q) }
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def next_active_queue
|
35
|
+
reset_next_queue if queues_unpaused_since?
|
36
|
+
|
37
|
+
size = @queues.length
|
38
|
+
size.times do
|
39
|
+
queue = @queues[@next_queue_index]
|
40
|
+
@next_queue_index = (@next_queue_index + 1) % size
|
41
|
+
return queue unless queue_paused?(queue)
|
42
|
+
end
|
43
|
+
|
44
|
+
nil
|
45
|
+
end
|
46
|
+
|
47
|
+
def queues_unpaused_since?
|
48
|
+
last = @last_unpause_check
|
49
|
+
now = @last_unpause_check = @timer.now
|
50
|
+
|
51
|
+
last && @paused_until.values.any? { |t| t > last && t <= now }
|
52
|
+
end
|
53
|
+
|
54
|
+
def reset_next_queue
|
55
|
+
@next_queue_index = 0
|
56
|
+
end
|
57
|
+
|
58
|
+
def queue_paused?(queue)
|
59
|
+
@paused_until[queue] > @timer.now
|
60
|
+
end
|
61
|
+
|
62
|
+
def pause(queue)
|
63
|
+
return unless PAUSE_AFTER_EMPTY > 0
|
64
|
+
@paused_until[queue] = @timer.now + PAUSE_AFTER_EMPTY
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Eventboss
|
2
|
+
module Polling
|
3
|
+
class TimedRoundRobin
|
4
|
+
PAUSE_AFTER_EMPTY = 2 # seconds
|
5
|
+
|
6
|
+
def initialize(queues, timer = Time)
|
7
|
+
@queues = queues.to_a
|
8
|
+
@timer = timer
|
9
|
+
@next_queue_index = 0
|
10
|
+
@paused_until = @queues.each_with_object(Hash.new) do |queue, hash|
|
11
|
+
hash[queue] = @timer.at(0)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def next_queue
|
16
|
+
size = @queues.length
|
17
|
+
size.times do
|
18
|
+
queue = @queues[@next_queue_index]
|
19
|
+
@next_queue_index = (@next_queue_index + 1) % size
|
20
|
+
return queue unless queue_paused?(queue)
|
21
|
+
end
|
22
|
+
|
23
|
+
nil
|
24
|
+
end
|
25
|
+
|
26
|
+
def messages_found(queue, messages_count)
|
27
|
+
pause(queue) if messages_count == 0
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def queue_paused?(queue)
|
33
|
+
@paused_until[queue] > @timer.now
|
34
|
+
end
|
35
|
+
|
36
|
+
def pause(queue)
|
37
|
+
return unless PAUSE_AFTER_EMPTY > 0
|
38
|
+
@paused_until[queue] = @timer.now + PAUSE_AFTER_EMPTY
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Eventboss
|
2
|
+
class Publisher
|
3
|
+
def initialize(event_name, sns_client, configuration, opts = {})
|
4
|
+
@event_name = event_name
|
5
|
+
@sns_client = sns_client
|
6
|
+
@configuration = configuration
|
7
|
+
@generic = opts[:generic]
|
8
|
+
end
|
9
|
+
|
10
|
+
def publish(payload)
|
11
|
+
sns_client.publish({
|
12
|
+
topic_arn: topic_arn,
|
13
|
+
message: json_payload(payload)
|
14
|
+
})
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
attr_reader :event_name, :sns_client, :configuration
|
20
|
+
|
21
|
+
def json_payload(payload)
|
22
|
+
payload.is_a?(String) ? payload : payload.to_json
|
23
|
+
end
|
24
|
+
|
25
|
+
def topic_arn
|
26
|
+
src_selector = @generic ? "" : "-#{configuration.eventboss_app_name}"
|
27
|
+
|
28
|
+
"arn:aws:sns:#{configuration.eventboss_region}:#{configuration.eventboss_account_id}:\
|
29
|
+
eventboss#{src_selector}-#{event_name}-#{Eventboss.env}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Eventboss
|
2
|
+
class Queue
|
3
|
+
include Comparable
|
4
|
+
attr_reader :name
|
5
|
+
|
6
|
+
def self.build_name(source:, destination:, event:, env:, generic:)
|
7
|
+
source =
|
8
|
+
if generic
|
9
|
+
''
|
10
|
+
else
|
11
|
+
"-#{source}"
|
12
|
+
end
|
13
|
+
|
14
|
+
"#{destination}-eventboss#{source}-#{event}-#{env}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def initialize(name, configuration = Eventboss.configuration)
|
18
|
+
@client = configuration.sqs_client
|
19
|
+
@name = name
|
20
|
+
end
|
21
|
+
|
22
|
+
def url
|
23
|
+
@url ||= client.get_queue_url(queue_name: name).queue_url
|
24
|
+
end
|
25
|
+
|
26
|
+
def <=>(another_queue)
|
27
|
+
name <=> another_queue&.name
|
28
|
+
end
|
29
|
+
|
30
|
+
def eql?(another_queue)
|
31
|
+
name == another_queue&.name
|
32
|
+
end
|
33
|
+
|
34
|
+
def hash
|
35
|
+
name.hash
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
attr_reader :client
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
module Eventboss
|
2
|
+
class QueueListener
|
3
|
+
class << self
|
4
|
+
def list
|
5
|
+
Hash[Eventboss::Listener::ACTIVE_LISTENERS.map do |src_app_event, listener|
|
6
|
+
[Eventboss::Queue.new("#{Eventboss.configuration.eventboss_app_name}#{Eventboss.configuration.sns_sqs_name_infix}#{src_app_event}-#{Eventboss.env}"), listener]
|
7
|
+
end]
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module Eventboss
|
2
|
+
class Runner
|
3
|
+
class << self
|
4
|
+
def launch
|
5
|
+
queues = Eventboss::QueueListener.list
|
6
|
+
client = Eventboss.configuration.sqs_client
|
7
|
+
config = Eventboss.configuration
|
8
|
+
|
9
|
+
Eventboss::Instrumentation.add(queues)
|
10
|
+
|
11
|
+
launcher = Launcher.new(queues, client, worker_count: config.concurrency)
|
12
|
+
|
13
|
+
self_read, _self_write = IO.pipe
|
14
|
+
begin
|
15
|
+
launcher.start
|
16
|
+
while (_readable_io = IO.select([self_read]))
|
17
|
+
# handle_signal(readable_io.first[0].gets.strip)
|
18
|
+
end
|
19
|
+
rescue Interrupt
|
20
|
+
launcher.stop
|
21
|
+
exit 0
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def start
|
26
|
+
configuration = Eventboss.configuration
|
27
|
+
|
28
|
+
queue_listeners = Eventboss::QueueListener.list
|
29
|
+
Eventboss::Instrumentation.add(queue_listeners)
|
30
|
+
polling_strategy = configuration.polling_strategy.call(queue_listeners.keys)
|
31
|
+
|
32
|
+
fetcher = Eventboss::Fetcher.new(configuration)
|
33
|
+
executor = Concurrent.global_io_executor
|
34
|
+
|
35
|
+
manager = Eventboss::Manager.new(
|
36
|
+
fetcher,
|
37
|
+
polling_strategy,
|
38
|
+
executor,
|
39
|
+
queue_listeners,
|
40
|
+
configuration.concurrency,
|
41
|
+
configuration.error_handlers
|
42
|
+
)
|
43
|
+
|
44
|
+
manager.start
|
45
|
+
|
46
|
+
self_read, self_write = IO.pipe
|
47
|
+
begin
|
48
|
+
while (readable_io = IO.select([self_read]))
|
49
|
+
signal = readable_io.first[0].gets.strip
|
50
|
+
# handle_signal(signal)
|
51
|
+
end
|
52
|
+
rescue Interrupt
|
53
|
+
executor.shutdown
|
54
|
+
executor.wait_for_termination
|
55
|
+
exit 0
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Eventboss
|
2
|
+
# SafeThread includes thread handling with automatic error reporting
|
3
|
+
module SafeThread
|
4
|
+
def safe_thread(name)
|
5
|
+
Thread.new do
|
6
|
+
begin
|
7
|
+
Thread.current[:ah_eventboss_label] = name
|
8
|
+
yield
|
9
|
+
rescue Exception => exception
|
10
|
+
handle_exception(exception, name: name)
|
11
|
+
raise exception
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def handle_exception(exception, context)
|
17
|
+
context.freeze
|
18
|
+
Eventboss.configuration.error_handlers.each do |handler|
|
19
|
+
handler.call(exception, context)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
load 'tasks/eventboss.rake'
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Eventboss
|
2
|
+
class Sender
|
3
|
+
def initialize(client:, queue:)
|
4
|
+
@client = client
|
5
|
+
@queue = queue
|
6
|
+
end
|
7
|
+
|
8
|
+
def send_batch(payload)
|
9
|
+
client.send_message_batch(
|
10
|
+
queue_url: queue.url,
|
11
|
+
entries: Array(build_entries(payload))
|
12
|
+
)
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
attr_reader :queue, :client
|
18
|
+
|
19
|
+
def build_entries(messages)
|
20
|
+
messages.map do |message|
|
21
|
+
{ id: SecureRandom.hex, message_body: message.to_json }
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module Eventboss
|
2
|
+
class NotConfigured < StandardError; end
|
3
|
+
|
4
|
+
class SnsClient
|
5
|
+
def initialize(configuration)
|
6
|
+
@configuration = configuration
|
7
|
+
end
|
8
|
+
|
9
|
+
def publish(payload)
|
10
|
+
backend.publish(payload)
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
attr_reader :configuration
|
16
|
+
|
17
|
+
def backend
|
18
|
+
if configured?
|
19
|
+
options = {
|
20
|
+
region: configuration.eventboss_region,
|
21
|
+
credentials: ::Aws::Credentials.new(
|
22
|
+
configuration.aws_access_key_id,
|
23
|
+
configuration.aws_secret_access_key
|
24
|
+
)
|
25
|
+
}
|
26
|
+
if configuration.aws_sns_endpoint
|
27
|
+
options[:endpoint] = configuration.aws_sns_endpoint
|
28
|
+
end
|
29
|
+
Aws::SNS::Client.new(
|
30
|
+
options
|
31
|
+
)
|
32
|
+
elsif configuration.raise_on_missing_configuration
|
33
|
+
raise NotConfigured, 'Eventboss is not configured'
|
34
|
+
else
|
35
|
+
Mock.new
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def configured?
|
40
|
+
!!(
|
41
|
+
configuration.eventboss_region &&
|
42
|
+
configuration.eventboss_account_id &&
|
43
|
+
configuration.eventboss_app_name
|
44
|
+
)
|
45
|
+
end
|
46
|
+
|
47
|
+
class Mock
|
48
|
+
def publish(_)
|
49
|
+
Eventboss::Logger.info('Eventboss is not configured. Skipping message publishing!')
|
50
|
+
return
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Eventboss
|
2
|
+
# UnitOfWork handles calls a listener for each message and deletes on success
|
3
|
+
class UnitOfWork
|
4
|
+
include Logging
|
5
|
+
include SafeThread
|
6
|
+
|
7
|
+
attr_accessor :queue, :listener, :message
|
8
|
+
|
9
|
+
def initialize(queue, listener, message)
|
10
|
+
@queue = queue
|
11
|
+
@listener = listener
|
12
|
+
@message = message
|
13
|
+
end
|
14
|
+
|
15
|
+
def run(client)
|
16
|
+
logger.debug('Started', @message.message_id)
|
17
|
+
processor = @listener.new
|
18
|
+
processor.receive(JSON.parse(@message.body))
|
19
|
+
logger.info('Finished', @message.message_id)
|
20
|
+
rescue StandardError => exception
|
21
|
+
handle_exception(exception, processor: processor, message_id: @message.message_id)
|
22
|
+
else
|
23
|
+
cleanup(client)
|
24
|
+
end
|
25
|
+
|
26
|
+
def cleanup(client)
|
27
|
+
client.delete_message(
|
28
|
+
queue_url: @queue.url, receipt_handle: @message.receipt_handle
|
29
|
+
)
|
30
|
+
logger.debug('Deleting', @message.message_id)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module Eventboss
|
2
|
+
# Worker is part of a pool of workers, handles UnitOfWork lifecycle
|
3
|
+
class Worker
|
4
|
+
include Logging
|
5
|
+
include SafeThread
|
6
|
+
|
7
|
+
attr_reader :id
|
8
|
+
|
9
|
+
def initialize(launcher, id, client, bus)
|
10
|
+
@id = "worker-#{id}"
|
11
|
+
@launcher = launcher
|
12
|
+
@client = client
|
13
|
+
@bus = bus
|
14
|
+
@thread = nil
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
@thread = safe_thread(id, &method(:run))
|
19
|
+
end
|
20
|
+
|
21
|
+
def run
|
22
|
+
while (work = @bus.pop)
|
23
|
+
work.run(@client)
|
24
|
+
end
|
25
|
+
@launcher.worker_stopped(self)
|
26
|
+
rescue Eventboss::Shutdown
|
27
|
+
@launcher.worker_stopped(self)
|
28
|
+
rescue Exception => exception
|
29
|
+
handle_exception(exception, worker_id: id)
|
30
|
+
# Restart the worker in case of hard exception
|
31
|
+
# Message won't be delete from SQS and will be visible later
|
32
|
+
@launcher.worker_stopped(self, restart: true)
|
33
|
+
end
|
34
|
+
|
35
|
+
def terminate(wait = false)
|
36
|
+
stop_token
|
37
|
+
return unless @thread
|
38
|
+
@thread.value if wait
|
39
|
+
end
|
40
|
+
|
41
|
+
def kill(wait = false)
|
42
|
+
stop_token
|
43
|
+
return unless @thread
|
44
|
+
@thread.raise Eventboss::Shutdown
|
45
|
+
@thread.value if wait
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
# stops the loop, by enqueuing falsey value
|
51
|
+
def stop_token
|
52
|
+
@bus << nil
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|