osbourne 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +674 -0
- data/README.md +106 -0
- data/Rakefile +30 -0
- data/bin/cli/base.rb +41 -0
- data/bin/osbourne +33 -0
- data/lib/generators/osbourne/install/install_generator.rb +14 -0
- data/lib/generators/osbourne/install/templates/osbourne_initializer_template.template +30 -0
- data/lib/generators/osbourne/install/templates/osbourne_yaml_template.template +15 -0
- data/lib/generators/osbourne/worker/USAGE +11 -0
- data/lib/generators/osbourne/worker/templates/worker_template.template +35 -0
- data/lib/generators/osbourne/worker/worker_generator.rb +20 -0
- data/lib/osbourne.rb +43 -0
- data/lib/osbourne/config/file_loader.rb +22 -0
- data/lib/osbourne/config/shared_configs.rb +37 -0
- data/lib/osbourne/existing_subscriptions.rb +40 -0
- data/lib/osbourne/launcher.rb +60 -0
- data/lib/osbourne/locks/base.rb +69 -0
- data/lib/osbourne/locks/memory.rb +69 -0
- data/lib/osbourne/locks/noop.rb +25 -0
- data/lib/osbourne/locks/redis.rb +56 -0
- data/lib/osbourne/message.rb +55 -0
- data/lib/osbourne/poller.rb +0 -0
- data/lib/osbourne/queue.rb +43 -0
- data/lib/osbourne/railtie.rb +20 -0
- data/lib/osbourne/runner.rb +86 -0
- data/lib/osbourne/services/queue_provisioner.rb +14 -0
- data/lib/osbourne/services/sns.rb +17 -0
- data/lib/osbourne/services/sqs.rb +17 -0
- data/lib/osbourne/subscription.rb +36 -0
- data/lib/osbourne/topic.rb +34 -0
- data/lib/osbourne/version.rb +5 -0
- data/lib/osbourne/worker_base.rb +91 -0
- data/lib/tasks/message_plumber_tasks.rake +6 -0
- metadata +348 -0
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Osbourne
|
4
|
+
module ExistingSubscriptions
|
5
|
+
attr_reader :existing_subscriptions
|
6
|
+
def existing_subscriptions_for(topic)
|
7
|
+
Osbourne.cache.fetch("osbourne_existng_subs_for_#{topic.name}") do
|
8
|
+
results = []
|
9
|
+
handled = Osbourne.lock.try_with_lock("osbourne_lock_subs_for_#{topic.name}") do
|
10
|
+
results = fetch_existing_subscriptions_for(topic)
|
11
|
+
end
|
12
|
+
return results if handled
|
13
|
+
|
14
|
+
sleep(0.5)
|
15
|
+
existing_subscriptions_for(topic)
|
16
|
+
end
|
17
|
+
# @existing_subscriptions ||= {}
|
18
|
+
# @existing_subscriptions[topic.name] ||=
|
19
|
+
end
|
20
|
+
|
21
|
+
def clear_subscriptions_for(topic)
|
22
|
+
Osbourne.cache.delete("osbourne_existng_subs_for_#{topic.name}")
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def fetch_existing_subscriptions_for(topic)
|
28
|
+
results = []
|
29
|
+
r = nil
|
30
|
+
loop do
|
31
|
+
params = {topic_arn: topic.arn}
|
32
|
+
params[:next_token] = r.next_token if r.try(:next_token)
|
33
|
+
r = Osbourne.sns_client.list_subscriptions_by_topic(params)
|
34
|
+
results << r.subscriptions.map(&:endpoint)
|
35
|
+
break unless r.try(:next_token).presence
|
36
|
+
end
|
37
|
+
results.flatten.uniq
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "osbourne"
|
4
|
+
require "byebug"
|
5
|
+
|
6
|
+
module Osbourne
|
7
|
+
class Launcher
|
8
|
+
attr_accessor :threads
|
9
|
+
def initialize; end
|
10
|
+
|
11
|
+
def start!
|
12
|
+
Osbourne.logger.info("Launching Osbourne workers")
|
13
|
+
@stop = false
|
14
|
+
@threads = polling_threads
|
15
|
+
polling_threads.each(&:run)
|
16
|
+
end
|
17
|
+
|
18
|
+
def stop
|
19
|
+
@stop = true
|
20
|
+
end
|
21
|
+
|
22
|
+
def stop!
|
23
|
+
@threads.each {|thr| Thread.kill(thr) }
|
24
|
+
end
|
25
|
+
|
26
|
+
def polling_threads
|
27
|
+
Osbourne::WorkerBase.descendants.map do |worker|
|
28
|
+
Thread.new {
|
29
|
+
worker_instance = worker.new
|
30
|
+
loop do
|
31
|
+
poll(worker_instance)
|
32
|
+
sleep(Osbourne.sleep_time)
|
33
|
+
break if @stop
|
34
|
+
end
|
35
|
+
}
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def poll(worker)
|
40
|
+
worker.polling_queue.receive_messages(
|
41
|
+
wait_time_seconds: worker.config[:max_wait],
|
42
|
+
max_number_of_messages: worker.config[:max_batch_size]
|
43
|
+
).each {|msg| process(worker, Osbourne::Message.new(msg)) }
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def process(worker, message)
|
49
|
+
Osbourne.logger.info("[MSG] Worker: #{worker.class.name} Valid: #{message.valid?} ID: #{message.id} Body: #{message.raw_body}") # rubocop:disable Metrics/LineLength
|
50
|
+
return unless message.valid?
|
51
|
+
|
52
|
+
# hard_lock to prevent duplicate processing over the hard_lock lifespan
|
53
|
+
Osbourne.lock.try_with_lock(message.id, hard_lock: true) do
|
54
|
+
message.delete if worker.process(message)
|
55
|
+
end
|
56
|
+
rescue Exception => ex # rubocop:disable Lint/RescueException
|
57
|
+
Osbourne.logger.error("[MSG ID: #{message.id}] #{ex.message}")
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "osbourne"
|
4
|
+
|
5
|
+
module Osbourne
|
6
|
+
module Locks
|
7
|
+
module Base
|
8
|
+
DEFAULT_SOFT_TTL = (5 * 60).freeze # 5 minutes
|
9
|
+
DEFAULT_HARD_TTL = (24 * 60 * 60).freeze # 24 hours
|
10
|
+
|
11
|
+
attr_reader :soft_ttl, :hard_ttl
|
12
|
+
|
13
|
+
def initialize(options={})
|
14
|
+
self.soft_ttl = options.fetch(:soft_ttl, DEFAULT_SOFT_TTL)
|
15
|
+
self.hard_ttl = options.fetch(:hard_ttl, DEFAULT_HARD_TTL)
|
16
|
+
end
|
17
|
+
|
18
|
+
def soft_lock(id)
|
19
|
+
lock(lock_key(id), soft_ttl)
|
20
|
+
end
|
21
|
+
|
22
|
+
def hard_lock(id)
|
23
|
+
lock!(lock_key(id), hard_ttl)
|
24
|
+
end
|
25
|
+
|
26
|
+
def unlock(id)
|
27
|
+
unlock!(lock_key(id))
|
28
|
+
end
|
29
|
+
|
30
|
+
def try_with_lock(id, hard_lock: false)
|
31
|
+
if soft_lock(id)
|
32
|
+
begin
|
33
|
+
yield
|
34
|
+
rescue => e # rubocop:disable Style/RescueStandardError
|
35
|
+
unlock(id)
|
36
|
+
raise e
|
37
|
+
end
|
38
|
+
|
39
|
+
hard_lock(id) if hard_lock
|
40
|
+
true
|
41
|
+
else
|
42
|
+
false
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
protected
|
47
|
+
|
48
|
+
def lock(_key, _ttl)
|
49
|
+
raise NotImplementedError
|
50
|
+
end
|
51
|
+
|
52
|
+
def lock!(_key, _ttl)
|
53
|
+
raise NotImplementedError
|
54
|
+
end
|
55
|
+
|
56
|
+
def unlock!(_key)
|
57
|
+
raise NotImplementedError
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
attr_writer :soft_ttl, :hard_ttl
|
63
|
+
|
64
|
+
def lock_key(id)
|
65
|
+
"osbourne:lock:#{id}"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "osbourne"
|
4
|
+
|
5
|
+
module Osbourne
|
6
|
+
module Locks
|
7
|
+
class Memory
|
8
|
+
include Base
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def store
|
12
|
+
@store ||= {}
|
13
|
+
end
|
14
|
+
|
15
|
+
def semaphore
|
16
|
+
@semaphore ||= Mutex.new
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
protected
|
21
|
+
|
22
|
+
def lock(key, ttl)
|
23
|
+
reap
|
24
|
+
|
25
|
+
store do |store|
|
26
|
+
if store.key?(key)
|
27
|
+
false
|
28
|
+
else
|
29
|
+
store[key] = Time.current + ttl
|
30
|
+
true
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def lock!(key, ttl)
|
36
|
+
reap
|
37
|
+
|
38
|
+
store do |store|
|
39
|
+
store[key] = Time.current + ttl
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def unlock!(key)
|
44
|
+
store do |store|
|
45
|
+
store.delete(key)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def store
|
52
|
+
semaphore.synchronize do
|
53
|
+
yield self.class.store
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def reap
|
58
|
+
store do |store|
|
59
|
+
now = Time.current
|
60
|
+
store.delete_if {|_, expires_at| expires_at <= now }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def semaphore
|
65
|
+
self.class.semaphore
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "osbourne"
|
4
|
+
|
5
|
+
module Osbourne
|
6
|
+
module Locks
|
7
|
+
class NOOP
|
8
|
+
include Base
|
9
|
+
|
10
|
+
protected
|
11
|
+
|
12
|
+
def lock(_key, _ttl)
|
13
|
+
true
|
14
|
+
end
|
15
|
+
|
16
|
+
def lock!(key, ttl)
|
17
|
+
# do nothing
|
18
|
+
end
|
19
|
+
|
20
|
+
def unlock!(key)
|
21
|
+
# do nothing
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "osbourne"
|
4
|
+
|
5
|
+
module Osbourne
|
6
|
+
module Locks
|
7
|
+
class Redis
|
8
|
+
include Base
|
9
|
+
|
10
|
+
def initialize(options={})
|
11
|
+
super(options)
|
12
|
+
|
13
|
+
self.client = options.fetch(:client) do
|
14
|
+
require "redis"
|
15
|
+
::Redis.new(options)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
protected
|
20
|
+
|
21
|
+
def lock(key, ttl)
|
22
|
+
with_pool do |client|
|
23
|
+
client.set(key, (Time.current + ttl).to_i, ex: ttl, nx: true)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def lock!(key, ttl)
|
28
|
+
with_pool do |client|
|
29
|
+
client.set(key, (Time.current + ttl).to_i, ex: ttl)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def unlock!(key)
|
34
|
+
with_pool do |client|
|
35
|
+
client.del(key)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
attr_accessor :client
|
42
|
+
|
43
|
+
def with_pool(&block)
|
44
|
+
if pool?
|
45
|
+
client.with(&block)
|
46
|
+
else
|
47
|
+
yield client
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def pool?
|
52
|
+
defined?(ConnectionPool) && client.is_a?(ConnectionPool)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "osbourne"
|
4
|
+
|
5
|
+
module Osbourne
|
6
|
+
class Message
|
7
|
+
attr_accessor :message
|
8
|
+
def initialize(message)
|
9
|
+
@message = message
|
10
|
+
end
|
11
|
+
|
12
|
+
def json?
|
13
|
+
!JSON.parse(parsed_content["Message"]).nil?
|
14
|
+
rescue JSON::ParserError
|
15
|
+
false
|
16
|
+
end
|
17
|
+
|
18
|
+
def valid?
|
19
|
+
message.md5_of_body == Digest::MD5.hexdigest(message.body)
|
20
|
+
end
|
21
|
+
|
22
|
+
def id
|
23
|
+
message.message_id
|
24
|
+
end
|
25
|
+
|
26
|
+
def parsed_body
|
27
|
+
JSON.parse(parsed_content["Message"])
|
28
|
+
rescue JSON::ParserError
|
29
|
+
parsed_content["Message"]
|
30
|
+
end
|
31
|
+
|
32
|
+
def delete
|
33
|
+
message.delete
|
34
|
+
Osbourne.logger.info "[MSG ID: #{id}] Cleared"
|
35
|
+
end
|
36
|
+
|
37
|
+
def topic
|
38
|
+
parsed_content["TopicArn"].split(":").last
|
39
|
+
end
|
40
|
+
|
41
|
+
def raw_body
|
42
|
+
message.body
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def parsed_content
|
48
|
+
@parsed_content ||= JSON.parse(message.body)
|
49
|
+
end
|
50
|
+
|
51
|
+
def body
|
52
|
+
parsed_content["Message"]
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
File without changes
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Osbourne
|
4
|
+
class Queue
|
5
|
+
include Services::SQS
|
6
|
+
attr_reader :name
|
7
|
+
def initialize(name)
|
8
|
+
@name = name
|
9
|
+
arn
|
10
|
+
end
|
11
|
+
|
12
|
+
def url
|
13
|
+
@url ||= ensure_queue
|
14
|
+
end
|
15
|
+
|
16
|
+
def arn
|
17
|
+
@arn ||= get_attributes["QueueArn"]
|
18
|
+
end
|
19
|
+
|
20
|
+
def redrive(retries, dead_letter_arn)
|
21
|
+
sqs.set_queue_attributes(queue_url: url,
|
22
|
+
attributes: {
|
23
|
+
'RedrivePolicy': {
|
24
|
+
'deadLetterTargetArn': dead_letter_arn,
|
25
|
+
'maxReceiveCount': retries
|
26
|
+
}.to_json
|
27
|
+
})
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def ensure_queue
|
33
|
+
Osbourne.logger.debug "Ensuring queue `#{name}` exists"
|
34
|
+
Osbourne.cache.fetch("osbourne_url_for_#{name}") do
|
35
|
+
sqs.create_queue(queue_name: name).queue_url
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def get_attributes(attrs: %w[QueueArn])
|
40
|
+
sqs.get_queue_attributes(queue_url: url, attribute_names: attrs).attributes
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails"
|
4
|
+
require "osbourne/config/file_loader"
|
5
|
+
|
6
|
+
module Osbourne
|
7
|
+
class Railtie < Rails::Railtie
|
8
|
+
config.osbourne = ActiveSupport::OrderedOptions.new
|
9
|
+
|
10
|
+
initializer "osbourne.configure", after: :load_config_initializers do |_app|
|
11
|
+
env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
12
|
+
|
13
|
+
%w[config/osbourne.yml.erb config/osbourne.yml].find do |filename|
|
14
|
+
Osbourne::Config::FileLoader.load(filename, env)
|
15
|
+
end
|
16
|
+
|
17
|
+
Osbourne.provision_worker_queues
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|