osbourne 1.1.5 → 1.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/osbourne/launcher.rb +6 -4
- data/lib/osbourne/message.rb +1 -1
- data/lib/osbourne/queue.rb +5 -4
- data/lib/osbourne/runner.rb +6 -6
- data/lib/osbourne/services/queue_provisioner.rb +2 -2
- data/lib/osbourne/subscription.rb +5 -5
- data/lib/osbourne/test/mock_pubsub.rb +3 -3
- data/lib/osbourne/topic.rb +2 -2
- data/lib/osbourne/version.rb +1 -1
- data/lib/osbourne/worker_base.rb +4 -4
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0601b6c1105b1ee73019fd9d0776cc1dce5cf02370c84120780be45344c17c68
|
4
|
+
data.tar.gz: 771b77ff9768e57d92cce22ca7b9aa79aaf60a4772b8cc009efb04d4593f28ca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ab8b46e9adbb97518e776a16cd488997a446ab1ce9b3e6504bf3a24e5d8a5d51a0e4d036c1d0f14ae366c4389b78cc0ce37a01ce7166411d92d1ad91ef94c03f
|
7
|
+
data.tar.gz: 158fa1e740d28ac8a14ca3a506a7ef6da7d6e3b4d877baf378ca06b0b869fe2aec40ddd9c8e5080bf05c79b4f63d54708bf2bc3aaeb5990531c089c27b576f16
|
data/lib/osbourne/launcher.rb
CHANGED
@@ -8,7 +8,7 @@ module Osbourne
|
|
8
8
|
def initialize; end
|
9
9
|
|
10
10
|
def start!
|
11
|
-
Osbourne.logger.info("Launching Osbourne workers")
|
11
|
+
Osbourne.logger.info("[Osbourne] Launching Osbourne workers")
|
12
12
|
@stop = false
|
13
13
|
@threads = global_polling_threads
|
14
14
|
end
|
@@ -29,7 +29,7 @@ module Osbourne
|
|
29
29
|
|
30
30
|
def global_polling_threads
|
31
31
|
Osbourne::WorkerBase.descendants.map do |worker|
|
32
|
-
Osbourne.logger.debug("Spawning thread for #{worker.name}")
|
32
|
+
Osbourne.logger.debug("[Osbourne] Spawning thread for #{worker.name}")
|
33
33
|
Thread.new { poll(worker) }
|
34
34
|
end
|
35
35
|
end
|
@@ -46,9 +46,11 @@ module Osbourne
|
|
46
46
|
worker.polling_queue.poll(wait_time_seconds: worker.config[:max_wait_time],
|
47
47
|
max_number_of_messages: worker.config[:max_batch_size],
|
48
48
|
skip_delete: true) do |messages|
|
49
|
+
Osbourne.logger.debug("[Osbourne] Recieved #{messages.count} on #{worker.name}")
|
49
50
|
messages.map do |msg|
|
50
51
|
worker.polling_queue.delete_message(msg) if process(worker, Osbourne::Message.new(msg))
|
51
52
|
end
|
53
|
+
Osbourne.logger.debug("[Osbourne] Waiting for more messages on #{worker.name} for max of #{worker.config[:max_wait_time]} seconds")
|
52
54
|
throw :stop_polling if @stop
|
53
55
|
end
|
54
56
|
end
|
@@ -56,14 +58,14 @@ module Osbourne
|
|
56
58
|
private
|
57
59
|
|
58
60
|
def process(worker, message)
|
59
|
-
Osbourne.logger.info("[MSG] Worker: #{worker.name} Valid: #{message.valid?} ID: #{message.id}")
|
61
|
+
Osbourne.logger.info("[Osbourne] [MSG] Worker: #{worker.name} Valid: #{message.valid?} ID: #{message.id}")
|
60
62
|
return false unless message.valid? && Osbourne.lock.soft_lock(message.id)
|
61
63
|
|
62
64
|
Osbourne.cache.fetch(message.id, ex: 24.hours) do
|
63
65
|
worker.new.process(message).tap {|_| Osbourne.lock.unlock(message.id) }
|
64
66
|
end
|
65
67
|
rescue Exception => ex # rubocop:disable Lint/RescueException
|
66
|
-
Osbourne.logger.error("[MSG ID: #{message.id}] [#{ex.message}]\n #{ex.backtrace_locations.join("\n")}")
|
68
|
+
Osbourne.logger.error("[Osbourne] [MSG ID: #{message.id}] [#{ex.message}]\n #{ex.backtrace_locations.join("\n")}")
|
67
69
|
false
|
68
70
|
end
|
69
71
|
end
|
data/lib/osbourne/message.rb
CHANGED
data/lib/osbourne/queue.rb
CHANGED
@@ -3,9 +3,10 @@
|
|
3
3
|
module Osbourne
|
4
4
|
class Queue
|
5
5
|
include Services::SQS
|
6
|
-
attr_reader :name
|
6
|
+
attr_reader :name, :prefixed_name
|
7
7
|
def initialize(name)
|
8
8
|
@name = name
|
9
|
+
@prefixed_name = Osbourne.prefixer(@name)
|
9
10
|
arn
|
10
11
|
end
|
11
12
|
|
@@ -30,9 +31,9 @@ module Osbourne
|
|
30
31
|
private
|
31
32
|
|
32
33
|
def ensure_queue
|
33
|
-
Osbourne.logger.debug "Ensuring queue `#{
|
34
|
-
Osbourne.cache.fetch("osbourne_url_for_#{
|
35
|
-
sqs.create_queue(queue_name:
|
34
|
+
Osbourne.logger.debug "[Osbourne] Ensuring queue `#{@prefixed_name}` exists"
|
35
|
+
Osbourne.cache.fetch("osbourne_url_for_#{@prefixed_name}") do
|
36
|
+
sqs.create_queue(queue_name: @prefixed_name).queue_url
|
36
37
|
end
|
37
38
|
end
|
38
39
|
|
data/lib/osbourne/runner.rb
CHANGED
@@ -47,38 +47,38 @@ module Osbourne
|
|
47
47
|
private
|
48
48
|
|
49
49
|
def execute_soft_shutdown
|
50
|
-
Osbourne.logger.info { "Received USR1, will soft shutdown down" }
|
50
|
+
Osbourne.logger.info { "[Osbourne] Received USR1, will soft shutdown down" }
|
51
51
|
|
52
52
|
@launcher.stop
|
53
53
|
exit 0
|
54
54
|
end
|
55
55
|
|
56
56
|
def execute_terminal_stop
|
57
|
-
Osbourne.logger.info { "Received TSTP, will stop accepting new work" }
|
57
|
+
Osbourne.logger.info { "[Osbourne] Received TSTP, will stop accepting new work" }
|
58
58
|
|
59
59
|
@launcher.stop!
|
60
60
|
end
|
61
61
|
|
62
62
|
def print_threads_backtrace
|
63
63
|
Thread.list.each do |thread|
|
64
|
-
Osbourne.logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
|
64
|
+
Osbourne.logger.info { "[Osbourne] Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
|
65
65
|
if thread.backtrace
|
66
66
|
Osbourne.logger.info { thread.backtrace.join("\n") }
|
67
67
|
else
|
68
|
-
Osbourne.logger.info { "<no backtrace available>" }
|
68
|
+
Osbourne.logger.info { "[Osbourne] <no backtrace available>" }
|
69
69
|
end
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
73
|
def handle_signal(sig)
|
74
|
-
Osbourne.logger.debug "Got #{sig} signal"
|
74
|
+
Osbourne.logger.debug "[Osbourne] Got #{sig} signal"
|
75
75
|
|
76
76
|
case sig
|
77
77
|
when "USR1" then execute_soft_shutdown
|
78
78
|
when "TTIN" then print_threads_backtrace
|
79
79
|
when "TSTP" then execute_terminal_stop
|
80
80
|
when "TERM", "INT"
|
81
|
-
Osbourne.logger.info { "Received #{sig}, will shutdown" }
|
81
|
+
Osbourne.logger.info { "[Osbourne] Received #{sig}, will shutdown" }
|
82
82
|
|
83
83
|
raise Interrupt
|
84
84
|
end
|
@@ -7,8 +7,8 @@ module Osbourne
|
|
7
7
|
Dir[File.expand_path("app/workers/**/*.rb")].each {|f| require f }
|
8
8
|
return if Osbourne.test_mode?
|
9
9
|
|
10
|
-
Osbourne.logger.info "Workers found: #{Osbourne::WorkerBase.descendants.map(&:name).join(', ')}"
|
11
|
-
Osbourne.logger.info "Provisioning queues for all workers"
|
10
|
+
Osbourne.logger.info "[Osbourne] Workers found: #{Osbourne::WorkerBase.descendants.map(&:name).join(', ')}"
|
11
|
+
Osbourne.logger.info "[Osbourne] Provisioning queues for all workers"
|
12
12
|
Osbourne::WorkerBase.descendants.each(&:provision)
|
13
13
|
end
|
14
14
|
end
|
@@ -19,16 +19,16 @@ module Osbourne
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def subscribe(topic)
|
22
|
-
Osbourne.logger.info("Checking subscription for #{queue.
|
22
|
+
Osbourne.logger.info("[Osbourne] Checking subscription for #{queue.prefixed_name} to #{topic.prefixed_name}")
|
23
23
|
return if Osbourne.existing_subscriptions_for(topic).include? queue.arn
|
24
24
|
|
25
|
-
Osbourne.logger.info("Subscribing #{queue.
|
25
|
+
Osbourne.logger.info("[Osbourne] Subscribing #{queue.prefixed_name} to #{topic.prefixed_name}")
|
26
26
|
sns.subscribe(topic_arn: topic.arn, protocol: "sqs", endpoint: queue.arn).subscription_arn
|
27
27
|
Osbourne.clear_subscriptions_for(topic)
|
28
28
|
end
|
29
29
|
|
30
30
|
def set_queue_policy
|
31
|
-
Osbourne.logger.info("Setting policy for #{queue.
|
31
|
+
Osbourne.logger.info("[Osbourne] Setting policy for #{queue.prefixed_name} (attributes: #{build_policy})")
|
32
32
|
sqs.set_queue_attributes(queue_url: queue.url, attributes: build_policy)
|
33
33
|
end
|
34
34
|
|
@@ -37,7 +37,7 @@ module Osbourne
|
|
37
37
|
{
|
38
38
|
"Policy" => {
|
39
39
|
"Version" => "2012-10-17",
|
40
|
-
"Id" => "Osbourne/#{queue.
|
40
|
+
"Id" => "Osbourne/#{queue.prefixed_name}/SNSPolicy",
|
41
41
|
"Statement" => topics.map {|t| build_policy_statement(t) }
|
42
42
|
}.to_json
|
43
43
|
}
|
@@ -45,7 +45,7 @@ module Osbourne
|
|
45
45
|
|
46
46
|
def build_policy_statement(topic)
|
47
47
|
{
|
48
|
-
"Sid" => "Sid#{topic.
|
48
|
+
"Sid" => "Sid#{topic.prefixed_name}",
|
49
49
|
"Effect" => "Allow",
|
50
50
|
"Principal" => {"AWS" => "*"},
|
51
51
|
"Action" => "SQS:SendMessage",
|
@@ -6,11 +6,11 @@ module Osbourne
|
|
6
6
|
class MockPubsub
|
7
7
|
class << self
|
8
8
|
def mock_publish(topic, message)
|
9
|
-
prefixed_topic = Osbourne.prefixer(topic)
|
9
|
+
# prefixed_topic = Osbourne.prefixer(topic)
|
10
10
|
parsed_message = parse(message)
|
11
11
|
Osbourne::WorkerBase.descendants.each do |worker|
|
12
|
-
msg = Osbourne::Test::Message.new(topic:
|
13
|
-
worker.new.process(msg) if worker.config[:topic_names].include?
|
12
|
+
msg = Osbourne::Test::Message.new(topic: topic, body: parsed_message)
|
13
|
+
worker.new.process(msg) if worker.config[:topic_names].include? topic
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
data/lib/osbourne/topic.rb
CHANGED
@@ -19,7 +19,7 @@ module Osbourne
|
|
19
19
|
parsed_message = parse(message)
|
20
20
|
return if Osbourne.test_mode?
|
21
21
|
|
22
|
-
Osbourne.logger.info "[PUB] TOPIC: `#{prefixed_name}` MESSAGE: `#{parsed_message}`"
|
22
|
+
Osbourne.logger.info "[Osbourne] [PUB] TOPIC: `#{prefixed_name}` MESSAGE: `#{parsed_message}`"
|
23
23
|
sns.publish(topic_arn: arn, message: parsed_message)
|
24
24
|
end
|
25
25
|
|
@@ -28,7 +28,7 @@ module Osbourne
|
|
28
28
|
def ensure_topic
|
29
29
|
return if Osbourne.test_mode?
|
30
30
|
|
31
|
-
Osbourne.logger.debug "Ensuring topic `#{prefixed_name}` exists"
|
31
|
+
Osbourne.logger.debug "[Osbourne] Ensuring topic `#{prefixed_name}` exists"
|
32
32
|
Osbourne.cache.fetch("osbourne_existing_topic_arn_for_#{prefixed_name}", ex: 1.minute) do
|
33
33
|
sns.create_topic(name: prefixed_name).topic_arn
|
34
34
|
end
|
data/lib/osbourne/version.rb
CHANGED
data/lib/osbourne/worker_base.rb
CHANGED
@@ -53,12 +53,12 @@ module Osbourne
|
|
53
53
|
def register_dead_letter_queue
|
54
54
|
return unless config[:dead_letter]
|
55
55
|
|
56
|
-
Osbourne.logger.info "#{self.class.name} dead letter queue: arn: [#{dead_letter_queue.arn}], max retries: #{config[:max_retry_count]}"
|
56
|
+
Osbourne.logger.info "[Osbourne] #{self.class.name} dead letter queue: arn: [#{dead_letter_queue.arn}], max retries: #{config[:max_retry_count]}"
|
57
57
|
queue.redrive(config[:max_retry_count], dead_letter_queue.arn)
|
58
58
|
end
|
59
59
|
|
60
60
|
def register
|
61
|
-
Osbourne.logger.info "#{self.class.name} subscriptions: Topics: [#{config[:topic_names].join(', ')}], Queue: [#{config[:queue_name]}]"
|
61
|
+
Osbourne.logger.info "[Osbourne] #{self.class.name} subscriptions: Topics: [#{config[:topic_names].join(', ')}], Queue: [#{config[:queue_name]}]"
|
62
62
|
self.topics = config[:topic_names].map {|tn| Topic.new(tn) }
|
63
63
|
self.queue = Queue.new(config[:queue_name])
|
64
64
|
self.subscriptions = Subscription.new(topics, queue)
|
@@ -81,8 +81,8 @@ module Osbourne
|
|
81
81
|
dead_letter_queue: true,
|
82
82
|
max_retry_count: Osbourne.max_retry_count)
|
83
83
|
self.config = {
|
84
|
-
topic_names: Array(topics)
|
85
|
-
queue_name:
|
84
|
+
topic_names: Array(topics),
|
85
|
+
queue_name: queue_name,
|
86
86
|
max_batch_size: max_batch_size,
|
87
87
|
max_wait: max_wait,
|
88
88
|
threads: threads,
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: osbourne
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Steve Allen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-core
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '3'
|
19
|
+
version: '3.37'
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '4'
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: '3'
|
29
|
+
version: '3.37'
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '4'
|
@@ -71,7 +71,7 @@ dependencies:
|
|
71
71
|
- !ruby/object:Gem::Version
|
72
72
|
version: '2'
|
73
73
|
- !ruby/object:Gem::Dependency
|
74
|
-
name:
|
74
|
+
name: railties
|
75
75
|
requirement: !ruby/object:Gem::Requirement
|
76
76
|
requirements:
|
77
77
|
- - ">="
|