osbourne 1.1.5 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/osbourne/launcher.rb +6 -4
- data/lib/osbourne/message.rb +1 -1
- data/lib/osbourne/queue.rb +5 -4
- data/lib/osbourne/runner.rb +6 -6
- data/lib/osbourne/services/queue_provisioner.rb +2 -2
- data/lib/osbourne/subscription.rb +5 -5
- data/lib/osbourne/test/mock_pubsub.rb +3 -3
- data/lib/osbourne/topic.rb +2 -2
- data/lib/osbourne/version.rb +1 -1
- data/lib/osbourne/worker_base.rb +4 -4
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0601b6c1105b1ee73019fd9d0776cc1dce5cf02370c84120780be45344c17c68
|
4
|
+
data.tar.gz: 771b77ff9768e57d92cce22ca7b9aa79aaf60a4772b8cc009efb04d4593f28ca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ab8b46e9adbb97518e776a16cd488997a446ab1ce9b3e6504bf3a24e5d8a5d51a0e4d036c1d0f14ae366c4389b78cc0ce37a01ce7166411d92d1ad91ef94c03f
|
7
|
+
data.tar.gz: 158fa1e740d28ac8a14ca3a506a7ef6da7d6e3b4d877baf378ca06b0b869fe2aec40ddd9c8e5080bf05c79b4f63d54708bf2bc3aaeb5990531c089c27b576f16
|
data/lib/osbourne/launcher.rb
CHANGED
@@ -8,7 +8,7 @@ module Osbourne
|
|
8
8
|
def initialize; end
|
9
9
|
|
10
10
|
def start!
|
11
|
-
Osbourne.logger.info("Launching Osbourne workers")
|
11
|
+
Osbourne.logger.info("[Osbourne] Launching Osbourne workers")
|
12
12
|
@stop = false
|
13
13
|
@threads = global_polling_threads
|
14
14
|
end
|
@@ -29,7 +29,7 @@ module Osbourne
|
|
29
29
|
|
30
30
|
def global_polling_threads
|
31
31
|
Osbourne::WorkerBase.descendants.map do |worker|
|
32
|
-
Osbourne.logger.debug("Spawning thread for #{worker.name}")
|
32
|
+
Osbourne.logger.debug("[Osbourne] Spawning thread for #{worker.name}")
|
33
33
|
Thread.new { poll(worker) }
|
34
34
|
end
|
35
35
|
end
|
@@ -46,9 +46,11 @@ module Osbourne
|
|
46
46
|
worker.polling_queue.poll(wait_time_seconds: worker.config[:max_wait_time],
|
47
47
|
max_number_of_messages: worker.config[:max_batch_size],
|
48
48
|
skip_delete: true) do |messages|
|
49
|
+
Osbourne.logger.debug("[Osbourne] Recieved #{messages.count} on #{worker.name}")
|
49
50
|
messages.map do |msg|
|
50
51
|
worker.polling_queue.delete_message(msg) if process(worker, Osbourne::Message.new(msg))
|
51
52
|
end
|
53
|
+
Osbourne.logger.debug("[Osbourne] Waiting for more messages on #{worker.name} for max of #{worker.config[:max_wait_time]} seconds")
|
52
54
|
throw :stop_polling if @stop
|
53
55
|
end
|
54
56
|
end
|
@@ -56,14 +58,14 @@ module Osbourne
|
|
56
58
|
private
|
57
59
|
|
58
60
|
def process(worker, message)
|
59
|
-
Osbourne.logger.info("[MSG] Worker: #{worker.name} Valid: #{message.valid?} ID: #{message.id}")
|
61
|
+
Osbourne.logger.info("[Osbourne] [MSG] Worker: #{worker.name} Valid: #{message.valid?} ID: #{message.id}")
|
60
62
|
return false unless message.valid? && Osbourne.lock.soft_lock(message.id)
|
61
63
|
|
62
64
|
Osbourne.cache.fetch(message.id, ex: 24.hours) do
|
63
65
|
worker.new.process(message).tap {|_| Osbourne.lock.unlock(message.id) }
|
64
66
|
end
|
65
67
|
rescue Exception => ex # rubocop:disable Lint/RescueException
|
66
|
-
Osbourne.logger.error("[MSG ID: #{message.id}] [#{ex.message}]\n #{ex.backtrace_locations.join("\n")}")
|
68
|
+
Osbourne.logger.error("[Osbourne] [MSG ID: #{message.id}] [#{ex.message}]\n #{ex.backtrace_locations.join("\n")}")
|
67
69
|
false
|
68
70
|
end
|
69
71
|
end
|
data/lib/osbourne/message.rb
CHANGED
data/lib/osbourne/queue.rb
CHANGED
@@ -3,9 +3,10 @@
|
|
3
3
|
module Osbourne
|
4
4
|
class Queue
|
5
5
|
include Services::SQS
|
6
|
-
attr_reader :name
|
6
|
+
attr_reader :name, :prefixed_name
|
7
7
|
def initialize(name)
|
8
8
|
@name = name
|
9
|
+
@prefixed_name = Osbourne.prefixer(@name)
|
9
10
|
arn
|
10
11
|
end
|
11
12
|
|
@@ -30,9 +31,9 @@ module Osbourne
|
|
30
31
|
private
|
31
32
|
|
32
33
|
def ensure_queue
|
33
|
-
Osbourne.logger.debug "Ensuring queue `#{
|
34
|
-
Osbourne.cache.fetch("osbourne_url_for_#{
|
35
|
-
sqs.create_queue(queue_name:
|
34
|
+
Osbourne.logger.debug "[Osbourne] Ensuring queue `#{@prefixed_name}` exists"
|
35
|
+
Osbourne.cache.fetch("osbourne_url_for_#{@prefixed_name}") do
|
36
|
+
sqs.create_queue(queue_name: @prefixed_name).queue_url
|
36
37
|
end
|
37
38
|
end
|
38
39
|
|
data/lib/osbourne/runner.rb
CHANGED
@@ -47,38 +47,38 @@ module Osbourne
|
|
47
47
|
private
|
48
48
|
|
49
49
|
def execute_soft_shutdown
|
50
|
-
Osbourne.logger.info { "Received USR1, will soft shutdown down" }
|
50
|
+
Osbourne.logger.info { "[Osbourne] Received USR1, will soft shutdown down" }
|
51
51
|
|
52
52
|
@launcher.stop
|
53
53
|
exit 0
|
54
54
|
end
|
55
55
|
|
56
56
|
def execute_terminal_stop
|
57
|
-
Osbourne.logger.info { "Received TSTP, will stop accepting new work" }
|
57
|
+
Osbourne.logger.info { "[Osbourne] Received TSTP, will stop accepting new work" }
|
58
58
|
|
59
59
|
@launcher.stop!
|
60
60
|
end
|
61
61
|
|
62
62
|
def print_threads_backtrace
|
63
63
|
Thread.list.each do |thread|
|
64
|
-
Osbourne.logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
|
64
|
+
Osbourne.logger.info { "[Osbourne] Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
|
65
65
|
if thread.backtrace
|
66
66
|
Osbourne.logger.info { thread.backtrace.join("\n") }
|
67
67
|
else
|
68
|
-
Osbourne.logger.info { "<no backtrace available>" }
|
68
|
+
Osbourne.logger.info { "[Osbourne] <no backtrace available>" }
|
69
69
|
end
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
73
|
def handle_signal(sig)
|
74
|
-
Osbourne.logger.debug "Got #{sig} signal"
|
74
|
+
Osbourne.logger.debug "[Osbourne] Got #{sig} signal"
|
75
75
|
|
76
76
|
case sig
|
77
77
|
when "USR1" then execute_soft_shutdown
|
78
78
|
when "TTIN" then print_threads_backtrace
|
79
79
|
when "TSTP" then execute_terminal_stop
|
80
80
|
when "TERM", "INT"
|
81
|
-
Osbourne.logger.info { "Received #{sig}, will shutdown" }
|
81
|
+
Osbourne.logger.info { "[Osbourne] Received #{sig}, will shutdown" }
|
82
82
|
|
83
83
|
raise Interrupt
|
84
84
|
end
|
@@ -7,8 +7,8 @@ module Osbourne
|
|
7
7
|
Dir[File.expand_path("app/workers/**/*.rb")].each {|f| require f }
|
8
8
|
return if Osbourne.test_mode?
|
9
9
|
|
10
|
-
Osbourne.logger.info "Workers found: #{Osbourne::WorkerBase.descendants.map(&:name).join(', ')}"
|
11
|
-
Osbourne.logger.info "Provisioning queues for all workers"
|
10
|
+
Osbourne.logger.info "[Osbourne] Workers found: #{Osbourne::WorkerBase.descendants.map(&:name).join(', ')}"
|
11
|
+
Osbourne.logger.info "[Osbourne] Provisioning queues for all workers"
|
12
12
|
Osbourne::WorkerBase.descendants.each(&:provision)
|
13
13
|
end
|
14
14
|
end
|
@@ -19,16 +19,16 @@ module Osbourne
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def subscribe(topic)
|
22
|
-
Osbourne.logger.info("Checking subscription for #{queue.
|
22
|
+
Osbourne.logger.info("[Osbourne] Checking subscription for #{queue.prefixed_name} to #{topic.prefixed_name}")
|
23
23
|
return if Osbourne.existing_subscriptions_for(topic).include? queue.arn
|
24
24
|
|
25
|
-
Osbourne.logger.info("Subscribing #{queue.
|
25
|
+
Osbourne.logger.info("[Osbourne] Subscribing #{queue.prefixed_name} to #{topic.prefixed_name}")
|
26
26
|
sns.subscribe(topic_arn: topic.arn, protocol: "sqs", endpoint: queue.arn).subscription_arn
|
27
27
|
Osbourne.clear_subscriptions_for(topic)
|
28
28
|
end
|
29
29
|
|
30
30
|
def set_queue_policy
|
31
|
-
Osbourne.logger.info("Setting policy for #{queue.
|
31
|
+
Osbourne.logger.info("[Osbourne] Setting policy for #{queue.prefixed_name} (attributes: #{build_policy})")
|
32
32
|
sqs.set_queue_attributes(queue_url: queue.url, attributes: build_policy)
|
33
33
|
end
|
34
34
|
|
@@ -37,7 +37,7 @@ module Osbourne
|
|
37
37
|
{
|
38
38
|
"Policy" => {
|
39
39
|
"Version" => "2012-10-17",
|
40
|
-
"Id" => "Osbourne/#{queue.
|
40
|
+
"Id" => "Osbourne/#{queue.prefixed_name}/SNSPolicy",
|
41
41
|
"Statement" => topics.map {|t| build_policy_statement(t) }
|
42
42
|
}.to_json
|
43
43
|
}
|
@@ -45,7 +45,7 @@ module Osbourne
|
|
45
45
|
|
46
46
|
def build_policy_statement(topic)
|
47
47
|
{
|
48
|
-
"Sid" => "Sid#{topic.
|
48
|
+
"Sid" => "Sid#{topic.prefixed_name}",
|
49
49
|
"Effect" => "Allow",
|
50
50
|
"Principal" => {"AWS" => "*"},
|
51
51
|
"Action" => "SQS:SendMessage",
|
@@ -6,11 +6,11 @@ module Osbourne
|
|
6
6
|
class MockPubsub
|
7
7
|
class << self
|
8
8
|
def mock_publish(topic, message)
|
9
|
-
prefixed_topic = Osbourne.prefixer(topic)
|
9
|
+
# prefixed_topic = Osbourne.prefixer(topic)
|
10
10
|
parsed_message = parse(message)
|
11
11
|
Osbourne::WorkerBase.descendants.each do |worker|
|
12
|
-
msg = Osbourne::Test::Message.new(topic:
|
13
|
-
worker.new.process(msg) if worker.config[:topic_names].include?
|
12
|
+
msg = Osbourne::Test::Message.new(topic: topic, body: parsed_message)
|
13
|
+
worker.new.process(msg) if worker.config[:topic_names].include? topic
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
data/lib/osbourne/topic.rb
CHANGED
@@ -19,7 +19,7 @@ module Osbourne
|
|
19
19
|
parsed_message = parse(message)
|
20
20
|
return if Osbourne.test_mode?
|
21
21
|
|
22
|
-
Osbourne.logger.info "[PUB] TOPIC: `#{prefixed_name}` MESSAGE: `#{parsed_message}`"
|
22
|
+
Osbourne.logger.info "[Osbourne] [PUB] TOPIC: `#{prefixed_name}` MESSAGE: `#{parsed_message}`"
|
23
23
|
sns.publish(topic_arn: arn, message: parsed_message)
|
24
24
|
end
|
25
25
|
|
@@ -28,7 +28,7 @@ module Osbourne
|
|
28
28
|
def ensure_topic
|
29
29
|
return if Osbourne.test_mode?
|
30
30
|
|
31
|
-
Osbourne.logger.debug "Ensuring topic `#{prefixed_name}` exists"
|
31
|
+
Osbourne.logger.debug "[Osbourne] Ensuring topic `#{prefixed_name}` exists"
|
32
32
|
Osbourne.cache.fetch("osbourne_existing_topic_arn_for_#{prefixed_name}", ex: 1.minute) do
|
33
33
|
sns.create_topic(name: prefixed_name).topic_arn
|
34
34
|
end
|
data/lib/osbourne/version.rb
CHANGED
data/lib/osbourne/worker_base.rb
CHANGED
@@ -53,12 +53,12 @@ module Osbourne
|
|
53
53
|
def register_dead_letter_queue
|
54
54
|
return unless config[:dead_letter]
|
55
55
|
|
56
|
-
Osbourne.logger.info "#{self.class.name} dead letter queue: arn: [#{dead_letter_queue.arn}], max retries: #{config[:max_retry_count]}"
|
56
|
+
Osbourne.logger.info "[Osbourne] #{self.class.name} dead letter queue: arn: [#{dead_letter_queue.arn}], max retries: #{config[:max_retry_count]}"
|
57
57
|
queue.redrive(config[:max_retry_count], dead_letter_queue.arn)
|
58
58
|
end
|
59
59
|
|
60
60
|
def register
|
61
|
-
Osbourne.logger.info "#{self.class.name} subscriptions: Topics: [#{config[:topic_names].join(', ')}], Queue: [#{config[:queue_name]}]"
|
61
|
+
Osbourne.logger.info "[Osbourne] #{self.class.name} subscriptions: Topics: [#{config[:topic_names].join(', ')}], Queue: [#{config[:queue_name]}]"
|
62
62
|
self.topics = config[:topic_names].map {|tn| Topic.new(tn) }
|
63
63
|
self.queue = Queue.new(config[:queue_name])
|
64
64
|
self.subscriptions = Subscription.new(topics, queue)
|
@@ -81,8 +81,8 @@ module Osbourne
|
|
81
81
|
dead_letter_queue: true,
|
82
82
|
max_retry_count: Osbourne.max_retry_count)
|
83
83
|
self.config = {
|
84
|
-
topic_names: Array(topics)
|
85
|
-
queue_name:
|
84
|
+
topic_names: Array(topics),
|
85
|
+
queue_name: queue_name,
|
86
86
|
max_batch_size: max_batch_size,
|
87
87
|
max_wait: max_wait,
|
88
88
|
threads: threads,
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: osbourne
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Steve Allen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-core
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '3'
|
19
|
+
version: '3.37'
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '4'
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: '3'
|
29
|
+
version: '3.37'
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '4'
|
@@ -71,7 +71,7 @@ dependencies:
|
|
71
71
|
- !ruby/object:Gem::Version
|
72
72
|
version: '2'
|
73
73
|
- !ruby/object:Gem::Dependency
|
74
|
-
name:
|
74
|
+
name: railties
|
75
75
|
requirement: !ruby/object:Gem::Requirement
|
76
76
|
requirements:
|
77
77
|
- - ">="
|