osbourne 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +674 -0
  3. data/README.md +106 -0
  4. data/Rakefile +30 -0
  5. data/bin/cli/base.rb +41 -0
  6. data/bin/osbourne +33 -0
  7. data/lib/generators/osbourne/install/install_generator.rb +14 -0
  8. data/lib/generators/osbourne/install/templates/osbourne_initializer_template.template +30 -0
  9. data/lib/generators/osbourne/install/templates/osbourne_yaml_template.template +15 -0
  10. data/lib/generators/osbourne/worker/USAGE +11 -0
  11. data/lib/generators/osbourne/worker/templates/worker_template.template +35 -0
  12. data/lib/generators/osbourne/worker/worker_generator.rb +20 -0
  13. data/lib/osbourne.rb +43 -0
  14. data/lib/osbourne/config/file_loader.rb +22 -0
  15. data/lib/osbourne/config/shared_configs.rb +37 -0
  16. data/lib/osbourne/existing_subscriptions.rb +40 -0
  17. data/lib/osbourne/launcher.rb +60 -0
  18. data/lib/osbourne/locks/base.rb +69 -0
  19. data/lib/osbourne/locks/memory.rb +69 -0
  20. data/lib/osbourne/locks/noop.rb +25 -0
  21. data/lib/osbourne/locks/redis.rb +56 -0
  22. data/lib/osbourne/message.rb +55 -0
  23. data/lib/osbourne/poller.rb +0 -0
  24. data/lib/osbourne/queue.rb +43 -0
  25. data/lib/osbourne/railtie.rb +20 -0
  26. data/lib/osbourne/runner.rb +86 -0
  27. data/lib/osbourne/services/queue_provisioner.rb +14 -0
  28. data/lib/osbourne/services/sns.rb +17 -0
  29. data/lib/osbourne/services/sqs.rb +17 -0
  30. data/lib/osbourne/subscription.rb +36 -0
  31. data/lib/osbourne/topic.rb +34 -0
  32. data/lib/osbourne/version.rb +5 -0
  33. data/lib/osbourne/worker_base.rb +91 -0
  34. data/lib/tasks/message_plumber_tasks.rake +6 -0
  35. metadata +348 -0
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Osbourne
4
+ module ExistingSubscriptions
5
+ attr_reader :existing_subscriptions
6
+ def existing_subscriptions_for(topic)
7
+ Osbourne.cache.fetch("osbourne_existng_subs_for_#{topic.name}") do
8
+ results = []
9
+ handled = Osbourne.lock.try_with_lock("osbourne_lock_subs_for_#{topic.name}") do
10
+ results = fetch_existing_subscriptions_for(topic)
11
+ end
12
+ return results if handled
13
+
14
+ sleep(0.5)
15
+ existing_subscriptions_for(topic)
16
+ end
17
+ # @existing_subscriptions ||= {}
18
+ # @existing_subscriptions[topic.name] ||=
19
+ end
20
+
21
+ def clear_subscriptions_for(topic)
22
+ Osbourne.cache.delete("osbourne_existng_subs_for_#{topic.name}")
23
+ end
24
+
25
+ private
26
+
27
+ def fetch_existing_subscriptions_for(topic)
28
+ results = []
29
+ r = nil
30
+ loop do
31
+ params = {topic_arn: topic.arn}
32
+ params[:next_token] = r.next_token if r.try(:next_token)
33
+ r = Osbourne.sns_client.list_subscriptions_by_topic(params)
34
+ results << r.subscriptions.map(&:endpoint)
35
+ break unless r.try(:next_token).presence
36
+ end
37
+ results.flatten.uniq
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "osbourne"
4
+ require "byebug"
5
+
6
+ module Osbourne
7
+ class Launcher
8
+ attr_accessor :threads
9
+ def initialize; end
10
+
11
+ def start!
12
+ Osbourne.logger.info("Launching Osbourne workers")
13
+ @stop = false
14
+ @threads = polling_threads
15
+ polling_threads.each(&:run)
16
+ end
17
+
18
+ def stop
19
+ @stop = true
20
+ end
21
+
22
+ def stop!
23
+ @threads.each {|thr| Thread.kill(thr) }
24
+ end
25
+
26
+ def polling_threads
27
+ Osbourne::WorkerBase.descendants.map do |worker|
28
+ Thread.new {
29
+ worker_instance = worker.new
30
+ loop do
31
+ poll(worker_instance)
32
+ sleep(Osbourne.sleep_time)
33
+ break if @stop
34
+ end
35
+ }
36
+ end
37
+ end
38
+
39
+ def poll(worker)
40
+ worker.polling_queue.receive_messages(
41
+ wait_time_seconds: worker.config[:max_wait],
42
+ max_number_of_messages: worker.config[:max_batch_size]
43
+ ).each {|msg| process(worker, Osbourne::Message.new(msg)) }
44
+ end
45
+
46
+ private
47
+
48
+ def process(worker, message)
49
+ Osbourne.logger.info("[MSG] Worker: #{worker.class.name} Valid: #{message.valid?} ID: #{message.id} Body: #{message.raw_body}") # rubocop:disable Metrics/LineLength
50
+ return unless message.valid?
51
+
52
+ # hard_lock to prevent duplicate processing over the hard_lock lifespan
53
+ Osbourne.lock.try_with_lock(message.id, hard_lock: true) do
54
+ message.delete if worker.process(message)
55
+ end
56
+ rescue Exception => ex # rubocop:disable Lint/RescueException
57
+ Osbourne.logger.error("[MSG ID: #{message.id}] #{ex.message}")
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "osbourne"
4
+
5
+ module Osbourne
6
+ module Locks
7
+ module Base
8
+ DEFAULT_SOFT_TTL = (5 * 60).freeze # 5 minutes
9
+ DEFAULT_HARD_TTL = (24 * 60 * 60).freeze # 24 hours
10
+
11
+ attr_reader :soft_ttl, :hard_ttl
12
+
13
+ def initialize(options={})
14
+ self.soft_ttl = options.fetch(:soft_ttl, DEFAULT_SOFT_TTL)
15
+ self.hard_ttl = options.fetch(:hard_ttl, DEFAULT_HARD_TTL)
16
+ end
17
+
18
+ def soft_lock(id)
19
+ lock(lock_key(id), soft_ttl)
20
+ end
21
+
22
+ def hard_lock(id)
23
+ lock!(lock_key(id), hard_ttl)
24
+ end
25
+
26
+ def unlock(id)
27
+ unlock!(lock_key(id))
28
+ end
29
+
30
+ def try_with_lock(id, hard_lock: false)
31
+ if soft_lock(id)
32
+ begin
33
+ yield
34
+ rescue => e # rubocop:disable Style/RescueStandardError
35
+ unlock(id)
36
+ raise e
37
+ end
38
+
39
+ hard_lock(id) if hard_lock
40
+ true
41
+ else
42
+ false
43
+ end
44
+ end
45
+
46
+ protected
47
+
48
+ def lock(_key, _ttl)
49
+ raise NotImplementedError
50
+ end
51
+
52
+ def lock!(_key, _ttl)
53
+ raise NotImplementedError
54
+ end
55
+
56
+ def unlock!(_key)
57
+ raise NotImplementedError
58
+ end
59
+
60
+ private
61
+
62
+ attr_writer :soft_ttl, :hard_ttl
63
+
64
+ def lock_key(id)
65
+ "osbourne:lock:#{id}"
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "osbourne"
4
+
5
+ module Osbourne
6
+ module Locks
7
+ class Memory
8
+ include Base
9
+
10
+ class << self
11
+ def store
12
+ @store ||= {}
13
+ end
14
+
15
+ def semaphore
16
+ @semaphore ||= Mutex.new
17
+ end
18
+ end
19
+
20
+ protected
21
+
22
+ def lock(key, ttl)
23
+ reap
24
+
25
+ store do |store|
26
+ if store.key?(key)
27
+ false
28
+ else
29
+ store[key] = Time.current + ttl
30
+ true
31
+ end
32
+ end
33
+ end
34
+
35
+ def lock!(key, ttl)
36
+ reap
37
+
38
+ store do |store|
39
+ store[key] = Time.current + ttl
40
+ end
41
+ end
42
+
43
+ def unlock!(key)
44
+ store do |store|
45
+ store.delete(key)
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ def store
52
+ semaphore.synchronize do
53
+ yield self.class.store
54
+ end
55
+ end
56
+
57
+ def reap
58
+ store do |store|
59
+ now = Time.current
60
+ store.delete_if {|_, expires_at| expires_at <= now }
61
+ end
62
+ end
63
+
64
+ def semaphore
65
+ self.class.semaphore
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "osbourne"
4
+
5
+ module Osbourne
6
+ module Locks
7
+ class NOOP
8
+ include Base
9
+
10
+ protected
11
+
12
+ def lock(_key, _ttl)
13
+ true
14
+ end
15
+
16
+ def lock!(key, ttl)
17
+ # do nothing
18
+ end
19
+
20
+ def unlock!(key)
21
+ # do nothing
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "osbourne"
4
+
5
+ module Osbourne
6
+ module Locks
7
+ class Redis
8
+ include Base
9
+
10
+ def initialize(options={})
11
+ super(options)
12
+
13
+ self.client = options.fetch(:client) do
14
+ require "redis"
15
+ ::Redis.new(options)
16
+ end
17
+ end
18
+
19
+ protected
20
+
21
+ def lock(key, ttl)
22
+ with_pool do |client|
23
+ client.set(key, (Time.current + ttl).to_i, ex: ttl, nx: true)
24
+ end
25
+ end
26
+
27
+ def lock!(key, ttl)
28
+ with_pool do |client|
29
+ client.set(key, (Time.current + ttl).to_i, ex: ttl)
30
+ end
31
+ end
32
+
33
+ def unlock!(key)
34
+ with_pool do |client|
35
+ client.del(key)
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ attr_accessor :client
42
+
43
+ def with_pool(&block)
44
+ if pool?
45
+ client.with(&block)
46
+ else
47
+ yield client
48
+ end
49
+ end
50
+
51
+ def pool?
52
+ defined?(ConnectionPool) && client.is_a?(ConnectionPool)
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "osbourne"
4
+
5
+ module Osbourne
6
+ class Message
7
+ attr_accessor :message
8
+ def initialize(message)
9
+ @message = message
10
+ end
11
+
12
+ def json?
13
+ !JSON.parse(parsed_content["Message"]).nil?
14
+ rescue JSON::ParserError
15
+ false
16
+ end
17
+
18
+ def valid?
19
+ message.md5_of_body == Digest::MD5.hexdigest(message.body)
20
+ end
21
+
22
+ def id
23
+ message.message_id
24
+ end
25
+
26
+ def parsed_body
27
+ JSON.parse(parsed_content["Message"])
28
+ rescue JSON::ParserError
29
+ parsed_content["Message"]
30
+ end
31
+
32
+ def delete
33
+ message.delete
34
+ Osbourne.logger.info "[MSG ID: #{id}] Cleared"
35
+ end
36
+
37
+ def topic
38
+ parsed_content["TopicArn"].split(":").last
39
+ end
40
+
41
+ def raw_body
42
+ message.body
43
+ end
44
+
45
+ private
46
+
47
+ def parsed_content
48
+ @parsed_content ||= JSON.parse(message.body)
49
+ end
50
+
51
+ def body
52
+ parsed_content["Message"]
53
+ end
54
+ end
55
+ end
File without changes
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Osbourne
4
+ class Queue
5
+ include Services::SQS
6
+ attr_reader :name
7
+ def initialize(name)
8
+ @name = name
9
+ arn
10
+ end
11
+
12
+ def url
13
+ @url ||= ensure_queue
14
+ end
15
+
16
+ def arn
17
+ @arn ||= get_attributes["QueueArn"]
18
+ end
19
+
20
+ def redrive(retries, dead_letter_arn)
21
+ sqs.set_queue_attributes(queue_url: url,
22
+ attributes: {
23
+ 'RedrivePolicy': {
24
+ 'deadLetterTargetArn': dead_letter_arn,
25
+ 'maxReceiveCount': retries
26
+ }.to_json
27
+ })
28
+ end
29
+
30
+ private
31
+
32
+ def ensure_queue
33
+ Osbourne.logger.debug "Ensuring queue `#{name}` exists"
34
+ Osbourne.cache.fetch("osbourne_url_for_#{name}") do
35
+ sqs.create_queue(queue_name: name).queue_url
36
+ end
37
+ end
38
+
39
+ def get_attributes(attrs: %w[QueueArn])
40
+ sqs.get_queue_attributes(queue_url: url, attribute_names: attrs).attributes
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails"
4
+ require "osbourne/config/file_loader"
5
+
6
+ module Osbourne
7
+ class Railtie < Rails::Railtie
8
+ config.osbourne = ActiveSupport::OrderedOptions.new
9
+
10
+ initializer "osbourne.configure", after: :load_config_initializers do |_app|
11
+ env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
12
+
13
+ %w[config/osbourne.yml.erb config/osbourne.yml].find do |filename|
14
+ Osbourne::Config::FileLoader.load(filename, env)
15
+ end
16
+
17
+ Osbourne.provision_worker_queues
18
+ end
19
+ end
20
+ end