osbourne 0.2.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 699bb698e3dae6272c3038bdc052597e0a6677e8f380bc1f64c12ae1c18c63a0
-  data.tar.gz: ca0bf27de44bff4a2e96755d0c041a73bbb75bc941c7166c44dfdd144630d3e1
+  metadata.gz: 30bc10e5c04691de985ff77f554ac23596052bbdd70de1d642f875378065f7e8
+  data.tar.gz: aebe87fe5fff351c0ab9f9add37508b81f1f1e5147b06e8c41cfd555db216c23
 SHA512:
-  metadata.gz: b464098af275a72c4aa7a2ba600b1bdb709485f4ddaac98462f6513b8c2ae6930c9e8b8bdb478cf58618e39f1d224070957af61fa4633a4032e53ea0e10c3c9e
-  data.tar.gz: 32b5e08de9d525e95408525e0d9c0723383a33d3ac8aa688e323af9c49db1bbb15023e2fde9de099750e10a166770d58e3aba8826392702f61355bfd715bb308
+  metadata.gz: 93a0d8128556f2a77abe7c82a31fdd73d41c2647c6021a76d1aa8a6bfdceb86c32d7a361d0a14d01597b75fd84644090038bedc7f98e0dbad28fcedadd54948e
+  data.tar.gz: 4fe3bea017685121b90e49289aee1998a25b606ec1ab0b01a7519cecffd8193e922cfe99bcb47dd402f1661defe1e073d89317def80a334a177da28df509961f

data/README.md

@@ -148,7 +148,7 @@ This will generate a `WorkerNameWorker` in `app/workers/worker_name_worker.rb`,
 
 There is some configuration options available within the generated worker. See comments in the worker for options.
 
-SNS messages broadcast through an SQS queue will have some layers of envelop wrappings around them. The `message` object passed into the worker's `#perform` method contains some helpers to make parsing this easier. `#parsed_body` is the most important one, as it contains the actual string of the message that was originally broadcast.
+SNS messages broadcast through an SQS queue will have some layers of envelop wrappings around them. The `message` object passed into the worker's `#perform` method contains some helpers to make parsing this easier. `#message_body` is the most important one, as it contains the actual string of the message that was originally broadcast.
 
 ### Running workers
 

data/lib/generators/osbourne/install/templates/osbourne_initializer_template.template

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-Dir[File.expand_path("app/workers/**/*.rb")].each {|f| require f }
-
 Osbourne.configure do |config|
   # Defaults to a null cache. Uncomment to use the Rails cache,
   # or substitute any rails-cache compatible adapter of your choice

data/lib/generators/osbourne/worker/templates/worker_template.template

@@ -2,33 +2,23 @@
 
 class <%= name.singularize.camelcase %>Worker < Osbourne::WorkerBase
 
-  worker_config topics: %w[<%= topic.join(" ") %>] #, max_batch_size: 10, max_wait: 10
+  worker_config topics: %w[<%= topic.join(" ") %>] 
+  # Other available onfig options and their defaults:
+  # max_batch_size: 10
+  # max_wait: 10
+  # threads: Osbourne.threads_per_worker
+  # queue_name: <%= name %>_queue
+  # dead_letter_queue: true
+  # max_retry_count: Osbourne.max_retry_count
 
   def process(message)
-    puts message.parsed_body # If the message came from a SNS broadcast,
-                             #   as opposed to a direct SQS message, it will come from here 
-    puts message.topic       # The topic this message was published to.
-                             #   Useful if this worker subscribes to more than one topic
-    puts message.id          # The UUID for this message. Useful for validating if this is
-                             #   the first time it has been processed 
+    puts message.sns?         # Was this message broadcast via SNS?
+    puts message.raw_body     # The only way to access a message that wasn't sent via SNS
+    puts message.message_body # If the message came from a SNS broadcast,
+                              #   as opposed to a direct SQS message, it will come from here 
+    puts message.topic        # The topic this message was published to.
+                              #   Useful if this worker subscribes to more than one topic
+    puts message.id           # The UUID for this message. Useful for validating if this is
+                              #   the first time it has been processed 
   end
-
-  # class << self
-  #   # override this to set how many times a message will be retried before
-  #   # being redirected to the dead letter queue (if enabled)
-  #   def max_retry_count
-  #   end
-
-  #   # override this to set the worker's dead letter queue name
-  #   def dead_letter_queue_name
-  #   end
-
-  #   # override this to `false` to disable the dead letter queu for this worker
-  #   def dead_letter_queue
-  #   end
-
-  #   # override this to set the worker queue name
-  #   def queue_name
-  #   end
-  # end
 end

data/lib/osbourne.rb

@@ -22,7 +22,7 @@ module Osbourne
     include Osbourne::Config::SharedConfigs
     include Osbourne::Services::QueueProvisioner
     include Osbourne::ExistingSubscriptions
-    attr_writer :sns_client, :sqs_client
+    # attr_writer :sns_client, :sqs_client
 
     def sns_client
       return if Osbourne.test_mode?
@@ -36,6 +36,10 @@ module Osbourne
       @sqs_client ||= Aws::SQS::Client.new(Osbourne.config.sqs_config)
     end
 
+    attr_writer :sns_client
+
+    attr_writer :sqs_client
+
     def publish(topic, message)
       Topic.new(topic).publish(message)
     end

data/lib/osbourne/config/file_loader.rb

@@ -13,8 +13,8 @@ module Osbourne
         base_opts = YAML.safe_load(ERB.new(IO.read(cfile)).result) || {}
         env_opts = base_opts[environment] || {}
 
-        Osbourne.config.sns_config = env_opts["publisher"].symbolize_keys || {}
-        Osbourne.config.sqs_config = env_opts["subscriber"].symbolize_keys || {}
+        Osbourne.config.sns_config = env_opts["publisher"]&.symbolize_keys || {}
+        Osbourne.config.sqs_config = env_opts["subscriber"]&.symbolize_keys || {}
         true
       end
 

data/lib/osbourne/config/shared_configs.rb

@@ -34,15 +34,19 @@ module Osbourne
       end
 
       def logger
-        config.logger ||= Logger.new(STDOUT)
+        config.logger ||= Logger.new("log/osbourne.log")
       end
 
       def lock
-        config.lock || Osbourne::Locks::NOOP.new
+        config.lock ||= Osbourne::Locks::NOOP.new
       end
 
       def sleep_time
-        config.sleep_time || 15
+        config.sleep_time ||= 15
+      end
+
+      def threads_per_worker
+        config.threads_per_worker ||= 5
       end
     end
   end

data/lib/osbourne/existing_subscriptions.rb

@@ -4,6 +4,7 @@ module Osbourne
   module ExistingSubscriptions
     attr_reader :existing_subscriptions
     def existing_subscriptions_for(topic)
+      Osbourne.cache.delete("osbourne_existng_subs_for_#{topic.name}")
       Osbourne.cache.fetch("osbourne_existng_subs_for_#{topic.name}") do
         results = []
         handled = Osbourne.lock.try_with_lock("osbourne_lock_subs_for_#{topic.name}") do
@@ -14,8 +15,6 @@ module Osbourne
         sleep(0.5)
         existing_subscriptions_for(topic)
       end
-      # @existing_subscriptions ||= {}
-      # @existing_subscriptions[topic.name] ||=
     end
 
     def clear_subscriptions_for(topic)

data/lib/osbourne/launcher.rb

@@ -10,50 +10,61 @@ module Osbourne
     def start!
       Osbourne.logger.info("Launching Osbourne workers")
       @stop = false
-      @threads = polling_threads
-      polling_threads.each(&:run)
+      @threads = global_polling_threads
+    end
+
+    def wait!
+      threads.map(&:join)
     end
 
     def stop
+      puts "Signal caught. Terminating workers..."
       @stop = true
     end
 
     def stop!
+      puts "Signal caught. Terminating workers..."
       @threads.each {|thr| Thread.kill(thr) }
     end
 
-    def polling_threads
+    def global_polling_threads
       Osbourne::WorkerBase.descendants.map do |worker|
-        Thread.new {
-          worker_instance = worker.new
-          loop do
-            poll(worker_instance)
-            sleep(Osbourne.sleep_time)
-            break if @stop
-          end
-        }
+        Osbourne.logger.debug("Spawning thread for #{worker.name}")
+        Thread.new { poll(worker) }
+      end
+    end
+
+    def worker_polling_threads(worker)
+      my_threads = []
+      worker.config[:threads].times do
+        my_threads << Thread.new { poll(worker) }
       end
+      my_threads.each(&:join)
     end
 
     def poll(worker)
-      worker.polling_queue.receive_messages(
-        wait_time_seconds:      worker.config[:max_wait],
-        max_number_of_messages: worker.config[:max_batch_size]
-      ).each {|msg| process(worker, Osbourne::Message.new(msg)) }
+      worker.polling_queue.poll(wait_time_seconds:      worker.config[:max_wait_time],
+                                max_number_of_messages: worker.config[:max_batch_size],
+                                skip_delete:            true) do |messages|
+        messages.map do |msg|
+          worker.polling_queue.delete_message(msg) if process(worker, Osbourne::Message.new(msg))
+        end
+        throw :stop_polling if @stop
+      end
     end
 
     private
 
     def process(worker, message)
-      Osbourne.logger.info("[MSG] Worker: #{worker.class.name} Valid: #{message.valid?} ID: #{message.id} Body: #{message.raw_body}") # rubocop:disable Metrics/LineLength
-      return unless message.valid?
+      Osbourne.logger.info("[MSG] Worker: #{worker.name} Valid: #{message.valid?} ID: #{message.id}")
+      return false unless message.valid? && Osbourne.lock.soft_lock(message.id)
 
-      # hard_lock to prevent duplicate processing over the hard_lock lifespan
-      Osbourne.lock.try_with_lock(message.id, hard_lock: true) do
-        message.delete if worker.process(message)
+      Osbourne.cache.fetch(message.id, ex: 24.hours) do
+        worker.new.process(message).tap {|_| Osbourne.lock.unlock(message.id) }
       end
     rescue Exception => ex # rubocop:disable Lint/RescueException
       Osbourne.logger.error("[MSG ID: #{message.id}] #{ex.message}")
+      false
     end
   end
 end

data/lib/osbourne/locks/base.rb

@@ -28,9 +28,10 @@ module Osbourne
       end
 
       def try_with_lock(id, hard_lock: false)
-        if soft_lock(id)
+        if lock(lock_key(id), soft_ttl)
           begin
             yield
+            unlock(id)
           rescue => e # rubocop:disable Style/RescueStandardError
             unlock(id)
             raise e

data/lib/osbourne/message.rb

@@ -2,54 +2,99 @@
 
 require "osbourne"
 
+##
+# Represents a single message recieved by an Osbourne listener
+
 module Osbourne
   class Message
-    attr_accessor :message
+    attr_reader :message
     def initialize(message)
       @message = message
     end
 
+    ##
+    #
+    # @return [Boolean] This will be `true` if the SNS message is also JSON
     def json?
-      !JSON.parse(parsed_content["Message"]).nil?
-    rescue JSON::ParserError
-      false
+      return false unless valid?
+
+      sns_body.is_a?(Hash)
     end
 
+    ##
+    # Does the message match the checksum? If not, the message has likely been mangled in transit
+    # @return [Boolean]
     def valid?
-      message.md5_of_body == Digest::MD5.hexdigest(message.body)
+      @valid ||= message.md5_of_body == Digest::MD5.hexdigest(message.body)
     end
 
+    ##
+    # Osbourne has built-in message deduplication, but it's still a good idea to do some verification in a worker
+    #
+    # @return [String] The UUID of the recieved message
     def id
       message.message_id
     end
 
-    def parsed_body
-      JSON.parse(parsed_content["Message"])
-    rescue JSON::ParserError
-      parsed_content["Message"]
+    ##
+    # If the message was broadcast via SNS, the body will be available here.
+    #
+    # @return [Hash] If the message was JSON
+    # @return [String] If the message was not JSON
+    # @return [nil] If the message was not broadcast via SNS.
+    # @see #raw_body #raw_body for the raw body string
+    def message_body
+      sns_body
     end
 
+    ##
+    # Deletes the message from SQS to prevent retrying against another worker.
+    # Osbourne will automatically delete a message sent to a worker as long as the Osourbne::WorkerBase#process method returns `true`
+
     def delete
       message.delete
       Osbourne.logger.info "[MSG ID: #{id}] Cleared"
     end
 
+    ##
+    # The SNS topic that this message was broadcast to
+    # @return [String] if the message was broadcast via SNS, this will be the topic
+    # @return [nil] if the message was not broadcast via SNS
     def topic
-      parsed_content["TopicArn"].split(":").last
+      return nil unless sns?
+
+      json_body["TopicArn"].split(":").last
     end
 
+    ##
+    # @return [String] The raw string representation of the message
     def raw_body
       message.body
     end
 
+    ##
+    # Just because a message was recieved via SQS, doesn't mean it was originally broadcast via SNS
+    # @return [Boolean] Was the message broadcast via SNS?
+    def sns?
+      json_body.is_a?(Hash) && (json_body.keys <=> %w[Message Type TopicArn MessageId]).zero?
+    end
+
     private
 
-    def parsed_content
-      @parsed_content ||= JSON.parse(message.body)
+    def safe_json(content)
+      JSON.parse(content)
+    rescue JSON::ParserError
+      false
+    end
+
+    def sns_body
+      return unless sns?
+
+      @sns_body ||= safe_json(json_body["Message"]) || json_body["Message"]
     end
 
-    def body
-      parsed_content["Message"]
+    def json_body
+      @json_body || safe_json(message.body)
     end
   end
 end

data/lib/osbourne/runner.rb

@@ -36,7 +36,7 @@ module Osbourne
           signal = readable_io.first[0].gets.strip
           handle_signal(signal)
         end
-        @launcher.threads.map(&:join)
+        @launcher.wait!
       rescue Interrupt
         @launcher.stop!
         exit 0

data/lib/osbourne/services/queue_provisioner.rb

@@ -4,11 +4,11 @@ module Osbourne
   module Services
     module QueueProvisioner
       def provision_worker_queues
+        Dir[File.expand_path("app/workers/**/*.rb")].each {|f| require f }
         return if Osbourne.test_mode?
 
         Osbourne.logger.info "Workers found: #{Osbourne::WorkerBase.descendants.map(&:name).join(', ')}"
         Osbourne.logger.info "Provisioning queues for all workers"
-
         Osbourne::WorkerBase.descendants.each(&:provision)
       end
     end

data/lib/osbourne/subscription.rb

@@ -3,34 +3,57 @@
 module Osbourne
   class Subscription
     include Services::SNS
-    attr_reader :topic, :queue
-    def initialize(topic, queue)
-      @topic = topic
+    include Services::SQS
+    attr_reader :topics, :queue
+    def initialize(topics, queue)
+      @topics = topics
       @queue = queue
-      arn
-    end
-
-    def arn
-      @arn ||= subscribe
+      subscribe_all
     end
 
     private
 
-    def subscribe # rubocop:disable Metrics/AbcSize
+    def subscribe_all
+      topics.each {|topic| subscribe(topic) }
+      set_queue_policy
+    end
+
+    def subscribe(topic)
       Osbourne.logger.info("Checking subscription for #{queue.name} to #{topic.name}")
       return if Osbourne.existing_subscriptions_for(topic).include? queue.arn
 
-      handled = Osbourne.lock.try_with_lock("osbourne_sub_lock_#{topic.name}") do
-        Osbourne.logger.info("Subscribing #{queue.name} to #{topic.name}")
-        @arn = sns.subscribe(topic_arn: topic.arn, protocol: "sqs", endpoint: queue.arn).subscription_arn
-        Osbourne.clear_subscriptions_for(topic)
-      end
-      if handled
-        @arn
-      else
-        sleep(3)
-        subscribe
-      end
+      Osbourne.logger.info("Subscribing #{queue.name} to #{topic.name}")
+      sns.subscribe(topic_arn: topic.arn, protocol: "sqs", endpoint: queue.arn).subscription_arn
+      Osbourne.clear_subscriptions_for(topic)
+    end
+
+    def set_queue_policy
+      Osbourne.logger.info("Setting policy for #{queue.name} (attributes: #{build_policy})")
+      sqs.set_queue_attributes(queue_url: queue.url, attributes: build_policy)
+    end
+
+    def build_policy
+      # The aws ruby SDK doesn't have a policy builder :{
+      {
+        "Policy" => {
+          "Version"   => "2012-10-17",
+          "Id"        => "Osbourne/#{queue.name}/SNSPolicy",
+          "Statement" => topics.map {|t| build_policy_statement(t) }
+        }.to_json
+      }
+    end
+
+    def build_policy_statement(topic)
+      {
+        "Sid"       => "Sid#{topic.name}",
+        "Effect"    => "Allow",
+        "Principal" => {"AWS" => "*"},
+        "Action"    => "SQS:SendMessage",
+        "Resource"  => queue.arn,
+        "Condition" => {
+          "ArnEquals" => {"aws:SourceArn" => topic.arn}
+        }
+      }
     end
   end
 end

data/lib/osbourne/topic.rb

@@ -26,7 +26,7 @@ module Osbourne
       return if Osbourne.test_mode?
 
       Osbourne.logger.debug "Ensuring topic `#{name}` exists"
-      Osbourne.cache.fetch("osbourne_existing_topic_arn_for_#{name}") do
+      Osbourne.cache.fetch("osbourne_existing_topic_arn_for_#{name}", ex: 1.minute) do
         sns.create_topic(name: name).topic_arn
       end
     end

data/lib/osbourne/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Osbourne
-  VERSION = "0.2.2"
+  VERSION = "1.0.0"
 end

data/lib/osbourne/worker_base.rb

@@ -21,7 +21,7 @@ module Osbourne
     end
 
     def polling_queue
-      self.class.polling_queue
+      @polling_queue ||= self.class.polling_queue
     end
 
     class << self
@@ -36,24 +36,14 @@ module Osbourne
         register_dead_letter_queue
       end
 
-      def max_retry_count
-        Osbourne.max_retry_count
-      end
-
-      def dead_letter_queue_name
-        "#{config[:queue_name]}-dead-letter"
-      end
-
       def dead_letter_queue
-        @dead_letter_queue ||= Queue.new(dead_letter_queue_name)
-      end
+        return unless config[:dead_letter]
 
-      def queue_name
-        default_queue_name
+        @dead_letter_queue ||= Queue.new(dead_letter_queue_name)
       end
 
       def polling_queue
-        @polling_queue ||= Aws::SQS::Queue.new(queue.url, client: Osbourne.sqs_client)
+        Aws::SQS::QueuePoller.new(queue.url, client: Osbourne.sqs_client)
       end
     end
 
@@ -61,31 +51,46 @@ module Osbourne
       private
 
       def register_dead_letter_queue
-        return unless Osbourne.dead_letter
+        return unless config[:dead_letter]
 
-        Osbourne.logger.info "#{self.class.name} dead letter queue: arn: [#{dead_letter_queue.arn}], max retries: #{max_retry_count}" # rubocop:disable Metrics/LineLength
-        queue.redrive(max_retry_count, dead_letter_queue.arn)
+        Osbourne.logger.info "#{self.class.name} dead letter queue: arn: [#{dead_letter_queue.arn}], max retries: #{config[:max_retry_count]}" # rubocop:disable Metrics/LineLength
+        queue.redrive(config[:max_retry_count], dead_letter_queue.arn)
       end
 
       def register
         Osbourne.logger.info "#{self.class.name} subscriptions: Topics: [#{config[:topic_names].join(', ')}], Queue: [#{config[:queue_name]}]" # rubocop:disable Metrics/LineLength
         self.topics = config[:topic_names].map {|tn| Topic.new(tn) }
         self.queue = Queue.new(config[:queue_name])
-        self.subscriptions = topics.map {|t| Subscription.new(t, queue) }
+        self.subscriptions = Subscription.new(topics, queue)
       end
 
       def default_queue_name
         "#{name.underscore}_queue"
       end
 
-      def worker_config(topics: [], max_batch_size: 10, max_wait: 10)
+      def dead_letter_queue_name
+        "#{config[:queue_name]}-dead-letter"
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def worker_config(topics: [],
+                        max_batch_size: 10,
+                        max_wait: 10,
+                        threads: Osbourne.threads_per_worker,
+                        queue_name: default_queue_name,
+                        dead_letter_queue: true,
+                        max_retry_count: Osbourne.max_retry_count)
         self.config = {
-          topic_names:    Array(topics),
-          queue_name:     queue_name,
-          max_batch_size: max_batch_size,
-          max_wait:       max_wait
+          topic_names:     Array(topics),
+          queue_name:      queue_name,
+          max_batch_size:  max_batch_size,
+          max_wait:        max_wait,
+          threads:         threads,
+          dead_letter:     dead_letter_queue,
+          max_retry_count: max_retry_count
         }
       end
+      # rubocop:enable Metrics/ParameterLists
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osbourne
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Steve Allen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-10 00:00:00.000000000 Z
+date: 2018-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core