osbourne 0.1.3

data/lib/osbourne/existing_subscriptions.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Osbourne
+  module ExistingSubscriptions
+    attr_reader :existing_subscriptions
+    def existing_subscriptions_for(topic)
+      Osbourne.cache.fetch("osbourne_existng_subs_for_#{topic.name}") do
+        results = []
+        handled = Osbourne.lock.try_with_lock("osbourne_lock_subs_for_#{topic.name}") do
+          results = fetch_existing_subscriptions_for(topic)
+        end
+        return results if handled
+
+        sleep(0.5)
+        existing_subscriptions_for(topic)
+      end
+      # @existing_subscriptions ||= {}
+      # @existing_subscriptions[topic.name] ||=
+    end
+
+    def clear_subscriptions_for(topic)
+      Osbourne.cache.delete("osbourne_existng_subs_for_#{topic.name}")
+    end
+
+    private
+
+    def fetch_existing_subscriptions_for(topic)
+      results = []
+      r = nil
+      loop do
+        params = {topic_arn: topic.arn}
+        params[:next_token] = r.next_token if r.try(:next_token)
+        r = Osbourne.sns_client.list_subscriptions_by_topic(params)
+        results << r.subscriptions.map(&:endpoint)
+        break unless r.try(:next_token).presence
+      end
+      results.flatten.uniq
+    end
+  end
+end

data/lib/osbourne/launcher.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "osbourne"
+require "byebug"
+
+module Osbourne
+  class Launcher
+    attr_accessor :threads
+    def initialize; end
+
+    def start!
+      Osbourne.logger.info("Launching Osbourne workers")
+      @stop = false
+      @threads = polling_threads
+      polling_threads.each(&:run)
+    end
+
+    def stop
+      @stop = true
+    end
+
+    def stop!
+      @threads.each {|thr| Thread.kill(thr) }
+    end
+
+    def polling_threads
+      Osbourne::WorkerBase.descendants.map do |worker|
+        Thread.new {
+          worker_instance = worker.new
+          loop do
+            poll(worker_instance)
+            sleep(Osbourne.sleep_time)
+            break if @stop
+          end
+        }
+      end
+    end
+
+    def poll(worker)
+      worker.polling_queue.receive_messages(
+        wait_time_seconds:      worker.config[:max_wait],
+        max_number_of_messages: worker.config[:max_batch_size]
+      ).each {|msg| process(worker, Osbourne::Message.new(msg)) }
+    end
+
+    private
+
+    def process(worker, message)
+      Osbourne.logger.info("[MSG] Worker: #{worker.class.name} Valid: #{message.valid?} ID: #{message.id} Body: #{message.raw_body}") # rubocop:disable Metrics/LineLength
+      return unless message.valid?
+
+      # hard_lock to prevent duplicate processing over the hard_lock lifespan
+      Osbourne.lock.try_with_lock(message.id, hard_lock: true) do
+        message.delete if worker.process(message)
+      end
+    rescue Exception => ex # rubocop:disable Lint/RescueException
+      Osbourne.logger.error("[MSG ID: #{message.id}] #{ex.message}")
+    end
+  end
+end

data/lib/osbourne/locks/base.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "osbourne"
+
+module Osbourne
+  module Locks
+    module Base
+      DEFAULT_SOFT_TTL = (5 * 60).freeze        # 5 minutes
+      DEFAULT_HARD_TTL = (24 * 60 * 60).freeze  # 24 hours
+
+      attr_reader :soft_ttl, :hard_ttl
+
+      def initialize(options={})
+        self.soft_ttl = options.fetch(:soft_ttl, DEFAULT_SOFT_TTL)
+        self.hard_ttl = options.fetch(:hard_ttl, DEFAULT_HARD_TTL)
+      end
+
+      def soft_lock(id)
+        lock(lock_key(id), soft_ttl)
+      end
+
+      def hard_lock(id)
+        lock!(lock_key(id), hard_ttl)
+      end
+
+      def unlock(id)
+        unlock!(lock_key(id))
+      end
+
+      def try_with_lock(id, hard_lock: false)
+        if soft_lock(id)
+          begin
+            yield
+          rescue => e # rubocop:disable Style/RescueStandardError
+            unlock(id)
+            raise e
+          end
+
+          hard_lock(id) if hard_lock
+          true
+        else
+          false
+        end
+      end
+
+      protected
+
+      def lock(_key, _ttl)
+        raise NotImplementedError
+      end
+
+      def lock!(_key, _ttl)
+        raise NotImplementedError
+      end
+
+      def unlock!(_key)
+        raise NotImplementedError
+      end
+
+      private
+
+      attr_writer :soft_ttl, :hard_ttl
+
+      def lock_key(id)
+        "osbourne:lock:#{id}"
+      end
+    end
+  end
+end

data/lib/osbourne/locks/memory.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "osbourne"
+
+module Osbourne
+  module Locks
+    class Memory
+      include Base
+
+      class << self
+        def store
+          @store ||= {}
+        end
+
+        def semaphore
+          @semaphore ||= Mutex.new
+        end
+      end
+
+      protected
+
+      def lock(key, ttl)
+        reap
+
+        store do |store|
+          if store.key?(key)
+            false
+          else
+            store[key] = Time.current + ttl
+            true
+          end
+        end
+      end
+
+      def lock!(key, ttl)
+        reap
+
+        store do |store|
+          store[key] = Time.current + ttl
+        end
+      end
+
+      def unlock!(key)
+        store do |store|
+          store.delete(key)
+        end
+      end
+
+      private
+
+      def store
+        semaphore.synchronize do
+          yield self.class.store
+        end
+      end
+
+      def reap
+        store do |store|
+          now = Time.current
+          store.delete_if {|_, expires_at| expires_at <= now }
+        end
+      end
+
+      def semaphore
+        self.class.semaphore
+      end
+    end
+  end
+end

data/lib/osbourne/locks/noop.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "osbourne"
+
+module Osbourne
+  module Locks
+    class NOOP
+      include Base
+
+      protected
+
+      def lock(_key, _ttl)
+        true
+      end
+
+      def lock!(key, ttl)
+        # do nothing
+      end
+
+      def unlock!(key)
+        # do nothing
+      end
+    end
+  end
+end

data/lib/osbourne/locks/redis.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "osbourne"
+
+module Osbourne
+  module Locks
+    class Redis
+      include Base
+
+      def initialize(options={})
+        super(options)
+
+        self.client = options.fetch(:client) do
+          require "redis"
+          ::Redis.new(options)
+        end
+      end
+
+      protected
+
+      def lock(key, ttl)
+        with_pool do |client|
+          client.set(key, (Time.current + ttl).to_i, ex: ttl, nx: true)
+        end
+      end
+
+      def lock!(key, ttl)
+        with_pool do |client|
+          client.set(key, (Time.current + ttl).to_i, ex: ttl)
+        end
+      end
+
+      def unlock!(key)
+        with_pool do |client|
+          client.del(key)
+        end
+      end
+
+      private
+
+      attr_accessor :client
+
+      def with_pool(&block)
+        if pool?
+          client.with(&block)
+        else
+          yield client
+        end
+      end
+
+      def pool?
+        defined?(ConnectionPool) && client.is_a?(ConnectionPool)
+      end
+    end
+  end
+end

data/lib/osbourne/message.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "osbourne"
+
+module Osbourne
+  class Message
+    attr_accessor :message
+    def initialize(message)
+      @message = message
+    end
+
+    def json?
+      !JSON.parse(parsed_content["Message"]).nil?
+    rescue JSON::ParserError
+      false
+    end
+
+    def valid?
+      message.md5_of_body == Digest::MD5.hexdigest(message.body)
+    end
+
+    def id
+      message.message_id
+    end
+
+    def parsed_body
+      JSON.parse(parsed_content["Message"])
+    rescue JSON::ParserError
+      parsed_content["Message"]
+    end
+
+    def delete
+      message.delete
+      Osbourne.logger.info "[MSG ID: #{id}] Cleared"
+    end
+
+    def topic
+      parsed_content["TopicArn"].split(":").last
+    end
+
+    def raw_body
+      message.body
+    end
+
+    private
+
+    def parsed_content
+      @parsed_content ||= JSON.parse(message.body)
+    end
+
+    def body
+      parsed_content["Message"]
+    end
+  end
+end

data/lib/osbourne/queue.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Osbourne
+  class Queue
+    include Services::SQS
+    attr_reader :name
+    def initialize(name)
+      @name = name
+      arn
+    end
+
+    def url
+      @url ||= ensure_queue
+    end
+
+    def arn
+      @arn ||= get_attributes["QueueArn"]
+    end
+
+    def redrive(retries, dead_letter_arn)
+      sqs.set_queue_attributes(queue_url:  url,
+                               attributes: {
+                                 'RedrivePolicy': {
+                                   'deadLetterTargetArn': dead_letter_arn,
+                                   'maxReceiveCount':     retries
+                                 }.to_json
+                               })
+    end
+
+    private
+
+    def ensure_queue
+      Osbourne.logger.debug "Ensuring queue `#{name}` exists"
+      Osbourne.cache.fetch("osbourne_url_for_#{name}") do
+        sqs.create_queue(queue_name: name).queue_url
+      end
+    end
+
+    def get_attributes(attrs: %w[QueueArn])
+      sqs.get_queue_attributes(queue_url: url, attribute_names: attrs).attributes
+    end
+  end
+end

data/lib/osbourne/railtie.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "rails"
+require "osbourne/config/file_loader"
+
+module Osbourne
+  class Railtie < Rails::Railtie
+    config.osbourne = ActiveSupport::OrderedOptions.new
+
+    initializer "osbourne.configure", after: :load_config_initializers do |_app|
+      env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+
+      %w[config/osbourne.yml.erb config/osbourne.yml].find do |filename|
+        Osbourne::Config::FileLoader.load(filename, env)
+      end
+
+      Osbourne.provision_worker_queues
+    end
+  end
+end