osbourne 0.1.3

data/lib/osbourne/runner.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+$stdout.sync = true
+
+require "singleton"
+
+require "osbourne"
+
+module Osbourne
+  class Shutdown < Interrupt; end # rubocop:disable Lint/InheritException
+
+  class Runner
+    include Singleton
+
+    # rubocop:disable Metrics/MethodLength
+    def run
+      self_read, self_write = IO.pipe
+
+      %w[INT TERM USR1 TSTP TTIN].each do |sig|
+        begin
+          trap sig do
+            self_write.puts(sig)
+          end
+        rescue ArgumentError
+          puts "Signal #{sig} not supported"
+        end
+      end
+
+      @launcher = Osbourne::Launcher.new
+
+      begin
+        @launcher.start!
+
+        while readable_io = IO.select([self_read]) # rubocop:disable Lint/AssignmentInCondition
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+        @launcher.threads.map(&:join)
+      rescue Interrupt
+        @launcher.stop!
+        exit 0
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    private
+
+    def execute_soft_shutdown
+      Osbourne.logger.info { "Received USR1, will soft shutdown down" }
+
+      @launcher.stop
+      exit 0
+    end
+
+    def execute_terminal_stop
+      Osbourne.logger.info { "Received TSTP, will stop accepting new work" }
+
+      @launcher.stop!
+    end
+
+    def print_threads_backtrace
+      Thread.list.each do |thread|
+        Osbourne.logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
+        if thread.backtrace
+          Osbourne.logger.info { thread.backtrace.join("\n") }
+        else
+          Osbourne.logger.info { "<no backtrace available>" }
+        end
+      end
+    end
+
+    def handle_signal(sig)
+      Osbourne.logger.debug "Got #{sig} signal"
+
+      case sig
+      when "USR1" then execute_soft_shutdown
+      when "TTIN" then print_threads_backtrace
+      when "TSTP" then execute_terminal_stop
+      when "TERM", "INT"
+        Osbourne.logger.info { "Received #{sig}, will shutdown" }
+
+        raise Interrupt
+      end
+    end
+  end
+end

data/lib/osbourne/services/queue_provisioner.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Osbourne
+  module Services
+    module QueueProvisioner
+      def provision_worker_queues
+        Osbourne.logger.info "Workers found: #{Osbourne::WorkerBase.descendants.map(&:name).join(', ')}"
+        Osbourne.logger.info "Provisioning queues for all workers"
+
+        Osbourne::WorkerBase.descendants.each(&:provision)
+      end
+    end
+  end
+end

data/lib/osbourne/services/sns.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "aws-sdk-sns"
+
+module Osbourne
+  module Services
+    module SNS
+      def sns
+        @sns ||= Osbourne.sns_client
+      end
+
+      def sns=(client)
+        Osbourne.sns_client = client
+      end
+    end
+  end
+end

data/lib/osbourne/services/sqs.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "aws-sdk-sqs"
+
+module Osbourne
+  module Services
+    module SQS
+      def sqs
+        @sqs ||= Osbourne.sqs_client
+      end
+
+      def sqs=(client)
+        Osbourne.sqs_client = client
+      end
+    end
+  end
+end

data/lib/osbourne/subscription.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Osbourne
+  class Subscription
+    include Services::SNS
+    attr_reader :topic, :queue
+    def initialize(topic, queue)
+      @topic = topic
+      @queue = queue
+      arn
+    end
+
+    def arn
+      @arn ||= subscribe
+    end
+
+    private
+
+    def subscribe # rubocop:disable Metrics/AbcSize
+      Osbourne.logger.info("Checking subscription for #{queue.name} to #{topic.name}")
+      return if Osbourne.existing_subscriptions_for(topic).include? queue.arn
+
+      handled = Osbourne.lock.try_with_lock("osbourne_sub_lock_#{topic.name}") do
+        Osbourne.logger.info("Subscribing #{queue.name} to #{topic.name}")
+        @arn = sns.subscribe(topic_arn: topic.arn, protocol: "sqs", endpoint: queue.arn).subscription_arn
+        Osbourne.clear_subscriptions_for(topic)
+      end
+      if handled
+        @arn
+      else
+        sleep(3)
+        subscribe
+      end
+    end
+  end
+end

data/lib/osbourne/topic.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Osbourne
+  class Topic
+    include Services::SNS
+    attr_reader :name
+    def initialize(name)
+      @name = name
+      arn
+    end
+
+    def arn
+      @arn ||= ensure_topic
+    end
+
+    def publish(message)
+      Osbourne.logger.info "[PUB] TOPIC: `#{name}` MESSAGE: `#{message}`"
+      sns.publish(topic_arn: arn, message: message.is_a?(String) ? message : message.to_json)
+    end
+
+    private
+
+    def ensure_topic
+      Osbourne.logger.debug "Ensuring topic `#{name}` exists"
+      Osbourne.cache.fetch("osbourne_existing_topic_arn_for_#{name}") do
+        sns.create_topic(name: name).topic_arn
+      end
+    end
+
+    def subscriptions_cache_key
+      "existing_sqs_subscriptions_for_#{name}"
+    end
+  end
+end

data/lib/osbourne/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Osbourne
+  VERSION = "0.1.3"
+end

data/lib/osbourne/worker_base.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Osbourne
+  class WorkerBase
+    @config = {}
+
+    def config
+      self.class.config
+    end
+
+    def process(_message)
+      raise NotImplementedError, "#{self} must implement class method `process`"
+    end
+
+    def config=(config)
+      self.class.config = config
+    end
+
+    def queue
+      self.class.queue
+    end
+
+    def polling_queue
+      self.class.polling_queue
+    end
+
+    class << self
+      attr_accessor :config, :subscriptions, :topics, :queue
+
+      def descendants
+        ObjectSpace.each_object(Class).select {|klass| klass < self }
+      end
+
+      def provision
+        register
+        register_dead_letter_queue
+      end
+
+      def max_retry_count
+        Osbourne.max_retry_count
+      end
+
+      def dead_letter_queue_name
+        "#{config[:queue_name]}-dead-letter"
+      end
+
+      def dead_letter_queue
+        @dead_letter_queue ||= Queue.new(dead_letter_queue_name)
+      end
+
+      def queue_name
+        default_queue_name
+      end
+
+      def polling_queue
+        @polling_queue ||= Aws::SQS::Queue.new(queue.url, client: Osbourne.sqs_client)
+      end
+    end
+
+    class << self
+      private
+
+      def register_dead_letter_queue
+        return unless Osbourne.dead_letter
+
+        Osbourne.logger.info "#{self.class.name} dead letter queue: arn: [#{dead_letter_queue.arn}], max retries: #{max_retry_count}" # rubocop:disable Metrics/LineLength
+        queue.redrive(max_retry_count, dead_letter_queue.arn)
+      end
+
+      def register
+        Osbourne.logger.info "#{self.class.name} subscriptions: Topics: [#{config[:topic_names].join(', ')}], Queue: [#{config[:queue_name]}]" # rubocop:disable Metrics/LineLength
+        self.topics = config[:topic_names].map {|tn| Topic.new(tn) }
+        self.queue = Queue.new(config[:queue_name])
+        self.subscriptions = topics.map {|t| Subscription.new(t, queue) }
+      end
+
+      def default_queue_name
+        "#{name.underscore}_queue"
+      end
+
+      def worker_config(topics: [], max_batch_size: 10, max_wait: 10)
+        self.config = {
+          topic_names:    Array(topics),
+          queue_name:     queue_name,
+          max_batch_size: max_batch_size,
+          max_wait:       max_wait
+        }
+      end
+    end
+  end
+end

data/lib/tasks/message_plumber_tasks.rake

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# desc "Explaining what the task does"
+# task :osbourne do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,348 @@
+--- !ruby/object:Gem::Specification
+name: osbourne
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- Steve Allen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-10-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sns
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: guard-bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+- !ruby/object:Gem::Dependency
+  name: mock_redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.16'
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+description: |
+  This is a simple implementation of the fan-out pubsub pattern for Rails, using SQS & SNS as the message broker.
+  Includes a generator & runner for workers, as well as built-in provisioning for topics, subscriptions,
+  qeues, and dead-letter queues
+email:
+- sallen@amberstyle.ca
+executables:
+- osbourne
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- Rakefile
+- bin/cli/base.rb
+- bin/osbourne
+- lib/generators/osbourne/install/install_generator.rb
+- lib/generators/osbourne/install/templates/osbourne_initializer_template.template
+- lib/generators/osbourne/install/templates/osbourne_yaml_template.template
+- lib/generators/osbourne/worker/USAGE
+- lib/generators/osbourne/worker/templates/worker_template.template
+- lib/generators/osbourne/worker/worker_generator.rb
+- lib/osbourne.rb
+- lib/osbourne/config/file_loader.rb
+- lib/osbourne/config/shared_configs.rb
+- lib/osbourne/existing_subscriptions.rb
+- lib/osbourne/launcher.rb
+- lib/osbourne/locks/base.rb
+- lib/osbourne/locks/memory.rb
+- lib/osbourne/locks/noop.rb
+- lib/osbourne/locks/redis.rb
+- lib/osbourne/message.rb
+- lib/osbourne/poller.rb
+- lib/osbourne/queue.rb
+- lib/osbourne/railtie.rb
+- lib/osbourne/runner.rb
+- lib/osbourne/services/queue_provisioner.rb
+- lib/osbourne/services/sns.rb
+- lib/osbourne/services/sqs.rb
+- lib/osbourne/subscription.rb
+- lib/osbourne/topic.rb
+- lib/osbourne/version.rb
+- lib/osbourne/worker_base.rb
+- lib/tasks/message_plumber_tasks.rake
+homepage: https://github.com/stevenallen05/osbourne
+licenses:
+- GPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: A simple implementation of the fan-out pubsub pattern using SQS & SNS for
+  Rails
+test_files: []