aws_sqs_moniter 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35611d2ac97dd5a262318bc8e10fa7d77766ba27
-  data.tar.gz: 68b6834cef8d624d31e8f820234bc292dae9da75
+  metadata.gz: a99854dd21c0a9bce3dfd51f2608978d51154680
+  data.tar.gz: 1a760d80e4ce9e706ae510319898d226676102d4
 SHA512:
-  metadata.gz: cd483589e313e71114ed14c9e560b7c460cd852d5bbb3207bc1e11523924c8e9ca69f7de16e860fff0b682330ee3b94c8db81c6221e423bcb1455cb036f7438e
-  data.tar.gz: 14e42ca9c58c54b15c3874a2e056a98609705c84c570560d46acef7130547cc277d0d6d8c259c660a195b080d94bda880a0d9035c574458e57f7effdccd344ac
+  metadata.gz: f8b0dbc36da0ce368f5701381756ad76c93187c70b0f1875a89e9a4fb1ccb4338ab156e74eab11a306c17e011339c6cb3d997774905e272c6b35e9f8703d5905
+  data.tar.gz: d5dbc5cd2f0baec5d88dd3423da62c0d0650c384760478dfe30f06add657772b75b09eac027c1242f402d288b0675c2f2b1ab8176b5745a15126585d3ccf584c

data/lib/aws_sqs_moniter.rb

@@ -1,5 +1,29 @@
-require "aws_sqs_moniter/version"
+require 'shoryuken'
+require 'aws_sqs_moniter/version'
+require 'aws_sqs_moniter/logging'
+require 'aws_sqs_moniter/configuration'
+require 'aws_sqs_moniter/railtie' if defined? Rails
 
 module AwsSqsMoniter
-  # Your code goes here...
+  class << self
+    def configure
+      @config ||= Configuration.new.tap do |config|
+        yield config
+        config.validate!
+      end
+    end
+
+    def configuration
+      fail 'You must call AwsSqsMoniter .configure in an initializer.' unless @config
+      @config
+    end
+
+    def sns_client
+      ::Aws::SNS::Client.new configuration.sns_options
+    end
+
+    def sqs_client
+      ::Aws::SQS::Client.new configuration.sqs_options
+    end
+  end
 end

data/lib/aws_sqs_moniter/aws/arns.rb

@@ -0,0 +1,29 @@
+module AwsSqsMoniter
+  module Aws
+    class Arns
+      def sns_arn
+        if AwsSqsMoniter.configuration.sns_protocol == 'cqs'
+          "arn:cmb:cns:ccp:#{AwsSqsMoniter.configuration.aws_account_id}"
+        else
+          "arn:aws:sns:#{AwsSqsMoniter.configuration.aws_region}:#{AwsSqsMoniter.configuration.aws_account_id}"
+        end
+      end
+
+      def sns_topic_arn topic
+        "#{sns_arn}:#{topic}"
+      end
+
+      def sqs_arn
+        if AwsSqsMoniter.configuration.sqs_protocol == 'cqs'
+          "arn:cmb:cqs:ccp:#{AwsSqsMoniter.configuration.aws_account_id}"
+        else
+          "arn:aws:sqs:#{AwsSqsMoniter.configuration.aws_region}:#{AwsSqsMoniter.configuration.aws_account_id}"
+        end
+      end
+
+      def sqs_queue_arn queue
+        "#{sqs_arn}:#{queue}"
+      end
+    end
+  end
+end

data/lib/aws_sqs_moniter/aws/builder.rb

@@ -0,0 +1,48 @@
+require_relative 'arns'
+require_relative 'environmental_name'
+require_relative 'builder/application_policy_builder'
+require_relative 'builder/queue_builder'
+require_relative 'builder/subscription_builder'
+require_relative 'builder/topic_builder'
+
+module AwsSqsMoniter
+  module Aws
+    class Builder
+      def initialize logger = nil
+        @logger = logger || Shoryuken::Logging.logger
+      end
+
+      def build_policies
+        ApplicationPolicyBuilder.new(@logger).build
+      end
+
+      def build_queues
+        Shoryuken.worker_registry.queues.each do |queue|
+          QueueBuilder.new(@logger, queue).build
+        end
+      end
+
+      def build_topics
+        Shoryuken.worker_registry.topics.each do |topic|
+          TopicBuilder.new(@logger, topic).build
+        end
+      end
+
+      def build_subscriptions
+        Shoryuken.worker_registry.queues.each do |queue|
+          SubscriptionBuilder.new(@logger, queue).build
+        end
+      end
+
+      def delete_all
+        Shoryuken.worker_registry.queues.each do |queue|
+          QueueBuilder.new(@logger, queue).delete
+        end
+
+        Shoryuken.worker_registry.topics.each do |topics|
+          TopicBuilder.new(@logger, topics).delete
+        end
+      end
+    end
+  end
+end

data/lib/aws_sqs_moniter/aws/builder/application_policy_builder.rb

@@ -0,0 +1,73 @@
+module AwsSqsMoniter
+  module Aws
+    class Builder
+      class ApplicationPolicyBuilder
+        def initialize logger
+          @logger = logger
+          @arns = Arns.new
+        end
+
+        def build
+          file = Tempfile.create ['aws-application-policy', '.json']
+          file.write application_policy
+          @logger.info application_policy_written_to: file.path
+        end
+
+        private
+
+        def application_policy
+          topic_arns = Shoryuken.worker_registry.topics.map { |topic| @arns.sns_topic_arn topic }
+          queue_arns = Shoryuken.worker_registry.queues.map { |queue| @arns.sqs_queue_arn queue }
+
+          <<-EOS
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sns:CreateTopic",
+        "sns:Publish",
+        "sns:SetEndpointAttributes",
+        "sns:Subscribe"
+      ],
+      "Resource": [
+#{arn_array_policy_string topic_arns, '        '}
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sns:SetSubscriptionAttributes"
+      ],
+      "Resource": [
+        "#{@arns.sns_topic_arn '*'}"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "sqs:ChangeMessageVisibility",
+        "sqs:CreateQueue",
+        "sqs:DeleteMessage",
+        "sqs:GetQueueAttributes",
+        "sqs:GetQueueUrl",
+        "sqs:ReceiveMessage",
+        "sqs:SetQueueAttributes"
+      ],
+      "Resource": [
+#{arn_array_policy_string queue_arns, '        '}
+      ]
+    }
+  ]
+}
+EOS
+        end
+
+        def arn_array_policy_string arns, indent
+          arns.sort.map { |arn| %(#{indent}"#{arn}") }.join(",\n")
+        end
+      end
+    end
+  end
+end

data/lib/aws_sqs_moniter/aws/builder/queue_builder.rb

@@ -0,0 +1,100 @@
+module AwsSqsMoniter
+  module Aws
+    class Builder
+      class QueueBuilder
+        def initialize logger, name
+          @logger = logger
+          @name = name
+          @arns = Arns.new
+          @sqs = AwsSqsMoniter.sqs_client
+        end
+
+        def build
+          @logger.info building_queue: @name
+
+          queue_url = @sqs.create_queue(queue_name: @name)[:queue_url]
+
+          queue_arn = @sqs.get_queue_attributes(
+            queue_url: queue_url,
+            attribute_names: ['QueueArn']).attributes['QueueArn']
+
+          queue_configuration = AwsSqsMoniter.configuration.get_queue(@name)
+
+          attributes = {
+            'DelaySeconds' => queue_configuration.delay_seconds.to_s,
+            'MessageRetentionPeriod' => queue_configuration.message_retention_period.to_s,
+            'VisibilityTimeout' => queue_configuration.visibility_timeout.to_s
+          }
+
+          if AwsSqsMoniter.configuration.sqs_protocol == 'sqs'
+            if queue_configuration.respond_to?(:redrive_policy)
+              policy = '{}'
+              policy = redrive_policy(
+                queue_configuration.redrive_policy.dead_letter_queue,
+                queue_configuration.redrive_policy.max_receive_count) if queue_configuration.redrive_policy.enabled
+
+              attributes.merge! 'RedrivePolicy' => policy
+            end
+
+            policy = queue_policy
+            attributes.merge! 'Policy' => policy
+          end
+
+          @sqs.set_queue_attributes queue_url: queue_url, attributes: attributes
+        end
+
+        def delete
+          @logger.info deleting_queue: @name
+
+          queue_url = @sqs.get_queue_url(queue_name: @name)[:queue_url]
+          @sqs.delete_queue(queue_url: queue_url)
+        rescue ::Aws::SQS::Errors::NonExistentQueue
+        end
+
+        private
+
+        def arn_array_policy_string arns, indent
+          arns.sort.map { |arn| %(#{indent}"#{arn}") }.join(",\n")
+        end
+
+        def queue_policy
+          queue_arn = @arns.sqs_queue_arn @name
+          topic_arns = topics.map { |topic| @arns.sns_topic_arn topic }
+
+          <<-EOS
+{
+  "Version": "2008-10-17",
+  "Id": "#{queue_arn}/envoy-generated-policy",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "*"
+      },
+      "Action": "SQS:SendMessage",
+      "Resource": "#{queue_arn}",
+      "Condition": {
+        "ArnEquals": {
+          "aws:SourceArn": [
+#{arn_array_policy_string topic_arns, '            '}
+          ]
+        }
+      }
+    }
+  ]
+}
+EOS
+        end
+
+        def redrive_policy dead_letter_queue, max_receive_count
+          arn = @arns.sqs_queue_arn EnvironmentalName.new(dead_letter_queue).to_s
+          %({"maxReceiveCount":"#{max_receive_count}", "deadLetterTargetArn":"#{arn}"})
+        end
+
+        def topics
+          Shoryuken.worker_registry.topics(@name)
+        end
+      end
+    end
+  end
+end

data/lib/aws_sqs_moniter/aws/builder/subscription_builder.rb

@@ -0,0 +1,48 @@
+module AwsSqsMoniter
+  module Aws
+    class Builder
+      class SubscriptionBuilder
+        def initialize logger, queue
+          @logger = logger
+          @queue = queue
+          @arns = Arns.new
+          @sns = AwsSqsMoniter.sns_client
+          @sqs = AwsSqsMoniter.sqs_client
+        end
+
+        def build
+          queue_url = @sqs.get_queue_url(queue_name: @queue)[:queue_url]
+
+          queue_arn = @sqs.get_queue_attributes(
+            queue_url: queue_url,
+            attribute_names: ['QueueArn']).attributes['QueueArn']
+
+          topics.each do |topic|
+            topic = EnvironmentalName.new(topic).to_s
+            topic_arn = @arns.sns_topic_arn topic
+
+            if AwsSqsMoniter.configuration.sns_protocol == 'cqs'
+              @sns.add_permission topic_arn: topic_arn,
+                                  label: "subscribe-#{AwsSqsMoniter.configuration.aws_account_id}-#{Time.now.strftime('%Y%m%d%H%M%S')}",
+                                  aws_account_id: [AwsSqsMoniter.configuration.aws_account_id],
+                                  action_name: ['Subscribe']
+            end
+
+            @logger.info subscribing_queue: @queue, subscription_topic: topic
+
+            @sns.subscribe(
+              endpoint: queue_arn,
+              protocol: AwsSqsMoniter.configuration.sns_protocol,
+              topic_arn: topic_arn)
+          end
+        end
+
+        private
+
+        def topics
+          Shoryuken.worker_registry.topics(@queue)
+        end
+      end
+    end
+  end
+end

data/lib/aws_sqs_moniter/aws/builder/topic_builder.rb

@@ -0,0 +1,27 @@
+module AwsSqsMoniter
+  module Aws
+    class Builder
+      class TopicBuilder
+        def initialize logger, name
+          @logger = logger
+          @name = EnvironmentalName.new(name).to_s
+          @sns = AwsSqsMoniter.sns_client
+          @arns = Arns.new
+        end
+
+        def build
+          @logger.info building_topic: @name
+
+          @sns.create_topic name: @name
+        end
+
+        def delete
+          @logger.info deleting_topic: @name
+
+          @sns.delete_topic topic_arn: @arns.sns_topic_arn(@name)
+        rescue ::Aws::SNS::Errors::NotFound
+        end
+      end
+    end
+  end
+end

data/lib/aws_sqs_moniter/aws/environmental_name.rb

@@ -0,0 +1,13 @@
+module AwsSqsMoniter
+  module Aws
+    class EnvironmentalName
+      def initialize(name)
+        @name = [name.to_s.dasherize, Rails.env].join('-')
+      end
+
+      def to_s
+        @name
+      end
+    end
+  end
+end

data/lib/aws_sqs_moniter/configuration.rb

@@ -0,0 +1,110 @@
+require_relative 'configuration/validatable'
+require_relative 'configuration/queue_configuration'
+require_relative 'configuration/redrive_policy_configuration'
+
+module AwsSqsMoniter
+  class Configuration
+    include Validatable
+
+    def initialize
+      @queues = {}
+      @queue_defaults = QueueConfiguration.new 'queue_defaults'
+      @sns_protocol = @sqs_protocol = 'sqs'
+    end
+
+    attr_reader :queue_defaults, :sns_endpoint, :sns_protocol, :sqs_endpoint, :sqs_protocol
+
+    def add_queue name
+      environmental_name = Aws::EnvironmentalName.new(name).to_s
+      fail "A queue called #{name} already exists." if @queues.key? environmental_name
+
+      queue = QueueConfiguration.new name
+      @queue_defaults.copy_onto queue
+      yield queue if block_given?
+      @queues[environmental_name] = queue
+    end
+
+    def aws_options
+      @aws_options ||= shoryuken_config[:aws].slice :access_key_id, :region, :secret_access_key
+    end
+
+    def aws_account_id
+      shoryuken_config[:aws][:account_id]
+    end
+
+    def aws_region
+      aws_options[:region]
+    end
+
+    def enable_cmb_mode
+      @sns_protocol = @sqs_protocol = 'cqs'
+      @sns_endpoint = shoryuken_config[:aws][:sns_endpoint]
+      @sqs_endpoint = shoryuken_config[:aws][:sqs_endpoint]
+    end
+
+    def queues
+      @queues.values
+    end
+
+    def get_queue environmental_name
+      @queues[environmental_name]
+    end
+
+    def require_workers
+      Dir[Rails.root + 'app' + 'workers' + '*.rb'].each do |path|
+        require path
+      end
+    end
+
+    def shoryuken_config
+      @shoryuken_config ||= begin
+        YAML.load(
+          ERB.new(
+            IO.read(
+              Rails.root + 'config' + 'shoryuken.yml')
+            ).result
+          ).with_indifferent_access
+      end
+    end
+
+    def sns_options
+      aws_client_options :sns_endpoint
+    end
+
+    def sqs_options
+      aws_client_options :sqs_endpoint
+    end
+
+    def use_aws_sqs_moniter_logging
+      require 'aws_sqs_moniter/logging'
+      Shoryuken::Logging.logger = AwsSqsMoniter::Logging.logger
+    end
+
+    def use_typed_message_registry dead_letter_queue_name
+      require 'aws_sqs_moniter/worker_registries/typed_message_registry'
+      Shoryuken.worker_registry = AwsSqsMoniter::WorkerRegistries::TypedMessageRegistry.new dead_letter_queue_name
+    end
+
+    def validate
+      [queue_defaults, queues].flatten.compact.each do |section|
+        section.valid?
+        errors.push *(section.errors)
+      end
+    end
+
+    def validate!
+      return if valid?
+
+      fail %(Invalid configuration: \n\n#{errors.join("\n")}\n\n)
+    end
+
+    private
+
+    def aws_client_options endpoint_key
+      options = aws_options
+      endpoint = shoryuken_config[:aws][endpoint_key]
+      options = options.merge(endpoint: endpoint) unless endpoint.blank?
+      options
+    end
+  end
+end