cumulus-aws 0.11.1

data/lib/sns/SNS.rb

@@ -0,0 +1,40 @@
+require "conf/Configuration"
+
+require "aws-sdk"
+
+module Cumulus
+  module SNS
+    class << self
+      @@client = Aws::SNS::Client.new(Configuration.instance.client)
+
+      # Public: Static method that will get an SNS topic from AWS by its name
+      #
+      # name - the name of the topic to get
+      #
+      # Returns the Aws::SNS::Types::Topic
+      def get_aws(name)
+        topics.fetch(name)
+      rescue KeyError
+        puts "No SNS topic named #{name}"
+        exit
+      end
+
+      # Public: Provide a mapping of topics to their names.
+      # Lazily loads resources.
+      #
+      # Returns the topics mapped to their names
+      def topics
+        @topics ||= init_topics
+      end
+
+      private
+
+      # Internal: Load the topics and map them to their names
+      #
+      # Returns the topics mapped to their names
+      def init_topics
+        Hash[@@client.list_topics.topics.map { |t| [t.topic_arn[(t.topic_arn.rindex(":") + 1)..-1], t.topic_arn] }]
+      end
+    end
+  end
+end

data/lib/sqs/SQS.rb

@@ -0,0 +1,62 @@
+require "conf/Configuration"
+require "util/Colors"
+
+require "aws-sdk"
+
+module Cumulus
+  module SQS
+    class << self
+      @@client = Aws::SQS::Client.new(Configuration.instance.client)
+
+      # Public: Static method that will get the ARN of a Queue
+      #
+      # name - the name of the queue to get
+      #
+      # Returns the String ARN
+      def get_arn(queue_name)
+        queue_arns.fetch(queue_name)
+      rescue KeyError
+        puts Colors.red("No SQS queue named #{queue_name}")
+        exit 1
+      end
+
+      # Public: Returns a mapping of queue name to ARN
+      def queue_arns
+        @queue_arns ||= Hash[queue_attributes.map { |name, attrs| [name, attrs["QueueArn"]] }]
+      end
+
+      # Public: Return the policy of a queue as a Hash
+      def queue_policy(queue_name)
+        JSON.parse(URI.decode(queue_attributes[queue_name]["Policy"])) rescue nil
+      end
+
+      # Public: Return a mapping of queue name to url
+      def queue_urls
+        @queue_urls ||= init_urls
+      end
+
+      # Public: Return a mapping of queue name to attributes
+      def queue_attributes
+        @queue_attributes ||= Hash[queue_urls.map { |name, url| [name, init_attributes(url)] }]
+      end
+
+      private
+
+      # Internal: Returns a Hash of attribute names to values
+      def init_attributes(queue_url)
+        @@client.get_queue_attributes({
+          queue_url: queue_url,
+          attribute_names: ["All"]
+        }).attributes
+      rescue Aws::SQS::Errors::NonExistentQueue
+        puts "Queue #{queue_url} has been deleted. Please wait a few seconds until AWS updates the list of queues"
+        exit 1
+      end
+
+      # Internal: Map queue names to their urls.
+      def init_urls
+        Hash[@@client.list_queues.queue_urls.map { |u| [u[(u.rindex("/") + 1)..-1], u] }]
+      end
+    end
+  end
+end

data/lib/sqs/loader/Loader.rb

@@ -0,0 +1,34 @@
+require "common/BaseLoader"
+require "conf/Configuration"
+require "sqs/models/QueueConfig"
+
+require "aws-sdk"
+
+# Public: Load SQS assets
+module Cumulus
+  module SQS
+    module Loader
+      include Common::BaseLoader
+
+      @@queues_dir = Configuration.instance.sqs.queues_directory
+      @@policies_dir = Configuration.instance.sqs.policies_directory
+
+      # Public: Load all the queue configurations as QueueConfig objects
+      #
+      # Returns an array of QueueConfig
+      def self.queues
+        Common::BaseLoader::resources(@@queues_dir, &QueueConfig.method(:new))
+      end
+
+      # Public: Load the specified policy as a JSON object
+      #
+      # Returns the JSON object for the policy
+      def self.policy(policy_name)
+        Common::BaseLoader::resource(policy_name, @@policies_dir) do |policy_name, policy|
+          policy
+        end
+      end
+
+    end
+  end
+end

data/lib/sqs/manager/Manager.rb

@@ -0,0 +1,128 @@
+require "common/manager/Manager"
+require "conf/Configuration"
+require "util/Colors"
+require "sqs/models/QueueConfig"
+require "sqs/models/QueueDiff"
+require "sqs/SQS"
+
+require "aws-sdk"
+require "json"
+
+module Cumulus
+  module SQS
+    class Manager < Common::Manager
+
+      def initialize
+        super()
+        @create_asset = true
+        @client = Aws::SQS::Client.new(Configuration.instance.client)
+      end
+
+      def resource_name
+        "Queue"
+      end
+
+      def local_resources
+        @local_resources ||= Hash[Loader.queues.map { |local| [local.name, local] }]
+      end
+
+      def aws_resources
+        @aws_resources ||= Hash[SQS::queue_attributes.map { |name, attrs| [name, QueueConfig.new(name).populate!(attrs) ] }]
+      end
+
+      def unmanaged_diff(aws)
+        QueueDiff.unmanaged(aws)
+      end
+
+      def added_diff(local)
+        QueueDiff.added(local)
+      end
+
+      def diff_resource(local, aws)
+        local.diff(aws)
+      end
+
+      def urls
+        local_resources.keys.sort.each do |name|
+          url = SQS::queue_urls[name] || "does not exist"
+          puts "#{name} => #{url}"
+        end
+      end
+
+      def update(local, diffs)
+        @client.set_queue_attributes({
+          queue_url: SQS::queue_urls[local.name],
+          attributes: {
+            "DelaySeconds" => if diffs.any? { |d| d.type == QueueChange::DELAY } then local.delay end,
+            "MaximumMessageSize" => if diffs.any? { |d| d.type == QueueChange::MESSAGE_SIZE } then local.max_message_size end,
+            "MessageRetentionPeriod" => if diffs.any? { |d| d.type == QueueChange::MESSAGE_RETENTION } then local.message_retention end,
+            "Policy" => if diffs.any? { |d| d.type == QueueChange::POLICY }
+              if local.policy then JSON.generate(Loader.policy(local.policy)) else "" end
+            end,
+            "ReceiveMessageWaitTimeSeconds" => if diffs.any? { |d| d.type == QueueChange::RECEIVE_WAIT } then local.receive_wait_time end,
+            "VisibilityTimeout" => if diffs.any? { |d| d.type == QueueChange::VISIBILITY } then local.visibility_timeout end,
+            "RedrivePolicy" => if diffs.any? { |d| d.type == QueueChange::DEAD }
+              if local.dead_letter then JSON.generate(local.dead_letter.to_aws) else "" end
+            end
+          }.reject { |k, v| v.nil? }
+        })
+      end
+
+      def create(local)
+        url = @client.create_queue({
+          queue_name: local.name,
+          attributes: {
+            "DelaySeconds" => "#{local.delay}",
+            "MaximumMessageSize" => "#{local.max_message_size}",
+            "MessageRetentionPeriod" => "#{local.message_retention}",
+            "Policy" => if local.policy then JSON.generate(Loader.policy(local.policy)) end,
+            "ReceiveMessageWaitTimeSeconds" => "#{local.receive_wait_time}",
+            "VisibilityTimeout" => "#{local.visibility_timeout}",
+            "RedrivePolicy" => if local.dead_letter then JSON.generate(local.dead_letter.to_aws) end
+          }.reject { |k, v| v.nil? }
+        }).queue_url
+        puts Colors.blue("Queue #{local.name} was created with url #{url}")
+      end
+
+      # Public: Migrates the existing AWS config to Cumulus
+      def migrate
+        # Create the directories
+        sqs_dir = "#{@migration_root}/sqs"
+        policies_dir = "#{sqs_dir}/policies"
+        queues_dir = "#{sqs_dir}/queues"
+
+        if !Dir.exists?(@migration_root)
+          Dir.mkdir(@migration_root)
+        end
+        if !Dir.exists?(sqs_dir)
+          Dir.mkdir(sqs_dir)
+        end
+        if !Dir.exists?(policies_dir)
+          Dir.mkdir(policies_dir)
+        end
+        if !Dir.exists?(queues_dir)
+          Dir.mkdir(queues_dir)
+        end
+
+        puts Colors.blue("Migrating queues to #{queues_dir}")
+        aws_resources.each do |name, config|
+          puts Colors.blue("Migrating queue #{name}")
+
+          # If there is a policy, then save it to the policies dir with the name of the queue
+          queue_policy = SQS::queue_policy(name)
+          if queue_policy
+            policy_json = JSON.pretty_generate(queue_policy)
+            policy_file = "#{policies_dir}/#{name}.json"
+            puts "Migrating policy to #{policy_file}"
+            File.open("#{policy_file}", "w") { |f| f.write(policy_json) }
+          end
+
+          json = JSON.pretty_generate(config.to_hash)
+          File.open("#{queues_dir}/#{name}.json", "w") { |f| f.write(json) }
+        end
+
+      end
+
+    end
+  end
+end

data/lib/sqs/models/DeadLetterConfig.rb

@@ -0,0 +1,70 @@
+require "sqs/models/DeadLetterDiff"
+
+require "json"
+
+module Cumulus
+  module SQS
+
+    # Public: An object representing configuration for a queue's dead letter options
+    class DeadLetterConfig
+      attr_reader :target
+      attr_reader :max_receives
+
+      # Public: Constructor
+      #
+      # json - a hash containing the JSON configuration for dead letter options
+      def initialize(json = nil)
+        if !json.nil?
+          @target = json["target"]
+          @max_receives = json["max-receives"]
+        end
+      end
+
+      def to_hash
+        {
+          "target" => @target,
+          "max-receives" => @max_receives,
+        }
+      end
+
+      def to_aws
+        {
+          "deadLetterTargetArn" => SQS::queue_arns[@target],
+          "maxReceiveCount" => @max_receives
+        }
+      end
+
+      # Public: Populate a config object with AWS configuration
+      #
+      # aws - the JSON string containing dead letter attributes in AWS
+      def populate!(aws)
+        attributes = JSON.parse(URI.decode(aws))
+
+        @target = SQS::queue_arns.key(attributes["deadLetterTargetArn"])
+        @max_receives = if attributes["maxReceiveCount"] then attributes["maxReceiveCount"].to_i end
+
+        self
+      end
+
+      # Public: Produce an array of differences between two DeadLetterConfig objects
+      #
+      # aws - the DeadLetterConfig object built from aws config
+      #
+      # Returns an array of the DeadLetterDiffs that were found
+      def diff(aws)
+        diffs = []
+
+        if @target != aws.target
+          diffs << DeadLetterDiff.new(DeadLetterChange::TARGET, aws.target, @target)
+        end
+
+        if @max_receives != aws.max_receives
+          diffs << DeadLetterDiff.new(DeadLetterChange::RECEIVE, aws.max_receives, @max_receives)
+        end
+
+        diffs
+      end
+
+    end
+  end
+end

data/lib/sqs/models/DeadLetterDiff.rb

@@ -0,0 +1,35 @@
+require "common/models/Diff"
+require "util/Colors"
+
+module Cumulus
+  module SQS
+    # Public: The types of changes that can be made to a dead letter config
+    module DeadLetterChange
+      include Common::DiffChange
+
+      TARGET = Common::DiffChange.next_change_id
+      RECEIVE = Common::DiffChange.next_change_id
+    end
+
+    # Public: Represents a single difference between local configuration and AWS configuration
+    class DeadLetterDiff < Common::Diff
+      include DeadLetterChange
+
+      def diff_string
+        asset = case @type
+        when TARGET
+          "Target"
+        when RECEIVE
+          "Max Receive Count"
+        end
+
+        [
+          "#{asset}:",
+          Colors.aws_changes("\tAWS - #{aws}"),
+          Colors.local_changes("\tLocal - #{local}")
+        ].join("\n")
+
+      end
+    end
+  end
+end

data/lib/sqs/models/QueueConfig.rb

@@ -0,0 +1,115 @@
+require "sqs/loader/Loader"
+require "sqs/models/DeadLetterConfig"
+require "sqs/models/QueueDiff"
+
+require "json"
+
+module Cumulus
+  module SQS
+
+    # Public: An object representing configuration for a queue
+    class QueueConfig
+      attr_reader :name
+      attr_reader :delay
+      attr_reader :max_message_size
+      attr_reader :message_retention
+      attr_reader :policy
+      attr_reader :receive_wait_time
+      attr_reader :visibility_timeout
+      attr_reader :dead_letter
+
+      # Public: Constructor
+      #
+      # json - a hash containing the JSON configuration for the queue
+      def initialize(name, json = nil)
+        @name = name
+        if !json.nil?
+          @delay = json["delay"]
+          @max_message_size = json["max-message-size"]
+          @message_retention = json["message-retention"]
+          @policy = json["policy"]
+          @receive_wait_time = json["receive-wait-time"]
+          @visibility_timeout = json["visibility-timeout"]
+          @dead_letter = if json["dead-letter"] then DeadLetterConfig.new(json["dead-letter"]) end
+        end
+      end
+
+      def to_hash
+        {
+          "delay" => @delay,
+          "max-message-size" => @max_message_size,
+          "message-retention" => @message_retention,
+          "policy" => @policy,
+          "receive-wait-time" => @receive_wait_time,
+          "visibility-timeout" => @visibility_timeout,
+          "dead-letter" => if @dead_letter then @dead_letter.to_hash end,
+        }.reject { |k, v| v.nil? }
+      end
+
+      # Public: Populate a config object with AWS configuration
+      #
+      # attributes - the queue attributes in AWS
+      def populate!(attributes)
+        @delay = if attributes["DelaySeconds"] then attributes["DelaySeconds"].to_i end
+        @max_message_size = if attributes["MaximumMessageSize"] then attributes["MaximumMessageSize"].to_i end
+        @message_retention = if attributes["MessageRetentionPeriod"] then attributes["MessageRetentionPeriod"].to_i end
+        @receive_wait_time = if attributes["ReceiveMessageWaitTimeSeconds"] then attributes["ReceiveMessageWaitTimeSeconds"].to_i end
+        @visibility_timeout = if attributes["VisibilityTimeout"] then attributes["VisibilityTimeout"].to_i end
+        @dead_letter = if attributes["RedrivePolicy"] then DeadLetterConfig.new().populate!(attributes["RedrivePolicy"]) end
+
+        # Policy is handled specially because we store them in a separate file locally.
+        # For migrating we assume the policy is the same as the queue name, otherwise this
+        # attribute is not used from AWS config
+        @policy = if attributes["Policy"] then @name end
+
+        self
+      end
+
+      # Public: Produce an array of differences between this local configuration and the
+      # configuration in AWS
+      #
+      # aws - the AWS resource
+      #
+      # Returns an array of the QueueDiffs that were found
+      def diff(aws)
+        diffs = []
+
+        if @delay != aws.delay
+          diffs << QueueDiff.new(QueueChange::DELAY, aws.delay, @delay)
+        end
+
+        if @max_message_size != aws.max_message_size
+          diffs << QueueDiff.new(QueueChange::MESSAGE_SIZE, aws.max_message_size, @max_message_size)
+        end
+
+        if @message_retention != aws.message_retention
+          diffs << QueueDiff.new(QueueChange::MESSAGE_RETENTION, aws.message_retention, @message_retention)
+        end
+
+        if @receive_wait_time != aws.receive_wait_time
+          diffs << QueueDiff.new(QueueChange::RECEIVE_WAIT, aws.receive_wait_time, @receive_wait_time)
+        end
+
+        if @visibility_timeout != aws.visibility_timeout
+          diffs << QueueDiff.new(QueueChange::VISIBILITY, aws.visibility_timeout, @visibility_timeout)
+        end
+
+        aws_dead_letter = aws.dead_letter || DeadLetterConfig.new()
+        local_dead_letter = @dead_letter || DeadLetterConfig.new()
+        dead_diffs = local_dead_letter.diff(aws_dead_letter)
+        if !dead_diffs.empty?
+          diffs << QueueDiff.new(QueueChange::DEAD, aws_dead_letter, local_dead_letter, dead_diffs)
+        end
+
+        aws_policy = SQS::queue_policy(@name)
+        local_policy = if @policy then Loader.policy(@policy) end
+        if local_policy != aws_policy
+          diffs << QueueDiff.new(QueueChange::POLICY, aws_policy, local_policy)
+        end
+
+        diffs
+      end
+
+    end
+  end
+end