lucid-cumulus 0.11.2 → 0.11.3

data/spec/mocks/MockedLoader.rb

@@ -0,0 +1,37 @@
+module Cumulus
+  module Test
+    module MockedLoader
+      # keep a map of the paths we've stubbed to the arrays of json they
+      # should return
+      @@stubbed_directories = {}
+
+      # keep a map of the files we've stubbed to the contents of those
+      # files
+      @@stubbed_files = {}
+
+      def self.included(base)
+        base.instance_eval do
+          def stub_directory(path, json)
+            @@stubbed_directories[path] = json
+          end
+
+          def stub_file(path, json)
+            @@stubbed_files[path] = json
+          end
+
+          def resources(dir, json = true, &individual_loader)
+            @@stubbed_directories[dir].map do |json|
+              individual_loader.call(json[:name], json[:value])
+            end
+          end
+
+          def resource(file, dir, json = true, &loader)
+            path = File.join(dir, file)
+            contents = @@stubbed_files[path]
+            loader.call(path, if json then JSON.parse(contents) else contents end)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/mocks/MockedStatusCodes.rb

@@ -0,0 +1,17 @@
+require "util/StatusCodes"
+
+module Cumulus
+  module Test
+    module MockedStatusCodes
+      def self.included(base)
+        base.instance_eval do
+          def set_status(status)
+            if status == StatusCodes::EXCEPTION
+              @@CURRENT_STATUS = StatusCodes::EXCEPTION
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/rspec_config.rb

@@ -0,0 +1,5 @@
+RSpec.configure do |c|
+  c.fail_fast = true
+  c.color = true
+  c.formatter = "documentation"
+end

data/spec/sqs/SQSUtil.rb

@@ -0,0 +1,231 @@
+require "conf/Configuration"
+require "mocks/MockedConfiguration"
+Cumulus::Configuration.send :include, Cumulus::Test::MockedConfiguration
+
+require "common/BaseLoader"
+require "mocks/MockedLoader"
+Cumulus::Common::BaseLoader.send :include, Cumulus::Test::MockedLoader
+
+require "common/manager/Manager"
+require "util/ManagerUtil"
+Cumulus::Common::Manager.send :include, Cumulus::Test::ManagerUtil
+
+require "util/StatusCodes"
+require "mocks/MockedStatusCodes"
+Cumulus::StatusCodes.send :include, Cumulus::Test::MockedStatusCodes
+
+require "aws-sdk"
+require "json"
+require "sqs/manager/Manager"
+require "sqs/SQS"
+require "util/DeepMerge"
+
+module Cumulus
+  module Test
+    # Monkey patch Cumulus::SQS such that the cached values from the AWS client
+    # can be reset between tests.
+    module ResetSQS
+      def self.included(base)
+        base.instance_eval do
+          def reset_queue_urls
+            @queue_urls = nil
+          end
+
+          def reset_queue_attributes
+            @queue_attributes = nil
+          end
+
+          def client=(client)
+            @@client = client
+          end
+        end
+      end
+    end
+
+    # Monkey patch Cumulus::SQS such that the queue_arns contain the ARN for the
+    # default dead letter target
+    module DeadLetterArn
+      def self.included(base)
+        base.instance_eval do
+          singleton_class.send(:alias_method, :original_queue_arns, :queue_arns)
+
+          def queue_arns
+            arns = original_queue_arns
+            arns[SQS::DEFAULT_DEAD_LETTER_TARGET] = SQS::DEFAULT_DEAD_LETTER_TARGET
+            arns[SQS::DEFAULT_DEAD_LETTER_TARGET + "a"] = SQS::DEFAULT_DEAD_LETTER_TARGET + "a"
+            arns
+          end
+        end
+      end
+    end
+
+    module SQS
+      @queue_directory = "/mocked/sqs/queues"
+      @policy_directory = "/mocked/sqs/policies"
+      @default_policy_name = "example-policy"
+
+      DEFAULT_QUEUE_DELAY = 0
+      DEFAULT_QUEUE_MESSAGE_SIZE = 262144
+      DEFAULT_QUEUE_MESSAGE_RETENTION = 345600
+      DEFAULT_QUEUE_WAIT_TIME = 0
+      DEFAULT_QUEUE_VISIBILITY = 30
+      DEFAULT_QUEUE_POLICY = "{}"
+      DEFAULT_DEAD_LETTER_TARGET = "queue-2"
+      DEFAULT_DEAD_LETTER_RECEIVES = 3
+
+      @default_queue_attributes = {
+        "delay" => DEFAULT_QUEUE_DELAY,
+        "max-message-size" => DEFAULT_QUEUE_MESSAGE_SIZE,
+        "message-retention" => DEFAULT_QUEUE_MESSAGE_RETENTION,
+        "policy" => @default_policy_name,
+        "receive-wait-time" => DEFAULT_QUEUE_WAIT_TIME,
+        "visibility-timeout" => DEFAULT_QUEUE_VISIBILITY,
+        "dead-letter" => {
+          "target" => DEFAULT_DEAD_LETTER_TARGET,
+          "max-receives" => DEFAULT_DEAD_LETTER_RECEIVES
+        },
+      }
+
+      @default_aws_queue_attributes = {
+        "DelaySeconds" => "#{DEFAULT_QUEUE_DELAY}",
+        "MaximumMessageSize" => "#{DEFAULT_QUEUE_MESSAGE_SIZE}",
+        "MessageRetentionPeriod" => "#{DEFAULT_QUEUE_MESSAGE_RETENTION}",
+        "ReceiveMessageWaitTimeSeconds" => "#{DEFAULT_QUEUE_WAIT_TIME}",
+        "VisibilityTimeout" => "#{DEFAULT_QUEUE_VISIBILITY}",
+        "Policy" => DEFAULT_QUEUE_POLICY,
+        "RedrivePolicy" => JSON.generate({
+          "deadLetterTargetArn" => DEFAULT_DEAD_LETTER_TARGET,
+          "maxReceiveCount" => "#{DEFAULT_DEAD_LETTER_RECEIVES}"
+        }),
+      }
+
+      # Public: Reset the SQS module in between tests
+      def self.reset
+        Cumulus::Configuration.stub
+
+        if !Cumulus::SQS.respond_to? :reset_queue_urls
+          Cumulus::SQS.send :include, Cumulus::Test::ResetSQS
+        end
+
+        if !Cumulus::SQS.respond_to? :original_queue_arns
+          Cumulus::SQS.send :include, Cumulus::Test::DeadLetterArn
+        end
+
+        Cumulus::SQS::reset_queue_urls
+        Cumulus::SQS::reset_queue_attributes
+      end
+
+      # Public: Returns the String path of the "directory" that contains the
+      # policies.
+      def self.policy_directory
+        @policy_directory
+      end
+
+      # Public: Returns a Hash containing default queue attributes for a local
+      # queue definition with values overridden by the Hash passed in.
+      #
+      # overrides - optionally provide a Hash that will override default
+      #   attributes
+      def self.default_queue_attributes(overrides = nil)
+        Util::DeepMerge.deep_merge(@default_queue_attributes, overrides)
+      end
+
+      # Public: Returns a Hash containing default queue attributes for an AWS
+      # queue definition with values overridden by the Hash passed in.
+      #
+      # overrides - optionally provide a Hash that will override default
+      #   attributes
+      def self.default_aws_queue_attributes(overrides = nil)
+        Util::DeepMerge.deep_merge(@default_aws_queue_attributes, overrides)
+      end
+
+      # Public: Returns a fake queue url for a queue name
+      def self.queue_url(queue_name)
+        "http://sqs.us-east-1.amazonaws.com/123456789012/#{queue_name}"
+      end
+
+      # Public: Diff stubbed local configuration and stubbed AWS configuration.
+      #
+      # config - a Hash that contains two values, :local and :aws, which contain
+      #   the values to stub out.
+      #     :local contains :queues which is an Array of queues to stub the
+      #       directory with, and :policies, which is an Array of policy files
+      #       to stub out.
+      #     :aws is a hash of method names from the AWS Client to stub mapped
+      #       to the value the AWS Client should return
+      # test  - a block that tests the diffs returned by the Manager class
+      def self.do_diff(config, &test)
+        self.prepare_test(config)
+
+        # get the diffs and call the tester to determine the result of the test
+        manager = Cumulus::SQS::Manager.new
+        diffs = manager.diff_strings
+        test.call(diffs)
+      end
+
+      # Public: Sync stubbed local configuration and stubbed AWS configuration.
+      #
+      # config - a Hash that contains two values, :local and :aws, which contain
+      #   the values to stub out.
+      #     :local contains :queues which is an Array of queues to stub the
+      #       directory with, and :policies, which is an Array of policy files
+      #       to stub out.
+      #     :aws is a hash of method names from the AWS Client to stub mapped
+      #       to the value the AWS Client should return
+      # test - a block that tests the AWS Client after the syncing has been done
+      def self.do_sync(config, &test)
+        self.prepare_test(config)
+
+        # get the diffs and call the tester to determine the result of the test
+        manager = Cumulus::SQS::Manager.new
+        manager.sync
+        test.call(Cumulus::SQS::client)
+      end
+
+      private
+
+      def self.prepare_test(config)
+        self.reset
+        # we'll always just stub out the default policy
+        Cumulus::Common::BaseLoader.stub_file(
+          File.join(@policy_directory, @default_policy_name),
+          DEFAULT_QUEUE_POLICY
+        )
+
+        # stub out local queues
+        if config[:local][:queues]
+          Cumulus::Common::BaseLoader.stub_directory(
+            @queue_directory, config[:local][:queues]
+          )
+        end
+
+        # stub out local policies
+        if config[:local][:policies]
+          config[:local][:policies].each do |policy|
+            Cumulus::Common::BaseLoader.stub_file(
+              File.join(SQS::policy_directory, policy[:name]),
+              policy[:value]
+            )
+          end
+        end
+
+        # stub out aws responses
+        config[:aws].map do |call, value|
+          if value
+            Cumulus::SQS::client.stub_responses(call, value)
+          else
+            Cumulus::SQS::client.stub_responses(call)
+          end
+        end
+      end
+
+      def self.client_spy
+        if !Cumulus::SQS::client.respond_to? :have_received
+          Cumulus::SQS::client = ClientSpy.new(Cumulus::SQS::client)
+        end
+        Cumulus::SQS::client.clear_spy
+      end
+
+    end
+  end
+end

data/spec/sqs/SingleChangeTest.rb

@@ -0,0 +1,101 @@
+require "mocks/ClientSpy"
+require "sqs/SQSUtil"
+require "sqs/models/QueueDiff"
+
+module Cumulus
+  module Test
+    module SQS
+      # Public: A helper that makes defining a test for changing a single queue
+      # value as simple as defining some attributes. Automatically stubs local
+      # and AWS config and then tests that expected values are returned by the
+      # diffing function. The following attributes are available.
+      #
+      # Required
+      #   @path - the path in the local configuration to the value to override.
+      #     Each level in the JSON should be separated with "."
+      #   @new_value - the value to change to in the local configuration
+      #   @previous_value - the value to change from
+      #   @attribute_name - (for sync) the name of the attribute in AWS
+      #
+      # Optional
+      # @queue_name - the name of the queue
+      # @policy - if you need to stub out a policy, provide a Hash that contains
+      #   :name (the name of the policy file), and :value (the String contents
+      #   of the file)
+      # @test_value - supply if the correct test value should be different than
+      #   @new_value
+      class SingleChangeTest
+        include ::RSpec::Matchers
+        include ::RSpec::Mocks::ExampleMethods
+        include Cumulus::SQS::QueueChange
+        include Cumulus::SQS::DeadLetterChange
+
+        def initialize(&init)
+          @queue_name = "queue-name"
+          instance_eval(&init)
+        end
+
+        def self.execute(&init)
+          test = new(&init)
+          test.execute
+        end
+
+        def execute
+          SQS::do_diff(get_config) do |diffs|
+            diff_strings = diffs.map(&:to_s).join("\n").split("\n").map(&:strip)
+            expect(diff_strings).to eq @message
+          end
+        end
+
+        def self.execute_sync(&init)
+          test = new(&init)
+          test.execute_sync
+        end
+
+        def execute_sync
+          SQS::client_spy
+          SQS::do_sync(get_config) do |client|
+            set_attributes = client.spied_method(:set_queue_attributes)
+            expect(set_attributes.num_calls).to eq 1
+            arguments = set_attributes.arguments[0]
+            expect(arguments[:queue_url]).to eq Test::SQS::queue_url(@queue_name)
+
+            if @full_test_value
+              expect(arguments[:attributes]).to eq @full_test_value
+            else
+              expect(arguments[:attributes]).to eq ({ @attribute_name => @test_value || @new_value })
+            end
+          end
+        end
+
+        private
+
+        def get_config
+          keys = @path.split('.')
+          overrides = {}
+          keys.reduce(overrides) do |h, k|
+            h[k] = if k == keys.last then @new_value else {} end
+          end
+
+          {
+            local: {
+              queues: [{
+                name: @queue_name,
+                value: SQS::default_queue_attributes(overrides)
+              }],
+              policies: if @policy then [{
+                name: @policy[:name],
+                value: @policy[:value],
+              }] end
+            },
+            aws: {
+              list_queues: {queue_urls: [SQS::queue_url(@queue_name)]},
+              get_queue_attributes: {attributes: SQS::default_aws_queue_attributes}
+            }
+          }
+        end
+
+      end
+    end
+  end
+end

data/spec/sqs/diff_spec.rb

@@ -0,0 +1,152 @@
+require "sqs/SQSUtil"
+require "sqs/SingleChangeTest"
+
+module Cumulus
+  module Test
+    module SQS
+      describe Cumulus::SQS::Manager do
+        context "The SQS module's diffing functionality" do
+          it "should detect new queues defined locally" do
+            queue_name = "not-in-aws"
+            SQS::do_diff({
+              local: {queues: [{name: queue_name, value: {}}]},
+              aws: {list_queues: nil},
+            }) do |diffs|
+              expect(diffs.size).to eq 1
+              expect(diffs[0].to_s).to eq "Queue #{queue_name} will be created."
+            end
+          end
+
+          it "should detect new queues added in AWS" do
+            queue_name = "only-in-aws"
+            SQS::do_diff({
+              local: {queues: []},
+              aws: {
+                list_queues: {queue_urls: [SQS::queue_url(queue_name)]},
+                get_queue_attributes: {},
+              }
+            }) do |diffs|
+              expect(diffs.size).to eq 1
+              expect(diffs[0].to_s).to eq "Queue #{queue_name} is not managed by Cumulus."
+            end
+          end
+
+          it "should detect changes in delay" do
+            SingleChangeTest.execute do
+              @path = "delay"
+              @new_value = SQS::DEFAULT_QUEUE_DELAY - 1
+              @previous_value = SQS::DEFAULT_QUEUE_DELAY
+              @message = [
+                "Delay",
+                "AWS - #{@previous_value} seconds",
+                "Local - #{@new_value} seconds"
+              ]
+            end
+          end
+
+          it "should detect changes in max-message-size" do
+            SingleChangeTest.execute do
+              @path = "max-message-size"
+              @new_value = SQS::DEFAULT_QUEUE_MESSAGE_SIZE - 1
+              @previous_value = SQS::DEFAULT_QUEUE_MESSAGE_SIZE
+              @message = [
+                "Max Message Size",
+                "AWS - #{@previous_value} bytes",
+                "Local - #{@new_value} bytes"
+              ]
+            end
+          end
+
+          it "should detect changes in message-retention" do
+            SingleChangeTest.execute do
+              @path = "message-retention"
+              @new_value = SQS::DEFAULT_QUEUE_MESSAGE_RETENTION - 1
+              @previous_value = SQS::DEFAULT_QUEUE_MESSAGE_RETENTION
+              @message = [
+                "Message Retention Period",
+                "AWS - #{@previous_value} seconds",
+                "Local - #{@new_value} seconds"
+              ]
+            end
+          end
+
+          it "should detect changes in receive-wait-time" do
+            SingleChangeTest.execute do
+              @path = "receive-wait-time"
+              @new_value = SQS::DEFAULT_QUEUE_WAIT_TIME - 1
+              @previous_value = SQS::DEFAULT_QUEUE_WAIT_TIME
+              @message = [
+                "Receive Wait Time",
+                "AWS - #{@previous_value} seconds",
+                "Local - #{@new_value} seconds"
+              ]
+            end
+          end
+
+          it "should detect changes in visibility-timeout" do
+            SingleChangeTest.execute do
+              @path = "visibility-timeout"
+              @new_value = SQS::DEFAULT_QUEUE_VISIBILITY - 1
+              @previous_value = SQS::DEFAULT_QUEUE_VISIBILITY
+              @message = [
+                "Message Visibility",
+                "AWS - #{@previous_value} seconds",
+                "Local - #{@new_value} seconds"
+              ]
+            end
+          end
+
+          it "should detect changes in policy" do
+            new_value = "example-policy-2"
+            new_contents = "{\"a\":\"b\"}"
+            SingleChangeTest.execute do
+              @path = "policy"
+              @policy = {
+                name: new_value,
+                value: new_contents,
+              }
+              @new_value = new_value
+              @previous_value = JSON.parse(SQS::DEFAULT_QUEUE_POLICY)
+              test_value = JSON.pretty_generate(JSON.parse(new_contents)).split("\n").map(&:strip)
+              @message = [
+                "Policy:",
+                "Removing:",
+                JSON.pretty_generate(@previous_value).split("\n"),
+                "Adding:",
+                test_value
+              ].flatten
+            end
+          end
+
+          it "should detect changes in dead-letter target" do
+            SingleChangeTest.execute do
+              @path = "dead-letter.target"
+              @new_value = SQS::DEFAULT_DEAD_LETTER_TARGET + "a"
+              @previous_value = SQS::DEFAULT_DEAD_LETTER_TARGET
+              @message = [
+                "Dead Letter Queue",
+                "Target:",
+                "AWS - #{@previous_value}",
+                "Local - #{@new_value}"
+              ]
+            end
+          end
+
+          it "should detect changes in dead-letter max-receives" do
+            SingleChangeTest.execute do
+              @path = "dead-letter.max-receives"
+              @new_value = SQS::DEFAULT_DEAD_LETTER_RECEIVES + 1
+              @previous_value = SQS::DEFAULT_DEAD_LETTER_RECEIVES
+              @message = [
+                "Dead Letter Queue",
+                "Max Receive Count:",
+                "AWS - #{@previous_value}",
+                "Local - #{@new_value}"
+              ]
+            end
+          end
+        end
+      end
+    end
+  end
+end