lucid-cumulus 0.11.2 → 0.11.3

data/lib/iam/Commands.rb

@@ -0,0 +1,49 @@
+module Cumulus
+  module IAM
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.command_details
+        format_message([
+          "groups - Manage IAM groups and users associated with those groups",
+          ["diff", "get a list of groups that have different definitions locally than in AWS (supplying the name of the group will diff only that group)"],
+          ["list", "list the groups defined in configuration"],
+          ["migrate", "create group configuration files that match the definitions in AWS"],
+          ["sync", "sync the local group definition with AWS (supplying the name of the group will sync only that group). Also adds and removes users from groups"],
+          "roles - Manage IAM roles",
+          ["diff", "get a list of roles that have different definitions locally than in AWS (supplying the name of the role will diff only that role)"],
+          ["list", "list the roles defined in configuration"],
+          ["migrate", "create role configuration files that match the definitions in AWS"],
+          ["sync", "sync the local role definition with AWS (supplying the name of the role will sync only that role)"],
+          "users - Manage IAM users",
+          ["diff", "get a list of users that have different definitions locally than in AWS (supplying the name of the user will diff only that user)"],
+          ["list", "list the users defined in configuration"],
+          ["migrate", "create user configuration files that match the definitions in AWS"],
+          ["sync", "sync the local user definition with AWS (supplying the name of the user will sync only that user)"],
+        ], indent: 1)
+      end
+
+      def self.valid_options
+        [["groups", "roles", "users"], ["diff", "list", "migrate", "sync"]]
+      end
+
+      def self.manager
+        require "iam/manager/Manager"
+        Cumulus::IAM::Manager.new
+      end
+
+      def self.execute(arguments)
+        resource = super(arguments)
+
+        if arguments[1] == "diff" and arguments.size == 3
+          resource.diff_one(arguments[2])
+        elsif arguments[1] == "sync" and arguments.size == 3
+          resource.sync_one(arguments[2])
+        else
+          resource.method(arguments[1]).call
+        end
+      end
+
+    end
+  end
+end

data/lib/iam/models/ResourceWithPolicy.rb

@@ -167,7 +167,15 @@ module Cumulus
       def diff(aws_resource)
         diffs = []
 
-        aws_policies = Hash[aws_resource.policies.map { |p| [p.name, p] }]
+        aws_policies = Hash[aws_resource.policies.map do |policy|
+          sorted_policy = JSON.parse(URI.unescape(policy.policy_document))
+          sorted_policy["Statement"].each do |statement|
+            # Sort the statments before diffing to prevent false conflicts
+            statement["Action"].sort!
+            statement["Resource"].sort!
+          end
+          [policy.name, sorted_policy]
+        end]
         p = policy
         p.name = generated_policy_name
 
@@ -181,9 +189,8 @@ module Cumulus
           if name != generated_policy_name
             diffs << IamDiff.unmanaged_policy(name)
           else
-            aws_statements = JSON.parse(URI.unescape(aws_policy.policy_document))["Statement"]
+            aws_statements = aws_policy["Statement"]
             local_statements = p.as_hash["Statement"]
-
             if aws_statements != local_statements
               diff = IamDiff.new(IamChange::POLICY, aws_statements, p)
               diff.policy_name = generated_policy_name

data/lib/iam/models/StatementConfig.rb

@@ -12,8 +12,8 @@ module Cumulus
       # json - the Hash containing the JSON configuration for this StatementConfig
       def initialize(json)
         @effect = json["Effect"]
-        @action = json["Action"]
-        @resource = json["Resource"]
+        @action = json["Action"].sort
+        @resource = json["Resource"].sort
         @condition = json["Condition"]
       end
 

data/lib/kinesis/Commands.rb

@@ -0,0 +1,20 @@
+module Cumulus
+  module Kinesis
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.banner_message
+        format_message [
+          "kinesis: Manage Kinesis Streams",
+          "\tDiff and sync Kinesis configuration with AWS.",
+        ]
+      end
+
+      def self.manager
+        require "kinesis/manager/Manager"
+        Cumulus::Kinesis::Manager.new
+      end
+
+    end
+  end
+end

data/lib/route53/Commands.rb

@@ -0,0 +1,20 @@
+module Cumulus
+  module Route53
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.banner_message
+        format_message [
+          "route53: Manage Route53",
+          "\tDiff and sync Route53 configuration with AWS.",
+        ]
+      end
+
+      def self.manager
+        require "route53/manager/Manager"
+        Cumulus::Route53::Manager.new
+      end
+
+    end
+  end
+end

data/lib/s3/Commands.rb

@@ -0,0 +1,20 @@
+module Cumulus
+  module S3
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.manager
+        require "s3/manager/Manager"
+        Cumulus::S3::Manager.new
+      end
+
+      def self.banner_message
+        format_message [
+          "s3: Manage S3 Buckets",
+          "\tDiff and sync S3 bucket configuration with AWS.",
+        ]
+      end
+
+    end
+  end
+end

data/lib/s3/loader/Loader.rb

@@ -1,6 +1,7 @@
 require "common/BaseLoader"
 require "conf/Configuration"
 require "s3/models/BucketConfig"
+require "deepsort"
 
 require "aws-sdk"
 
@@ -58,7 +59,7 @@ module Cumulus
           name,
           @@policies_dir,
           vars,
-          &proc { |n, json| json.to_json }
+          &proc { |n, json| json.deep_sort.to_json }
         )
       end
     end

data/lib/security/Commands.rb

@@ -0,0 +1,24 @@
+module Cumulus
+  module SecurityGroups
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.manager_name
+        "security-groups"
+      end
+
+      def self.manager
+        require "security/manager/Manager"
+        Cumulus::SecurityGroups::Manager.new
+      end
+
+      def self.banner_message
+        format_message [
+          "security-groups: Manage EC2 Security Groups",
+          "\tDiff and sync EC2 security group configuration with AWS.",
+        ]
+      end
+
+    end
+  end
+end

data/lib/sqs/Commands.rb

@@ -0,0 +1,34 @@
+module Cumulus
+  module SQS
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.banner_message
+        format_message [
+          "SQS: Manage SQS",
+          "\tDiff and sync SQS configuration with AWS.",
+        ]
+      end
+
+      def self.command_details
+        format_message [
+          ["diff", "print out differences between local configuration and AWS (supplying the name of the queue will diff only that queue)"],
+          ["list", "list the locally defined queues"],
+          ["urls", "list the url for each locally defined queue"],
+          ["sync", "sync local queue definitions with AWS (supplying the name of the queue will sync only that queue)"],
+          ["migrate", "migrate AWS configuration to Cumulus"],
+        ]
+      end
+
+      def self.manager
+        require "sqs/manager/Manager"
+        Cumulus::SQS::Manager.new
+      end
+
+      def self.valid_options
+        [["diff", "list", "migrate", "sync", "urls"]]
+      end
+
+    end
+  end
+end

data/lib/sqs/SQS.rb

@@ -8,6 +8,12 @@ module Cumulus
     class << self
       @@client = Aws::SQS::Client.new(Configuration.instance.client)
 
+      # Public
+      # Returns the AWS client used in the module
+      def client
+        @@client
+      end
+
       # Public: Static method that will get the ARN of a Queue
       #
       # name - the name of the queue to get

data/lib/sqs/loader/Loader.rb

@@ -10,21 +10,26 @@ module Cumulus
     module Loader
       include Common::BaseLoader
 
-      @@queues_dir = Configuration.instance.sqs.queues_directory
-      @@policies_dir = Configuration.instance.sqs.policies_directory
+      def self.queues_dir
+        Configuration.instance.sqs.queues_directory
+      end
+
+      def self.policies_dir
+        Configuration.instance.sqs.policies_directory
+      end
 
       # Public: Load all the queue configurations as QueueConfig objects
       #
       # Returns an array of QueueConfig
       def self.queues
-        Common::BaseLoader::resources(@@queues_dir, &QueueConfig.method(:new))
+        Common::BaseLoader::resources(self.queues_dir, &QueueConfig.method(:new))
       end
 
       # Public: Load the specified policy as a JSON object
       #
       # Returns the JSON object for the policy
       def self.policy(policy_name)
-        Common::BaseLoader::resource(policy_name, @@policies_dir) do |policy_name, policy|
+        Common::BaseLoader::resource(policy_name, self.policies_dir) do |policy_name, policy|
           policy
         end
       end

data/lib/sqs/manager/Manager.rb

@@ -15,7 +15,6 @@ module Cumulus
       def initialize
         super()
         @create_asset = true
-        @client = Aws::SQS::Client.new(Configuration.instance.client)
       end
 
       def resource_name
@@ -50,16 +49,16 @@ module Cumulus
       end
 
       def update(local, diffs)
-        @client.set_queue_attributes({
+        SQS::client.set_queue_attributes({
           queue_url: SQS::queue_urls[local.name],
           attributes: {
-            "DelaySeconds" => if diffs.any? { |d| d.type == QueueChange::DELAY } then local.delay end,
-            "MaximumMessageSize" => if diffs.any? { |d| d.type == QueueChange::MESSAGE_SIZE } then local.max_message_size end,
-            "MessageRetentionPeriod" => if diffs.any? { |d| d.type == QueueChange::MESSAGE_RETENTION } then local.message_retention end,
+            "DelaySeconds" => if diffs.any? { |d| d.type == QueueChange::DELAY } then local.delay.to_s end,
+            "MaximumMessageSize" => if diffs.any? { |d| d.type == QueueChange::MESSAGE_SIZE } then local.max_message_size.to_s end,
+            "MessageRetentionPeriod" => if diffs.any? { |d| d.type == QueueChange::MESSAGE_RETENTION } then local.message_retention.to_s end,
             "Policy" => if diffs.any? { |d| d.type == QueueChange::POLICY }
               if local.policy then JSON.generate(Loader.policy(local.policy)) else "" end
             end,
-            "ReceiveMessageWaitTimeSeconds" => if diffs.any? { |d| d.type == QueueChange::RECEIVE_WAIT } then local.receive_wait_time end,
+            "ReceiveMessageWaitTimeSeconds" => if diffs.any? { |d| d.type == QueueChange::RECEIVE_WAIT } then local.receive_wait_time.to_s end,
             "VisibilityTimeout" => if diffs.any? { |d| d.type == QueueChange::VISIBILITY } then local.visibility_timeout end,
             "RedrivePolicy" => if diffs.any? { |d| d.type == QueueChange::DEAD }
               if local.dead_letter then JSON.generate(local.dead_letter.to_aws) else "" end
@@ -69,7 +68,7 @@ module Cumulus
       end
 
       def create(local)
-        url = @client.create_queue({
+        url = SQS::client.create_queue({
           queue_name: local.name,
           attributes: {
             "DelaySeconds" => "#{local.delay}",

data/lib/vpc/Commands.rb

@@ -0,0 +1,43 @@
+module Cumulus
+  module VPC
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.command_details
+        format_message [
+          ["diff", "print out differences between local configuration and AWS (supplying the name of the VPC will diff only that VPC)"],
+          ["list", "list the locally defined VPCs"],
+          ["sync", "sync local VPC definitions with AWS (supplying the name of the VPC will sync only that VPC)"],
+          ["migrate", "migrate AWS configuration to Cumulus"],
+          ["rename", "renames a cumulus asset and all references to it"],
+        ]
+      end
+
+      def self.manager
+        require "vpc/manager/Manager"
+        Cumulus::VPC::Manager.new
+      end
+
+      def self.valid_options
+        [["diff", "list", "migrate", "sync", "rename"]]
+      end
+
+      def self.execute(arguments)
+        if arguments[0] == "diff" and arguments.size == 2
+          manager.diff_one(arguments[1])
+        elsif arguments[0] == "sync" and arguments.size == 2
+          manager.sync_one(arguments[1])
+        elsif arguments[0] == "rename"
+          if arguments.size == 5
+            manager.rename(arguments[1], arguments[2], arguments[3])
+          else
+            puts "Usage: cumulus vpc rename [network-acl|policy|route-table|subnet|vpc] <old-asset-name> <new-asset-name>"
+          end
+        else
+          manager.method(arguments[0]).call
+        end
+      end
+
+    end
+  end
+end

data/lucid-cumulus.gemspec

@@ -4,7 +4,7 @@ require "bundler"
 
 Gem::Specification.new do |s|
   s.name                      = "lucid-cumulus"
-  s.version                   = "0.11.2"
+  s.version                   = "0.11.3"
   s.platform                  = Gem::Platform::RUBY
   s.authors                   = ["Keilan Jackson", "Mark Siebert"]
   s.email                     = "cumulus@lucidchart.com"
@@ -17,4 +17,5 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "aws-sdk", "2.2.30"
   s.add_runtime_dependency "parse-cron", "~> 0.1.4"
+  s.add_runtime_dependency "deepsort", "~> 0.1"
 end

data/rakefile.rb

@@ -1,8 +1,13 @@
 require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
 
 task :default => :spec
 
 task :spec do
-  puts "Running tests... \n"
-  puts "Tests have not been written yet :("
+  Rake::Task['spec_sqs'].execute
+end
+
+RSpec::Core::RakeTask.new(:spec_sqs) do |task|
+  task.rspec_opts = ['-r ./spec/rspec_config.rb']
+  task.pattern = 'spec/sqs/*_spec.rb'
 end

data/spec/mocks/ClientSpy.rb

@@ -0,0 +1,43 @@
+module Cumulus
+  module Test
+    SpiedMethod = Struct.new(:arguments_list) do
+      def num_calls
+        arguments_list.size
+      end
+
+      def arguments
+        arguments_list[0]
+      end
+    end
+
+    class ClientSpy
+      attr_reader :method_calls
+
+      def initialize(client)
+        @method_calls = {}
+        metaclass = class << self; self; end
+        client.methods.each do |m|
+          method_name = m.to_sym
+          metaclass.send(:define_method, method_name) do |*args|
+            if !@method_calls.has_key? method_name
+              @method_calls[method_name] = SpiedMethod.new([])
+            end
+            @method_calls[method_name] = SpiedMethod.new(
+              @method_calls[method_name].arguments_list.push(args)
+            )
+            client.method(method_name).call(*args)
+          end
+        end
+      end
+
+      def clear_spy
+        @method_calls = {}
+      end
+
+      def spied_method(name)
+        @method_calls[name]
+      end
+
+    end
+  end
+end

data/spec/mocks/MockedConfiguration.rb

@@ -0,0 +1,61 @@
+require "util/DeepMerge"
+
+module Cumulus
+  module Test
+    module MockedConfiguration
+      def self.included(base)
+        base.instance_eval do
+          include Cumulus::Test::Util::DeepMerge
+
+          def stub(overrides = nil)
+            overridden = Util::DeepMerge.deep_merge(@@default_config, overrides)
+            init_from_hash(overridden, '/mocked', nil, nil, false)
+          end
+
+          stub
+        end
+      end
+
+      @@default_config = {
+        "stub_aws_responses" => true,
+        "colors-enabled" => false,
+        "iam" => {
+          "policies" => {
+            "prefix" => "",
+            "suffix" => "",
+            "version" => "",
+          },
+        },
+        "autoscaling" => {
+          "groups" => {
+            "override-launch-config-on-sync" => false,
+          },
+        },
+        "route53" => {
+          "print-all-ignored" => true,
+        },
+        "s3" => {
+          "print-progress" => true,
+        },
+        "security" => {
+          "outbound-default-all-allowed" => true,
+          "subnet-files" => [
+            "security-groups/subnets.json",
+          ],
+        },
+        "ec2" => {
+          "instances" => {
+            "ignore-unmanaged" => true,
+            "default-image-id" => nil,
+            "volume-mounting" => {
+              "base" => "/dev/xvd",
+              "start" => "f",
+              "end" => "z",
+            },
+          },
+        },
+        "region" => "us-east-1",
+      }
+    end
+  end
+end