lucid-cumulus 0.11.2 → 0.11.3

data/lib/autoscaling/Commands.rb

@@ -0,0 +1,20 @@
+module Cumulus
+  module AutoScaling
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.banner_message
+        format_message [
+          "autoscaling: Manage AutoScaling groups.",
+          "\tCompiles AutoScaling groups, scaling policies, and alarms that are defined in configuration files and syncs the resulting AutoScaling groups with AWS.",
+        ]
+      end
+
+      def self.manager
+        require "autoscaling/manager/Manager"
+        Cumulus::AutoScaling::Manager.new
+      end
+
+    end
+  end
+end

data/lib/aws_extensions/s3/BucketPolicy.rb

@@ -1,17 +1,35 @@
 require "aws-sdk"
+require "json"
+require "deepsort"
 
 module AwsExtensions
   module S3
     module BucketPolicy
       # Public: Method that will either return the bucket policy, or an empty
-      # string if there is no policy. We have to do this because catching an
-      # exception is the ONLY way to determine if there is a policy.
+      # string if there is no policy.
       #
       # Returns the policy as a string.
       def policy_string
-        policy.string
+        # fetch the sorted policy
+        hash = policy_hash
+        # check if policy exists
+        unless hash.nil?
+          # convert the policy to string
+          JSON.generate(hash)
+        else
+          # if no policy exists, return an empty string
+          ""
+        end
+      end
+
+      # Public: Method returns the bucket policy as a sorted hash
+      # if no policy exists, returns nil
+      def policy_hash
+        # rescue and ignore all excpetions related to no policy existing
+        # We have to do this because catching an exception is the ONLY way to determine if there is a policy.
+        JSON.parse(policy.string).deep_sort
       rescue Aws::S3::Errors::NoSuchBucketPolicy
-        ""
+        nil
       end
     end
   end

data/lib/cloudfront/Commands.rb

@@ -0,0 +1,51 @@
+module Cumulus
+  module CloudFront
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.banner_message
+        format_message [
+          "cloudfront: Manage CloudFront",
+          "\tDiff and sync CloudFront configuration with AWS.",
+        ]
+      end
+
+      def self.command_details
+        format_message [
+          ["diff", "print out differences between local configuration and AWS (supplying the id of the distribution will diff only that distribution)"],
+          ["invalidate", "create an invalidation.  Must supply the name of the invalidation to run.  Specifying 'list' as an argument lists the local invalidation configurations"],
+          ["list", "list the locally defined distributions"],
+          ["migrate", "produce Cumulus CloudFront distribution configuration from current AWS configuration"],
+          ["sync", "sync local cloudfront distribution configuration with AWS (supplying the id of the distribution will sync only that distribution)"],
+        ]
+      end
+
+      def self.manager
+        require "cloudfront/manager/Manager"
+        Cumulus::CloudFront::Manager.new
+      end
+
+      def self.valid_options
+        [["diff", "invalidate", "list", "migrate", "sync"]]
+      end
+
+      def self.execute(arguments)
+        if arguments[0] == "invalidate"
+          if arguments.size != 2
+            puts "Specify one invalidation to run"
+            exit
+          else
+            if arguments[1] == "list"
+              manager.list_invalidations
+            else
+              manager.invalidate(arguments[1])
+            end
+          end
+        else
+          super(arguments)
+        end
+      end
+
+    end
+  end
+end

data/lib/common/BaseLoader.rb

@@ -27,7 +27,7 @@ module Cumulus
       # json    - indicates if the resources are in json format
       # loader  - the function that will handle the read json
       def self.resource(file, dir, json = true, &loader)
-        name = file.end_with?(".json") ? file.chomp(".json") : file
+        name = file.chomp(".json")
 
         begin
           contents = load_file(file, dir)
@@ -46,12 +46,14 @@ module Cumulus
       #
       # file    - the name of the file to load
       # dir     - the directory the file is located in
-      # vars    - the variables to apply to the template
+      # vars    - the variables to apply to the template (can be nil)
       # loader  - the function that will handle the read json
       def self.template(file, dir, vars, &loader)
         template = load_file(file, dir)
-        vars.each do |key, value|
-          template.gsub!("{{#{key}}}", "#{value}")
+        if vars
+          vars.each do |key, value|
+            template.gsub!("{{#{key}}}", "#{value}")
+          end
         end
         json = JSON.parse(template)
 

data/lib/common/Commands.rb

@@ -0,0 +1,131 @@
+module Cumulus
+  module Common
+    # Public: Base class for the command line parser class.
+    #
+    # Classes that extend this class must provide the following methods:
+    #
+    #   self.manager - returns the manager for the AWS module that is being used.
+    #
+    # Additionally, the following methods can be set to change the behavior of the parser:
+    #
+    #   self.usage_message - returns the usage instructions.
+    #   self.banner_message - returns the title, purpose, and behavior of the module. This is displayed in the help message.
+    #   self.command_details - returns basic instructions on how to use each module command. This is displayed in the help message.
+    #   self.valid_options - returns an array of the valid arguments where each argument is an array of valid commands.
+    #   self.verify - returns true/false of whether or not the arguments passed in are valid (proper order, right commands, etc).
+    #   self.execute - runs the correct method on the manager based on the array of arguments passed in.
+    #
+    # For your convenience, many of the help and usage messages can be formatted correctly with the self.format_message method.
+    #   To use this method, pass in the message as an array of 'lines' where each line is either a string, or a command-instruction pair.
+    #
+    # The following super class is one example of what each method could look like. Change them in inherited classes as necessary.
+    class Commands
+      def self.usage_message
+        options = valid_options
+        options[0] = options[0].push("help")
+        "Usage: cumulus #{manager_name}" + options.reduce(String.new) do |memo, param_list|
+          memo + " [" + param_list.join("|") + "]"
+        end + " <asset>"
+      end
+
+      def self.banner_message
+        format_message [
+          "#{manager_name}: Manage #{manager_name.upcase}s.",
+          "\tCompiles #{manager_name.upcase}s that are defined with configuration files and syncs the resulting #{manager_name.upcase} assets with AWS.",
+        ]
+      end
+
+      def self.command_details
+        format_message [
+          ["diff", "get a list of resources that have different definitions locally than in AWS (supplying the name of the group will diff only that group)"],
+          ["list", "list the resources defined in configuration"],
+          ["migrate", "create resource configuration files that match the definitions in AWS"],
+          ["sync", "sync the local resource definition with AWS (supplying the name of the resource will sync only that group). Also adds and removes users from groups"],
+        ]
+      end
+
+      def self.help_message
+        format_message [
+          "#{banner_message}",
+          "",
+          "#{usage_message}",
+          "",
+          "Commands",
+          "#{command_details}"
+        ]
+      end
+
+      def self.manager
+        require "common/manager/Manager"
+        Cumulus::Common::Manager.new
+      end
+
+      # Retrieves the AWS module name by checking what ruby module the class is in.
+      def self.manager_name
+        manager.class.to_s.split("::")[1].downcase
+      end
+
+      def self.valid_options
+        # The first array is the list of all possible first commands
+        # The second array is the list of all possible second commands and so on.
+        [["diff", "list", "migrate", "sync"]]
+      end
+
+      def self.verify(arguments)
+        (arguments.size >= valid_options.size) and
+        (valid_options.size < 1 or valid_options[0].include?(arguments[0])) and
+        (valid_options.size < 2 or valid_options[1].include?(arguments[1])) and
+        (valid_options.size < 3 or valid_options[2].include?(arguments[2]))
+      end
+
+      def self.execute(arguments)
+        if arguments[0] == "diff" and arguments.size == 2
+          manager.diff_one(arguments[1])
+        elsif arguments[0] == "sync" and arguments.size == 2
+          manager.sync_one(arguments[1])
+        else
+          manager.method(arguments[0]).call
+        end
+      end
+
+      # use this helper function to format help messages
+      def self.format_message(message, args = Hash.new)
+        # default pad is the size of the smallest command
+        pad = args.key?(:padding) ? args[:padding] : message.reduce(0) do |memo, line|
+          if line.class == Array && line.first.size > memo
+            line.first.size
+          else
+            memo
+          end
+        end
+
+        message = message.reduce(String.new) do |memo, line|
+          if line.class == Array
+            memo + "\t%-#{pad}s - %s\n" % line
+          else
+            memo + line + "\n"
+          end
+        end.chomp
+
+        message = "\t"*args[:indent] + message.gsub("\n", "\n" + "\t"*args[:indent]) if args.key?(:indent)
+
+        message
+      end
+
+      # the main function called by the command line parser. DON'T OVERRIDE
+      def self.parse(arguments)
+        if arguments.size >= 1 and arguments[0] == "help"
+          puts help_message
+          exit
+        end
+
+        if !verify(arguments)
+          puts usage_message
+          exit
+        end
+
+        execute(arguments)
+      end
+    end
+  end
+end

data/lib/conf/Configuration.rb

@@ -69,10 +69,10 @@ module Cumulus
         value
       end
     rescue KeyError => e
-      puts "Your configuration file is missing $.#{key}."
       if allow_missing
         nil
       else
+        puts "Your configuration file is missing $.#{key}."
         exit
       end
     end
@@ -91,15 +91,16 @@ module Cumulus
     # Internal: Constructor. Sets up the `instance` variable, which is the access
     # point for the Singleton.
     #
+    # json - a Hash containing the json configuration. If nil, this value will be read in from the default location
     # conf_dir  - The String path to the directory the configuration can be found in
     # profile       - The String profile name that will be used to make AWS API calls
     # assume_role - The ARN of the role to assume when making AWS API calls
     # autoscaling_force_size
     #               -  Determines whether autoscaling should use configured values for
     #                  min/max/desired group size
-    def initialize(conf_dir, profile, assume_role, autoscaling_force_size)
-      Config.conf_dir = conf_dir;
-      Config.json = JSON.parse(File.read(absolute_path("configuration.json")))
+    def initialize(conf_dir, profile, assume_role, autoscaling_force_size, json = nil)
+      Config.conf_dir = conf_dir
+      Config.json = json.nil? ? JSON.parse(File.read(absolute_path("configuration.json"))) : json
       @colors_enabled = conf "colors-enabled"
       @iam = IamConfig.new
       @autoscaling = AutoScalingConfig.new(autoscaling_force_size)
@@ -126,6 +127,7 @@ module Cumulus
         :region => region,
         :profile => profile,
         :credentials => credentials,
+        :stub_responses => conf("stub_aws_responses", true)
       }.reject { |_, v| v.nil? }
     end
 
@@ -140,8 +142,22 @@ module Cumulus
       #               -  Determines whether autoscaling should use configured values for
       #                  min/max/desired group size
       def init(conf_dir, profile, assume_role, autoscaling_force_size)
-        instance = new(conf_dir, profile, assume_role, autoscaling_force_size)
-        @@instance = instance
+        @@instance = new(conf_dir, profile, assume_role, autoscaling_force_size)
+      end
+
+      # Private: Initialize the Configuration Singleton. Must be called before any
+      # access to `Configuration.instance` is used. Used by the tests to pass in json
+      # rather than a file
+      #
+      # json - the Hash containing the json configuration
+      # conf_dir  - The String path to the directory the configuration can be found in
+      # profile       - The String profile name that will be used to make AWS API calls
+      # assume_role - The ARN of the role to assume when making AWS API calls
+      # autoscaling_force_size
+      #               -  Determines whether autoscaling should use configured values for
+      #                  min/max/desired group size
+      def init_from_hash(json, conf_dir, profile, assume_role, autoscaling_force_size)
+        @@instance = new(conf_dir, profile, assume_role, autoscaling_force_size, json)
       end
 
       # Public: The Singleton instance of Configuration.
@@ -152,6 +168,7 @@ module Cumulus
       end
 
       private :new
+      private :init_from_hash
     end
 
     # Public: Inner class that contains IAM configuration options

data/lib/ec2/Commands.rb

@@ -0,0 +1,51 @@
+module Cumulus
+  module EC2
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.command_details
+        format_message([
+          "ebs - Manage EBS volumes in groups",
+          ["diff", "get a list of groups that have different definitions locally than in AWS (supplying the name of the group will diff only that group)"],
+          ["list", "list the groups defined in configuration"],
+          ["migrate", "create group configuration files that match the definitions in AWS"],
+          ["sync", "sync the local group definition with AWS (supplying the name of the group will sync only that group). Also creates volumes in a group"],
+          "instances - Manage EC2 instances",
+          ["diff", "get a list of instances that have different definitions locally than in AWS (supplying the name of the instance will diff only that instance)"],
+          ["list", "list the instances defined in configuration"],
+          ["migrate", "create instances configuration files that match the definitions in AWS"],
+          ["sync", "sync the local instance definition with AWS (supplying the name of the instance will sync only that instance)"],
+        ], indent: 1)
+      end
+
+      def self.manager_name
+        "ec2"
+      end
+
+      def self.valid_options
+         [["ebs", "instances"], ["diff", "list", "migrate", "sync"]]
+      end
+
+      def self.execute(arguments)
+        manager = if arguments[0] == "ebs"
+          require "ec2/managers/EbsManager"
+          Cumulus::EC2::EbsManager.new
+        elsif arguments[0] == "instances"
+          require "ec2/managers/InstanceManager"
+          Cumulus::EC2::InstanceManager.new
+        else
+          nil
+        end
+
+        if arguments[1] == "diff" and arguments.size == 3
+          manager.diff_one(arguments[2])
+        elsif arguments[1] == "sync" and arguments.size == 3
+          manager.sync_one(arguments[2])
+        else
+          manager.method(arguments[1]).call
+        end
+      end
+
+    end
+  end
+end

data/lib/ec2/models/InstanceConfig.rb

@@ -187,7 +187,7 @@ module Cumulus
           if !group_diffs.empty?
             [group_name, EbsGroupDiff.modified(aws_config, group_config, group_diffs)]
           end
-        end].reject { |k, v| v.nil? }
+        end.reject { |v| v.nil? }]
 
         ebs_changes = Common::ListChange.new(added_groups, removed_groups, changed_groups)
         if !ebs_changes.empty?

data/lib/elb/Commands.rb

@@ -0,0 +1,40 @@
+module Cumulus
+  module ELB
+    require "common/Commands"
+    class Commands < Cumulus::Common::Commands
+
+      def self.command_details
+        format_message [
+          ["diff", "print out differences between local configuration and AWS (supplying the name of the elb will diff only that elb)"],
+          ["list", "list the locally defined ELBs"],
+          ["sync", "sync local ELB definitions with AWS (supplying the name of the elb will sync only that elb)"],
+          ["migrate", "migrate AWS configuration to Cumulus"],
+          format_message([
+            ["default-policies", "migrate default ELB policies from AWS to Cumulus"],
+            ["elbs", "migrate the current ELB configuration from AWS to Cumulus"],
+          ], indent: 1),
+        ]
+      end
+
+      def self.manager
+        require "elb/manager/Manager"
+        Cumulus::ELB::Manager.new
+      end
+
+      def self.execute(arguments)
+        if arguments[0] == "migrate"
+          if arguments[1] == "default-policies"
+            manager.migrate_default_policies
+          elsif arguments[1] == "elbs"
+            manager.migrate_elbs
+          else
+            puts "Usage: cumulus elb migrate [default-policies|elbs]"
+          end
+        else
+          super(arguments)
+        end
+      end
+
+    end
+  end
+end