lucid-cumulus 0.11.0

data/lib/common/BaseLoader.rb

@@ -0,0 +1,80 @@
+require "json"
+
+module Cumulus
+  module Common
+    # Public: A module that handles loading all the 4 configuration files and
+    # creating objects from them.
+    module BaseLoader
+      # Internal: Load the resources in a directory, handling each file with the
+      # function passed in.
+      #
+      # dir               - the directory to load resources from
+      # json              - indicates if the resources are in json format
+      # individual_loader - the function that loads a resource from each file name
+      #
+      # Returns an array of resources
+      def self.resources(dir, json = true, &individual_loader)
+        Dir.entries(dir)
+        .reject { |f| f == "." or f == ".." or File.directory?(File.join(dir, f)) }
+        .map { |f| resource(f, dir, json, &individual_loader) }.reject(&:nil?)
+      end
+
+      # Internal: Load the resource, passing the parsed JSON to the function passed
+      # in
+      #
+      # file    - the name of the file to load
+      # dir     - the directory the file is located in
+      # json    - indicates if the resources are in json format
+      # loader  - the function that will handle the read json
+      def self.resource(file, dir, json = true, &loader)
+        name = file.end_with?(".json") ? file[0...-5] : file
+
+        begin
+          contents = load_file(file, dir)
+          loader.call(
+            name,
+            if json then JSON.parse(contents) else contents end
+          )
+        rescue => e
+          puts "Unable to load resource #{file}: #{e}"
+          nil
+        end
+      end
+
+      # Internal: Load the template, apply variables, and pass the parsed JSON to
+      # the function passed in
+      #
+      # file    - the name of the file to load
+      # dir     - the directory the file is located in
+      # vars    - the variables to apply to the template
+      # loader  - the function that will handle the read json
+      def self.template(file, dir, vars, &loader)
+        template = load_file(file, dir)
+        vars.each do |key, value|
+          template.gsub!("{{#{key}}}", "#{value}")
+        end
+        json = JSON.parse(template)
+
+        loader.call(nil, json)
+      end
+
+      # Internal: Load a file. Will check if a file exists by the name passed in, or
+      # with a .json extension.
+      #
+      # file - the name of the file to load
+      # dir  - the directory the file is located in
+      #
+      # Returns the contents of the file
+      def self.load_file(file, dir)
+        path = File.join(dir, file)
+        if File.exist?(path)
+          File.read(path)
+        elsif File.exist?("#{path}.json")
+          File.read("#{path}.json")
+        else
+          throw "File does not exist: #{path}"
+        end
+      end
+    end
+  end
+end

data/lib/common/manager/Manager.rb

@@ -0,0 +1,148 @@
+require "common/models/Diff"
+require "util/Colors"
+require "util/StatusCodes"
+
+module Cumulus
+  module Common
+    # Public: Base class for AWS resource manager classes.
+    #
+    # Classes that extend this class should provide the following methods:
+    #
+    #   resource_name - return the resource name type (ie "Autoscaling Group", "Security Group", etc)
+    #   local_resources - return a Hash of local resource name to local resource config object
+    #   aws_resources - return a Hash of aws resource name to aws resource object
+    #   diff_resource - a function that will produce an array of differences between the local resource
+    #     passed in and the aws resource passed in
+    #   unmanaged_diff - return the correct type of diff from an AWS resource
+    #   added_diff - return the correct type of diff from a local configuration object
+    #   create - given a local configuration, create the AWS resource
+    #   update - given a local configuration and an array of diffs, update the AWS resource
+    #
+    # Additionally, the following instance variables can be set to change the behavior of the manager:
+    #
+    #   create_asset - if true, the asset will be created, if false, a warning will be printed about
+    #     the asset not being created
+    class Manager
+      def initialize
+        @migration_root = "generated"
+        @create_asset = true
+      end
+
+      # Public: Print a diff between local configuration and configuration in AWS
+      def diff
+        each_difference(local_resources, true) { |key, diffs| print_difference(key, diffs) }
+      end
+
+      # Public: Print the diff between local configuration and AWS for a single resource
+      #
+      # name - the name of the resource to diff
+      def diff_one(name)
+        each_difference(filter_local(name), false) { |key, diffs| print_difference(key, diffs) }
+      end
+
+      # Public: Print out the names of all resources managed by Cumulus
+      def list
+        puts local_resources.map { |key, l| l.name }.join(" ")
+      end
+
+      # Public: Sync local configuration to AWS
+      def sync
+        each_difference(local_resources, true) { |key, diffs| sync_difference(key, diffs) }
+      end
+
+      # Public: Sync local configuration to AWS for a single resource
+      #
+      # name - the name of the resource to sync
+      def sync_one(name)
+        each_difference(filter_local(name), false) { |key, diffs| sync_difference(key, diffs) }
+      end
+
+      # Public: Select local resources based on name
+      def filter_local(name)
+        local_resources.reject { |key, l| l.name != name }
+      end
+
+      private
+
+      # Internal: Loop through the differences between local configuration and AWS
+      #
+      # locals            - the local configurations to compare against
+      # include_unmanaged - whether to include unmanaged resources in the list of changes
+      # f                 - a function that will be passed the name of the resource and an array of
+      #                     diffs
+      def each_difference(locals, include_unmanaged, &f)
+
+        unmanaged = if include_unmanaged
+          Hash[aws_resources.map do |key, resource|
+            [key, [unmanaged_diff(resource)]] if !locals.include?(key)
+          end.compact]
+        else
+          {}
+        end
+
+        managed = Hash[locals.map do |key, resource|
+          if !aws_resources.include?(key)
+            [key, [added_diff(resource)]]
+          else
+            [key, diff_resource(resource, aws_resources[key])]
+          end
+        end]
+
+        combined = unmanaged.merge(managed)
+        sorted_keys = combined.keys.sort
+        sorted_keys.each do |key|
+          f.call(key, combined[key])
+        end
+      end
+
+      # Internal: Print differences.
+      #
+      # key   - the name of the resource to print
+      # diffs - the differences between local configuration and AWS
+      def print_difference(key, diffs)
+        if diffs.size > 0
+
+          if diffs.reject(&:info_only).size > 0
+            StatusCodes::set_status(StatusCodes::DIFFS)
+          end
+
+          if diffs.size == 1 and (diffs[0].type == DiffChange::ADD or
+            diffs[0].type == DiffChange::UNMANAGED)
+            puts diffs[0]
+          else
+            puts "#{resource_name} #{local_resources[key].name} has the following changes:"
+            diffs.each do |diff|
+              diff_string = diff.to_s.lines.map { |s| "\t#{s}" }.join
+              puts diff_string
+            end
+          end
+        end
+      end
+
+      # Internal: Sync differences.
+      #
+      # key   - the name of the resource to sync
+      # diffs - the differences between local configuration and AWS
+      def sync_difference(key, diffs)
+        if diffs.size > 0
+
+          StatusCodes::set_status(StatusCodes::SYNC_DIFFS)
+
+          if diffs[0].type == DiffChange::UNMANAGED
+            puts diffs[0]
+          elsif diffs[0].type == DiffChange::ADD
+            if @create_asset
+              puts Colors.added("creating #{local_resources[key].name}...")
+              create(local_resources[key])
+            else
+              puts "not creating #{local_resources[key].name}..."
+            end
+          else
+            puts Colors.blue("updating #{local_resources[key].name}...")
+            update(local_resources[key], diffs)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/common/models/Diff.rb

@@ -0,0 +1,114 @@
+require "util/Colors"
+
+module Cumulus
+  module Common
+    # Public: The types of changes common to all Diffs
+    module DiffChange
+      @@current = 0
+
+      # Public: Produce the next id for a change type. Use this to avoid id
+      # collisions.
+      #
+      # Returns the new id
+      def self.next_change_id
+        @@current += 1
+        @@current
+      end
+
+      ADD = next_change_id
+      UNMANAGED = next_change_id
+      MODIFIED = next_change_id
+    end
+
+    # Public: The base class for all Diff classes.
+    #
+    # To extend this class, do the following:
+    #
+    # 1. Provide a `diff_string` method. This method will be called if the default
+    #    to_s method cannot produce a result.
+    # 2. Provide a `asset_type` method. This method should return the string type of
+    #    asset for which this is a diff.
+    # 3. Provide an `aws_name` method. This method should give back the string name
+    #    of the aws asset.
+    # 4. (Optional) Replace the existing `local_name` method. This method produces the string name
+    #    of the local asset. Defaults to `name` on the local asset.
+    class Diff
+      include DiffChange
+
+      attr_reader :aws, :local, :type
+      attr_accessor :changes, :info_only
+
+      # Public: Static method that will produce an "unmanaged" diff
+      #
+      # aws - the aws resource that is unmanaged
+      #
+      # Returns the diff
+      def self.unmanaged(aws)
+        self.new(UNMANAGED, aws)
+      end
+
+      # Public: Static method that will produce an "added" diff
+      #
+      # local - the local configuration that is added
+      #
+      # Returns the diff
+      def self.added(local)
+        self.new(ADD, nil, local)
+      end
+
+      # Public: Static method that will produce a "modified" diff
+      #
+      # local - the local configuration
+      # aws - the aws resource
+      # changes - an object describing what was modified
+      def self.modified(aws, local, changes)
+        self.new(MODIFIED, aws, local, changes)
+      end
+
+      # Public: Constructor
+      #
+      # type  - the type of the difference
+      # aws   - the aws resource that's different (defaults to nil)
+      # local - the local resource that's difference (defaults to nil)
+      # changes - an object to describe what changed in a MODIFIED diff (defaults to nil)
+      def initialize(type, aws = nil, local = nil, changes = nil)
+        @aws = aws
+        @local = local
+        @type = type
+        @changes = changes
+        @info_only = false
+      end
+
+      def to_s
+        case @type
+        when ADD
+          Colors.added("#{asset_type} #{local_name} #{add_string}")
+        when UNMANAGED
+          Colors.unmanaged("#{asset_type} #{aws_name} #{unmanaged_string}")
+        else
+          diff_string
+        end
+      end
+
+      # Public: A method that produces the string that describes what will be done with new assets.
+      # This can be overridden for the case that the ADD case doesn't create the asset.
+      #
+      # Returns the string describing the action that will be taken.
+      def add_string
+        "will be created."
+      end
+
+      # Public: A method that produces the string that describes what will be done with unmanaged
+      # assets.  This can be overriden for the case that the UNMANAGED case does not ignore the asset.
+      #
+      # Returns the string describing the action that will be taken
+      def unmanaged_string
+        "is not managed by Cumulus."
+      end
+
+      def local_name
+        @local.name
+      end
+    end
+  end
+end

data/lib/common/models/ListChange.rb

@@ -0,0 +1,21 @@
+module Cumulus
+  module Common
+    ListChange = Struct.new(:added, :removed, :modified) do
+      # Public: Creates a ListChange from aws and local arrays with simple types
+      # where the ListChange only has added and removed
+      def self.simple_list_diff(aws, local)
+        added = local - aws
+        removed = aws - local
+
+        if !added.empty? or !removed.empty?
+          ListChange.new(added, removed, nil)
+        end
+      end
+
+      # Public: Returns true if all of added, removed, and modified are either nil or empty
+      def empty?
+        (self.added.nil? or self.added.empty?) and (self.removed.nil? or self.removed.empty?) and (self.modified.nil? or self.modified.empty?)
+      end
+    end
+  end
+end

data/lib/common/models/TagsDiff.rb

@@ -0,0 +1,55 @@
+module Cumulus
+  module Common
+    # Public: A module to be mixed in to Diff classes that have tags
+    module TagsDiff
+      # Public: Get the diff string for tag differences
+      #
+      # Returns the string to display
+      def tags_diff_string
+        lines = ["Tags:"]
+        lines << tags_to_remove.map { |k, v| "\t#{Colors.removed("#{k} => #{v}")}" }
+        lines << tags_to_add.map { |k, v| "\t#{Colors.added("#{k} => #{v}")}" }
+        lines.flatten.join("\n")
+      end
+
+      # Public: Get the tags that are in AWS that are not in local configuration
+      #
+      # Returns a hash of tags
+      def tags_to_remove
+        aws_tags.reject { |t, v| local_tags.include?(t) and local_tags[t] == v }
+      end
+
+      # Public: Get the tags that are in local configuration but not in AWS
+      #
+      # Returns a hash of tags
+      def tags_to_add
+        local_tags.reject { |t, v| aws_tags.include?(t) and aws_tags[t] == v }
+      end
+
+      private
+
+      # Internal: Override this method if the tags are not found on the tags attribute
+      # of the local object
+      #
+      # Returns the tags
+      def local_tags
+        @local.tags
+      end
+
+      # Internal: Get the tags in AWS as a hash of key to value
+      #
+      # Returns a hash of tags
+      def aws_tags
+        @aws_tags ||= Hash[aws_tags_list.map { |tag| [tag.key, tag.value] }]
+      end
+
+      # Internal: Override this method if tags are not found on the tags attribute of
+      # the aws object.
+      #
+      # Returns the tags
+      def aws_tags_list
+        @aws.tags
+      end
+    end
+  end
+end

data/lib/common/models/UTCTimeSource.rb

@@ -0,0 +1,17 @@
+module Cumulus
+  module Common
+    class UTCTimeSource
+
+      # Make now always return now in UTC
+      def now
+        Time.now.utc
+      end
+
+      # Make local always use utc time
+      def local(*args)
+        Time.utc(*args)
+      end
+
+    end
+  end
+end

data/lib/conf/Configuration.rb

@@ -0,0 +1,365 @@
+require "json"
+require "aws-sdk"
+
+module Cumulus
+  # Public: A module that contains helper methods for the configuration classes.
+  #
+  # When mixing in this module, make sure your class has a @node instance variable
+  # for what node in the json it expect to get config from, ie. "s3" or "iam"
+  module Config
+    @@json = nil
+    @@conf_dir = nil
+
+    class << self
+      def json
+        @@json
+      end
+
+      def json=(value)
+        @@json = value
+      end
+
+      def conf_dir
+        @@conf_dir
+      end
+
+      def conf_dir=(value)
+        @@conf_dir = value
+      end
+    end
+
+    private
+
+    # Internal: Take a path relative to the project root and turn it into an
+    # absolute path
+    #
+    # relative_path - The String path from `conf_dir` to the desired file
+    #
+    # Returns the absolute path as a String
+    def absolute_path(relative_path)
+      if relative_path.start_with?("/")
+        relative_path
+      else
+        File.join(@@conf_dir, relative_path)
+      end
+    end
+
+    # Internal: Handle any KeyErrors that occur while getting a configuration value
+    # by printing out a message describing the missing key and exiting.
+    #
+    # key     - the full key to get ex. `s3.buckets.directory`
+    # allow_missing - if true we will return nil for missing values instead of exiting
+    # handler - a block that will do additional processing on the key. If nil,
+    #           the value is returned as is.
+    #
+    # Returns the configuration value if successful
+    def conf(key, allow_missing = false, &handler)
+      value = nil
+      key.split(".").each do |part|
+        if value
+          value = value.fetch(part)
+        else
+          value = @@json.fetch(part)
+        end
+      end
+
+      if handler
+        handler.call(value)
+      else
+        value
+      end
+    rescue KeyError => e
+      puts "Your configuration file is missing $.#{key}."
+      if allow_missing
+        nil
+      else
+        exit
+      end
+    end
+
+  end
+
+  # Public: Contains the configuration values set in the configuration.json file.
+  # Provides a Singleton that can be accessed throughout the application.
+  class Configuration
+    include Config
+
+    attr_reader :colors_enabled
+    attr_reader :iam, :autoscaling, :route53, :s3, :security, :cloudfront, :elb, :vpc, :kinesis, :sqs, :ec2
+    attr_reader :client
+
+    # Internal: Constructor. Sets up the `instance` variable, which is the access
+    # point for the Singleton.
+    #
+    # conf_dir  - The String path to the directory the configuration can be found in
+    # profile       - The String profile name that will be used to make AWS API calls
+    # assume_role - The ARN of the role to assume when making AWS API calls
+    # autoscaling_force_size
+    #               -  Determines whether autoscaling should use configured values for
+    #                  min/max/desired group size
+    def initialize(conf_dir, profile, assume_role, autoscaling_force_size)
+      Config.conf_dir = conf_dir;
+      Config.json = JSON.parse(File.read(absolute_path("configuration.json")))
+      @colors_enabled = conf "colors-enabled"
+      @iam = IamConfig.new
+      @autoscaling = AutoScalingConfig.new(autoscaling_force_size)
+      @route53 = Route53Config.new
+      @security = SecurityConfig.new
+      @cloudfront = CloudFrontConfig.new
+      @s3 = S3Config.new
+      @elb = ELBConfig.new
+      @vpc = VpcConfig.new
+      @kinesis = KinesisConfig.new
+      @sqs = SQSConfig.new
+      @ec2 = EC2Config.new
+
+      region = conf "region"
+      credentials = if assume_role
+        Aws::AssumeRoleCredentials.new(
+          client: Aws::STS::Client.new(profile: profile, region: region),
+          role_arn: assume_role,
+          role_session_name: "#{region}-#{@profile}"
+        )
+      end
+
+      @client = {
+        :region => region,
+        :profile => profile,
+        :credentials => credentials,
+      }.reject { |_, v| v.nil? }
+    end
+
+    class << self
+      # Public: Initialize the Configuration Singleton. Must be called before any
+      # access to `Configuration.instance` is used.
+      #
+      # conf_dir  - The String path to the directory the configuration can be found in
+      # profile       - The String profile name that will be used to make AWS API calls
+      # assume_role - The ARN of the role to assume when making AWS API calls
+      # autoscaling_force_size
+      #               -  Determines whether autoscaling should use configured values for
+      #                  min/max/desired group size
+      def init(conf_dir, profile, assume_role, autoscaling_force_size)
+        instance = new(conf_dir, profile, assume_role, autoscaling_force_size)
+        @@instance = instance
+      end
+
+      # Public: The Singleton instance of Configuration.
+      #
+      # Returns the Configuration instance.
+      def instance
+        @@instance
+      end
+
+      private :new
+    end
+
+    # Public: Inner class that contains IAM configuration options
+    class IamConfig
+      include Config
+
+      attr_reader :groups_directory
+      attr_reader :policy_document_directory
+      attr_reader :policy_prefix
+      attr_reader :policy_suffix
+      attr_reader :policy_version
+      attr_reader :roles_directory
+      attr_reader :static_policy_directory
+      attr_reader :template_policy_directory
+      attr_reader :users_directory
+
+      # Public: Constructor.
+      def initialize
+        @groups_directory = absolute_path "iam/groups"
+        @policy_document_directory = absolute_path "iam/roles/policy-documents"
+        @policy_prefix = conf "iam.policies.prefix"
+        @policy_suffix = conf "iam.policies.suffix"
+        @policy_version = conf "iam.policies.version"
+        @roles_directory = absolute_path "iam/roles"
+        @static_policy_directory = absolute_path "iam/policies/static"
+        @template_policy_directory = absolute_path "iam/policies/template"
+        @users_directory = absolute_path "iam/users"
+      end
+    end
+
+    # Public: Inner class that contains AutoScaling configuration options
+    class AutoScalingConfig
+      include Config
+
+      attr_reader :groups_directory
+      attr_reader :override_launch_config_on_sync
+      attr_reader :static_policy_directory
+      attr_reader :template_policy_directory
+      attr_reader :force_size
+
+      # Public: Constructor.
+      def initialize(force_size = false)
+        @groups_directory = absolute_path "autoscaling/groups"
+        @override_launch_config_on_sync = conf "autoscaling.groups.override-launch-config-on-sync"
+        @static_policy_directory = absolute_path "autoscaling/policies/static"
+        @template_policy_directory = absolute_path "autoscaling/policies/templates"
+        @force_size = force_size
+      end
+
+    end
+
+    # Public: Inner class that contains Route53 configuration options
+    class Route53Config
+      include Config
+
+      attr_reader :includes_directory
+      attr_reader :print_all_ignored
+      attr_reader :zones_directory
+
+      # Public: Constructor
+      def initialize
+        @includes_directory = absolute_path "route53/includes"
+        @print_all_ignored = conf "route53.print-all-ignored"
+        @zones_directory = absolute_path "route53/zones"
+      end
+    end
+
+    # Public: Inner class that contains S3 configuration options
+    class S3Config
+      include Config
+
+      attr_reader :buckets_directory
+      attr_reader :cors_directory
+      attr_reader :policies_directory
+      attr_reader :print_progress
+
+      # Public: Constructor
+      def initialize
+        @node = "s3"
+        @buckets_directory = absolute_path "s3/buckets"
+        @cors_directory = absolute_path "s3/cors"
+        @policies_directory = absolute_path "s3/policies"
+        @print_progress = conf "s3.print-progress"
+      end
+    end
+
+    # Public: Inner class that contains Security Group configuration options
+    class SecurityConfig
+      include Config
+
+      attr_reader :groups_directory
+      attr_reader :rules_directory
+      attr_reader :outbound_default_all_allowed
+      attr_reader :subnet_files
+
+      # Public: Constructor.
+      def initialize
+        @groups_directory = absolute_path "security-groups/groups"
+        @rules_directory = absolute_path "security-groups/rules"
+        @outbound_default_all_allowed = conf "security.outbound-default-all-allowed"
+        @subnet_files = conf("security.subnet-files", true) { |paths| paths.map{ |p| absolute_path(p) } }
+
+        if !@subnet_files
+          default_file = absolute_path("security-groups/subnets.json")
+          @subnet_files = [default_file]
+          puts "Using default subnets file at #{default_file}"
+        end
+
+      end
+
+    end
+
+    # Public: Inner class that contains cloudfront configuration options
+    class CloudFrontConfig
+      include Config
+
+      attr_reader :distributions_directory
+      attr_reader :invalidations_directory
+
+      def initialize
+        @distributions_directory = absolute_path "cloudfront/distributions"
+        @invalidations_directory = absolute_path "cloudfront/invalidations"
+      end
+    end
+
+    # Public: Inner class that contains elb configuration options
+    class ELBConfig
+      include Config
+
+      attr_reader :load_balancers_directory
+      attr_reader :listeners_directory
+      attr_reader :policies_directory
+
+      def initialize
+        @load_balancers_directory = absolute_path "elb/load-balancers"
+        @listeners_directory = absolute_path "elb/listeners"
+        @policies_directory = absolute_path "elb/policies"
+      end
+    end
+
+    # Public: Inner class that contains vpc configuration options
+    class VpcConfig
+      include Config
+
+      attr_reader :vpcs_directory
+      attr_reader :subnets_directory
+      attr_reader :route_tables_directory
+      attr_reader :policies_directory
+      attr_reader :network_acls_directory
+
+      def initialize
+        @vpcs_directory = absolute_path "vpc/vpcs"
+        @subnets_directory = absolute_path "vpc/subnets"
+        @route_tables_directory = absolute_path "vpc/route-tables"
+        @policies_directory = absolute_path "vpc/policies"
+        @network_acls_directory = absolute_path "vpc/network-acls"
+      end
+    end
+
+    # Public: Inner class that contains kinesis configuration options
+    class KinesisConfig
+      include Config
+
+      attr_reader :directory
+
+      def initialize
+        @directory = absolute_path "kinesis"
+      end
+
+    end
+
+    # Public: Inner class that contains SQS configuration options
+    class SQSConfig
+      include Config
+
+      attr_reader :queues_directory
+      attr_reader :policies_directory
+
+      def initialize
+        @queues_directory =  absolute_path "sqs/queues"
+        @policies_directory = absolute_path "sqs/policies"
+      end
+    end
+
+    # Public: Inner class that contains EC2 configuration options
+    class EC2Config
+      include Config
+
+      attr_reader :ebs_directory
+      attr_reader :instances_directory
+      attr_reader :ignore_unmanaged_instances
+      attr_reader :user_data_directory
+      attr_reader :default_image_id
+      attr_reader :volume_mount_base
+      attr_reader :volume_mount_start
+      attr_reader :volume_mount_end
+
+      def initialize
+        @ebs_directory = absolute_path "ec2/ebs"
+        @instances_directory = absolute_path "ec2/instances"
+        @user_data_directory = absolute_path "ec2/user-data-scripts"
+        @ignore_unmanaged_instances = conf "ec2.instances.ignore-unmanaged"
+        @default_image_id = conf "ec2.instances.default-image-id"
+        @volume_mount_base = conf "ec2.instances.volume-mounting.base"
+        @volume_mount_start = conf "ec2.instances.volume-mounting.start"
+        @volume_mount_end = conf "ec2.instances.volume-mounting.end"
+      end
+    end
+
+  end
+end