cumulus-aws 0.11.1

data/lib/security/SecurityGroups.rb

@@ -0,0 +1,39 @@
+require "conf/Configuration"
+require "ec2/EC2"
+
+require "aws-sdk"
+
+module Cumulus
+  module SecurityGroups
+    class << self
+      @@client = Aws::EC2::Client.new(Configuration.instance.client)
+
+      require "aws_extensions/ec2/SecurityGroup"
+      Aws::EC2::Types::SecurityGroup.send(:include, AwsExtensions::EC2::SecurityGroup)
+
+      def id_security_groups
+        @id_security_groups ||= Hash[security_groups.map { |a| [a.group_id, a] }]
+      end
+
+      # Public: Returns a Hash of vpc id to Hash of security group name to group
+      def vpc_security_groups
+        @vpc_security_groups ||= Hash[security_groups.map(&:vpc_id).uniq.map do |vpc_id|
+          [vpc_id, Hash[security_groups.select { |g| g.vpc_id == vpc_id }.map { |g| [g.group_name, g] }]]
+        end]
+      end
+
+      # Describe all security groups
+      def security_groups
+        @security_groups ||= @@client.describe_security_groups.security_groups
+      end
+
+      # Public: Returns a Hash of vpc id to Hash of security group id to group name
+      def vpc_security_group_id_names
+        @vpc_security_group_id_names ||= Hash[vpc_security_groups.map do |vpc_id, group_hash|
+          [vpc_id, Hash[group_hash.map {|_, sg| [sg.group_id, sg.group_name]}]]
+        end]
+      end
+
+    end
+  end
+end

data/lib/security/loader/Loader.rb

@@ -0,0 +1,94 @@
+require "common/BaseLoader"
+require "conf/Configuration"
+require "security/models/SecurityGroupConfig"
+require "util/Colors"
+require "util/StatusCodes"
+
+module Cumulus
+  module SecurityGroups
+    # Public: Load Security Group assets
+    module Loader
+      include Common::BaseLoader
+
+      @@groups_dir = Configuration.instance.security.groups_directory
+      @@rules_dir = Configuration.instance.security.rules_directory
+      @@subnet_files = Configuration.instance.security.subnet_files
+
+      # Public: Load all the security group configurations as SecurityGroupConfig objects
+      #
+      # Returns an array of SecurityGroupConfig
+      def Loader.groups
+        # List all the directories to load groups from each vpc
+        vpc_dirs = Dir.entries(@@groups_dir).reject { |f| f == "." or f == ".."}.select { |f| File.directory?(File.join(@@groups_dir, f)) }
+
+        vpc_groups = vpc_dirs.map do |d|
+          aws_vpc = EC2::named_vpcs[d]
+
+          if aws_vpc.nil?
+            puts Colors.red("No VPC named #{d} exists")
+            exit StatusCodes::EXCEPTION
+          end
+
+          Common::BaseLoader.resources(File.join(@@groups_dir, d)) do |file_name, json|
+            name = "#{aws_vpc.name}/#{file_name}"
+            SecurityGroupConfig.new(name, aws_vpc.vpc_id, json)
+          end
+        end.flatten
+
+        non_vpc_groups = Common::BaseLoader.resources(@@groups_dir) do |file_name, json|
+          SecurityGroupConfig.new(file_name, nil, json)
+        end
+
+        if !EC2::supports_ec2_classic and !non_vpc_groups.empty?
+          puts "Ignoring Non-VPC Security Groups because your account does not support them"
+          non_vpc_groups = []
+        end
+
+        vpc_groups + non_vpc_groups
+      end
+
+      # Public: Load a single static rule
+      #
+      # Returns the static rule as json
+      def Loader.rule(rule_name)
+        Common::BaseLoader.resource(rule_name, @@rules_dir) { |_, json| json }
+      end
+
+      # Public: Get the local definition of a subnet group.
+      #
+      # name - the name of the subnet group to get
+      #
+      # Returns an array of ip addresses that is empty if there is no subnet group with that name
+      def Loader.subnet_group(name)
+        if self.subnet_groups[name].nil?
+          raise "Could not find subnet #{name}"
+        else
+          self.subnet_groups[name]
+        end
+      end
+
+      private
+
+      # Internal: Get the subnet group definitions
+      #
+      # Returns a hash that maps group name to an array of ips
+      def Loader.subnet_groups
+        @subnet_groups ||= self.load_subnet_groups
+      end
+
+      # Internal: Load the subnet group definitions
+      #
+      # Returns a hash that maps group name to an array of ips
+      def Loader.load_subnet_groups
+        @@subnet_files.reduce({}) do |sofar, f|
+          subnet_group = Common::BaseLoader.resource(f, "") { |_, json| json }
+          if subnet_group
+            subnet_group.merge(sofar)
+          else
+            sofar
+          end
+        end
+      end
+    end
+  end
+end

data/lib/security/manager/Manager.rb

@@ -0,0 +1,246 @@
+require "common/manager/Manager"
+require "conf/Configuration"
+require "security/loader/Loader"
+require "security/models/SecurityGroupConfig"
+require "security/models/SecurityGroupDiff"
+require "security/SecurityGroups"
+require "util/Colors"
+
+require "aws-sdk"
+require "json"
+
+module Cumulus
+  module SecurityGroups
+    class Manager < Common::Manager
+      def initialize
+        super()
+        @ec2 = Aws::EC2::Client.new(Configuration.instance.client)
+      end
+
+      # Public: Migrate AWS Security Groups to Cumulus configuration.
+      def migrate
+        groups_dir = "#{@migration_root}/security-groups"
+
+        if !Dir.exists?(@migration_root)
+          Dir.mkdir(@migration_root)
+        end
+        if !Dir.exists?(groups_dir)
+          Dir.mkdir(groups_dir)
+        end
+
+        # Make the directories needed for resources that require it
+        aws_resources.map do |name, _|
+          parts = name.partition("/")
+          if parts.length > 1
+            "#{groups_dir}/#{parts.first}"
+          end
+        end.uniq.compact.each do |dir|
+          if !Dir.exists?(dir)
+            Dir.mkdir(dir)
+          end
+        end
+
+        aws_resources.each_value do |resource|
+          puts "Processing #{resource.vpc_group_name}..."
+          config = SecurityGroupConfig.new(resource.vpc_group_name, resource.vpc_id)
+          config.populate!(resource)
+
+          puts "Writing #{resource.vpc_group_name} configuration to file..."
+          File.open("#{groups_dir}/#{config.name}.json", "w") { |f| f.write(config.pretty_json) }
+        end
+
+        File.open("#{@migration_root}/subnets.json", "w") do |f|
+          f.write(JSON.pretty_generate({
+            "all" => ["0.0.0.0/0"]
+          }))
+        end
+
+        puts Colors.blue("IP addresses for inbound and outbound rules have been left as is in each individual security group, except in the case of 0.0.0.0/0.")
+        puts Colors.blue("0.0.0.0/0 has been renamed to 'all' and is referenced as such in security group definitions.")
+        puts Colors.blue("See subnets.json to see the definition of the 'all' subnet group.")
+      end
+
+      def resource_name
+        "Security Group"
+      end
+
+      def local_resources
+        @local_resources ||= Hash[Loader.groups.map { |local| [local.name, local] }]
+      end
+
+      # Hash the aws security groups using the vpc and security group name
+      def aws_resources
+        @aws_resources ||= Hash[SecurityGroups::security_groups.map { |sg| [sg.vpc_group_name, sg] }]
+      end
+
+      def unmanaged_diff(aws)
+        SecurityGroupDiff.unmanaged(aws)
+      end
+
+      def added_diff(local)
+        SecurityGroupDiff.added(local)
+      end
+
+      def diff_resource(local, aws)
+        local.diff(aws)
+      end
+
+      def create(local)
+        result = @ec2.create_security_group({
+          group_name: local.name.split("/").last,
+          description: local.description,
+          vpc_id: local.vpc_id,
+        })
+        security_group_id = result.group_id
+
+        update_tags(security_group_id, local.tags, {})
+        update_inbound(local.vpc_id, security_group_id, local.inbound, [])
+
+        allow_all_rule = RuleConfig.allow_all
+        allow_all_outbound = Configuration.instance.security.outbound_default_all_allowed or local.outbound.find { |g| g.hash == allow_all_rule.hash }
+
+        outbound_remove = if allow_all_outbound
+          []
+        else
+          [allow_all_rule]
+        end
+
+        outbound_add = local.outbound.reject { |g| g.hash == allow_all_rule.hash }
+
+        update_outbound(local.vpc_id, security_group_id, outbound_add, outbound_remove)
+      end
+
+      def update(local, diffs)
+        diffs_by_type = diffs.group_by(&:type)
+
+        if diffs_by_type.include?(SecurityGroupChange::DESCRIPTION)
+          puts "\tUnfortunately, AWS's SDK does not allow updating the description."
+        else
+          diffs.each do |diff|
+            case diff.type
+            when SecurityGroupChange::TAGS
+              update_tags(diff.aws.group_id, diff.tags_to_add, diff.tags_to_remove)
+            when SecurityGroupChange::INBOUND
+              update_inbound(local.vpc_id, diff.aws.group_id, diff.added_inbounds, diff.removed_inbounds)
+            when SecurityGroupChange::OUTBOUND
+              update_outbound(local.vpc_id, diff.aws.group_id, diff.added_outbounds, diff.removed_outbounds)
+            end
+          end
+        end
+      end
+
+      private
+
+      # Internal: Update the tags associated with a security group.
+      #
+      # security_group_id - the id of the security group to update
+      # add               - the tags to add (expects a hash of key value pairs)
+      # remove            - the tags to remove (expects a hash of key value pairs)
+      def update_tags(security_group_id, add, remove)
+        if !add.empty?
+          puts Colors.blue("\tadding tags...")
+          @ec2.create_tags({
+            resources: [security_group_id],
+            tags: add.map { |k, v| { key: k, value: v } }
+          })
+        end
+        if !remove.empty?
+          puts Colors.blue("\tremoving tags...")
+          @ec2.delete_tags({
+            resources: [security_group_id],
+            tags: remove.map { |k, v| { key: k, value: v } }
+          })
+        end
+      end
+
+      # Internal: Update the inbound rules associated with a security group.
+      #
+      # vpc_id -          - the id of the vpc the security groups are in
+      # security_group_id - the id of the security group
+      # add               - the inbound rules to associate with the security group
+      # remove            - the inbound rules to dissociate from the security group
+      def update_inbound(vpc_id, security_group_id, add, remove)
+        update_rules(
+          vpc_id,
+          security_group_id,
+          add,
+          remove,
+          {
+            :type => "inbound",
+            :add_action => @ec2.method(:authorize_security_group_ingress),
+            :remove_action => @ec2.method(:revoke_security_group_ingress)
+          }
+        )
+      end
+
+      # Internal: Update the outbound rules associated with a security group.
+      #
+      # vpc_id -          - the id of the vpc the security groups are in
+      # security_group_id - the id of the security group
+      # add               - the outbound rules to associate with the security group
+      # remove            - the outbound rules to associate with the security group
+      #
+      def update_outbound(vpc_id, security_group_id, add, remove)
+        update_rules(
+          vpc_id,
+          security_group_id,
+          add,
+          remove,
+          {
+            :type => "outbound",
+            :add_action => @ec2.method(:authorize_security_group_egress),
+            :remove_action => @ec2.method(:revoke_security_group_egress)
+          }
+        )
+      end
+
+      # Internal: Update rules associated with a security group. Called by inbound
+      # and outbound.
+      #
+      # vpc_id -          - the id of the vpc the security groups are in
+      # security_group_id - the id of the security group
+      # add               - the rules to associate with the security group
+      # remove            - the rules to remove from the security group
+      # options           - a hash containing options about the operation to run. Should contain:
+      #   - type              - a string representing the type of rules to process
+      #   - add_action        - the client method to call to add rules
+      #   - remove_action     - the client method to call to remove rules
+      def update_rules(vpc_id, security_group_id, add, remove, options)
+        if !add.empty?
+          puts Colors.blue("\tadding #{options[:type]} rules...")
+          options[:add_action].call({
+            group_id: security_group_id,
+            ip_permissions: add.map do |added|
+              missing_group = added.security_groups.find do |s|
+                !SecurityGroups::vpc_security_groups[vpc_id].include?(s)
+              end
+              if missing_group
+                puts Colors.red("\t\tNo such security group: #{missing_group}. Security group not added.")
+              end
+
+              added.to_aws(vpc_id)
+            end
+          })
+        end
+
+        if !remove.empty?
+          puts Colors.blue("\tremoving #{options[:type]} rules...")
+          options[:remove_action].call({
+            group_id: security_group_id,
+            ip_permissions: remove.map do |removed|
+              missing_group = removed.security_groups.find do |s|
+                !SecurityGroups::vpc_security_groups[vpc_id].include?(s)
+              end
+              if missing_group
+                puts Colors.red("\t\tNo such security group: #{missing_group}. Security group not removed.")
+              end
+
+              removed.to_aws(vpc_id)
+            end
+          })
+        end
+
+      end
+    end
+  end
+end

data/lib/security/models/RuleConfig.rb

@@ -0,0 +1,161 @@
+require "security/loader/Loader"
+require "security/SecurityGroups"
+
+module Cumulus
+  module SecurityGroups
+    # Public: An object representing configuration for a security group rule
+    class RuleConfig
+
+      attr_reader :from
+      attr_reader :protocol
+      attr_reader :security_groups
+      attr_reader :subnets
+      attr_reader :to
+
+      # Public: Static method that will produce a RuleConfig from an AWS rule resource.
+      #
+      # aws - the aws resource to use
+      #
+      # Returns a RuleConfig containing the data in the AWS rule
+      def RuleConfig.from_aws(aws)
+        RuleConfig.new({
+          "security-groups" => aws.user_id_group_pairs.map { |security| SecurityGroups::id_security_groups[security.group_id].group_name },
+          "protocol" => if aws.ip_protocol == "-1" then "all" else aws.ip_protocol end,
+          "from-port" => if aws.ip_protocol != "icmp" and aws.from_port != -1 then aws.from_port end,
+          "to-port" => if aws.ip_protocol != "icmp" and aws.to_port != -1 then aws.to_port end,
+          "icmp-type" => if aws.ip_protocol == "icmp"
+            if aws.from_port != -1 then aws.from_port else "all" end
+          end,
+          "icmp-code" => if aws.ip_protocol == "icmp"
+            if aws.to_port != -1 then aws.to_port  else "all" end
+          end,
+          "subnets" => aws.ip_ranges.map { |ip| ip.cidr_ip },
+        }.reject { |k, v| v.nil? })
+      end
+
+      # Public: Static method that will produce a RuleConfig that allows all access
+      #
+      # Returns the RuleConfig
+      def RuleConfig.allow_all
+        RuleConfig.new({
+          "protocol" => "all",
+          "subnets" => ["0.0.0.0/0"]
+        })
+      end
+
+      # Public: Static method that will produce multiple RuleConfigs, one for each port
+      # range.
+      #
+      # json - a hash containing the JSON configuration for the rule
+      #
+      # Returns an array of RuleConfigs
+      def RuleConfig.expand_ports(json)
+        ports = json["ports"]
+
+        if !ports.nil?
+          ports.map do |port|
+            rule_hash = json.clone
+
+            if port.is_a? String
+              parts = port.split("-").map(&:strip)
+              rule_hash["from-port"] = parts[0].to_i
+              rule_hash["to-port"] = parts[1].to_i
+            else
+              rule_hash["from-port"] = port
+              rule_hash["to-port"] = port
+            end
+
+            RuleConfig.new(rule_hash)
+          end
+        else
+          RuleConfig.new(json)
+        end
+      end
+
+      # Public: Constructor
+      #
+      # json - a hash containing the JSON configuration for the rule
+      def initialize(json)
+        @protocol = json["protocol"]
+
+        if @protocol.downcase == "icmp"
+          @from = json["icmp-type"]
+          @to = json["icmp-code"]
+        else
+          @from = json["from-port"]
+          @to = json["to-port"]
+        end
+
+        @security_groups = if !json["security-groups"].nil? then json["security-groups"] else [] end
+        @subnets = if !json["subnets"].nil?
+          json["subnets"].flat_map do |subnet|
+            if subnet.match(/\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}\/\d+/).nil?
+              Loader.subnet_group(subnet)
+            else
+              subnet
+            end
+          end.sort
+        else
+          []
+        end
+      end
+
+      # Public: Get the configuration as a hash
+      #
+      # Returns the hash
+      def hash
+        security_hashes = @security_groups.map do |security_group|
+          {
+            "security-groups" => [security_group],
+            "protocol" => @protocol,
+            "from-port" => if @protocol != "icmp" then @from end,
+            "to-port" => if @protocol != "icmp" then @to end,
+            "subnets" => [],
+            "icmp-type" => if @protocol == "icmp" then @from end,
+            "icmp-code" => if @protocol == "icmp" then @to end,
+          }.reject { |k, v| v.nil? }
+        end
+        subnet_hashes = @subnets.map do |subnet|
+          {
+            "security-groups" => [],
+            "protocol" => @protocol,
+            "from-port" => if @protocol != "icmp" then @from end,
+            "to-port" => if @protocol != "icmp" then @to end,
+            "subnets" => [subnet],
+            "icmp-type" => if @protocol == "icmp" then @from end,
+            "icmp-code" => if @protocol == "icmp" then @to end,
+          }.reject { |k, v| v.nil? }
+        end
+
+        security_hashes + subnet_hashes
+      end
+
+      # Public: Converts the RuleConfig into the format needed by AWS
+      # to authorize/deauthorize rules
+      #
+      # vpc_id - the id of the vpc that security group ids should be derived from
+      def to_aws(vpc_id)
+        {
+          ip_protocol: if @protocol == "all" then "-1" else @protocol end,
+          from_port: if @from == "all" then "-1" else @from end,
+          to_port: if @to == "all" then "-1" else @to end,
+          user_id_group_pairs: if !@security_groups.empty?
+            @security_groups.map do |sg|
+              {
+                group_id: SecurityGroups::vpc_security_group_id_names[vpc_id].key(sg)
+              }
+            end
+          end,
+          ip_ranges: if !@subnets.empty?
+            @subnets.map do |subnet|
+              {
+                cidr_ip: subnet
+              }
+            end
+          end
+        }
+      end
+
+    end
+  end
+end