furnish-aws 0.0.1

data/lib/furnish/provisioners/security_group.rb

@@ -0,0 +1,380 @@
+require 'furnish/provisioners/aws'
+
+module Furnish # :nodoc:
+  module Provisioner # :nodoc:
+    #
+    # Provision an EC2 Security Group. Currently receives no input and yields
+    # the group id created as output.
+    #
+    # See the attributes and constructor for more information about creating
+    # this provisioner.
+    #
+    class SecurityGroup < AWS
+
+      ##
+      # :attr: region
+      #
+      # Provided by Furnish::Provisioner::AWS#region -- required for this provisioner.
+      #
+
+      ##
+      # :attr: group_name
+      #
+      # Name of the group. Supply a string to name it explicitly, or :auto (the
+      # default).  If :auto, will pick a name for you based on #group_prefix
+      # and a random string of numbers that do not already correspond to a
+      # group.
+      #
+      furnish_property :group_name,
+        "Name of the group. Supply a string to name it. Default is :auto -- will pick a name for you and overwrite this accessor."
+
+      ##
+      # :attr: group_prefix
+      #
+      # If #group_name is :auto, will use this value as a prefix. See
+      # #group_name for more information.
+      #
+      furnish_property :group_prefix,
+        "If :group_name is :auto, will use this value as prefix, and a string of random numbers as the postfix. Default is 'furnish-auto-'",
+        String
+
+      ##
+      # :attr: description
+      #
+      # String description of the group. Optional.
+      #
+      furnish_property :description,
+        "Description of the group. Optional",
+        String
+
+      ##
+      # :attr: vpc
+      #
+      # VPC identifier. Optional, has significant effect on how security groups
+      # work. See the EC2 and VPC documentation for details.
+      #
+      furnish_property :vpc,
+        "VPC identifier. Optional, see EC2 and VPC documentation for details.",
+        String
+
+      ##
+      # :attr: kill_instances
+      #
+      # If true (the default), at shutdown time will terminate any instances
+      # that belong to the group provisioned by this provisioner so the
+      # security group can be destroyed.
+      #
+      furnish_property :kill_instances,
+        "If true (default), on shutdown will force termination of all instances in the security group."
+
+      ##
+      # :attr: ingress
+      #
+      # Hash containing two keys: :authorize and :revoke, which each contain
+      # array of arrays. Passed to authorize_ingress and revoke_ingress
+      # respectively.
+      #
+      # Example:
+      #
+      #     Furnish::Provisioners::SecurityGroup.new(
+      #       :ingress => {
+      #         :authorize => [
+      #           [:tcp, 22]
+      #           [:tcp, (1024..1234)]
+      #           [:tcp, 53, "127.0.0.1/32"]
+      #         ]
+      #       }
+      #     )
+      #
+      furnish_property :ingress,
+        "Hash containing two keys: :authorize and :revoke, which each contain array of arrays. Passed to authorize_ingress and revoke_ingress respectively.",
+        Hash
+
+      ##
+      # :attr: egress
+      #
+      # Same as #ingress but for egress rules. Note that egress rules only work
+      # against groups that belong to a VPCs.
+      #
+      furnish_property :egress,
+        "Same as :ingress, but only work on VPCs.",
+        Hash
+
+      ##
+      # :attr: allow_ping
+      #
+      # If true, allow ping from all hosts. If an array of strings, allow ping
+      # from that set of CIDR networks.
+      #
+      furnish_property :allow_ping,
+        "If true, allow ping from all hosts, if array of strings, treats them as CIDR networks."
+
+      ##
+      # :attr: disallow_ping
+      #
+      # Inverse of #allow_ping.
+      #
+      furnish_property :disallow_ping,
+        "Inverse of :allow_ping."
+
+      ##
+      # the group_id as returned from EC2, only set after provisioning has
+      # occurred.
+      attr_reader :group_id
+
+      #
+      # Construct a new security group provisioner.
+      #
+      # Example:
+      #
+      #     obj = Furnish::Provisioners::SecurityGroup.new(
+      #       :access_key => "foo",
+      #       :secret_key => "bar",
+      #       :region => "my-region"
+      #     )
+      #
+      # See Furnish::Provisioner::AWS for how constructor arguments and
+      # attributes interact.
+      #
+      def initialize(args)
+        super
+        check_region
+
+        @ingress        ||= { :authorize => [], :revoke => [] }
+        @egress         ||= { :authorize => [], :revoke => [] }
+        @allow_ping     ||= []
+        @disallow_ping  ||= []
+        @group_name     ||= :auto
+        @group_prefix   ||= "furnish-auto-"
+        @kill_instances   = args.has_key?(:kill_instances) ? args[:kill_instances] : true
+        @group_id         = nil
+      end
+
+      #
+      # Generate a group name (only used if :auto is assigned to #group_name).
+      # Ensures generated names are not already in use. Returns the value it
+      # ends up with.
+      #
+      def generate_group_name
+        unless group_prefix
+          raise ArgumentError, "group_prefix must be set when auto-generating security group names"
+        end
+
+        tmp_name = nil
+
+        loop do
+          tmp_name = group_prefix + (0..rand(10).to_i).map { rand(0..9).to_s }.join("")
+
+          if_debug(3) do
+            puts "Seeing if security group name #{tmp_name} is taken"
+          end
+
+          break unless find_group_by_name(tmp_name)
+
+          if_debug(3) do
+            puts "group exists, waiting to try again"
+          end
+
+          sleep 0.3
+        end
+
+        return tmp_name
+      end
+
+      #
+      # Applies network rules (see #ingress, #egress, #allow_ping,
+      # #disallow_ping attributes) to a created security group.
+      #
+      # Raises ArgumentError if security groups do not belong to a VPC and
+      # egress filters are set.
+      #
+      def apply_group_rules(group)
+        # XXX this meta programs passing the contents set on the object to the
+        #     group object.
+        [:allow_ping, :disallow_ping].each do |sym|
+          if send(sym).kind_of?(Array)
+            ary = send(sym)
+
+            if_debug(3) do
+              puts "group #{group.id}/#{group.name}: applying #{sym} to #{ary.inspect}"
+            end
+
+            group.send(sym, *ary)
+          elsif send(sym)
+            if_debug(3) do
+              puts "group #{group.id}/#{group.name}: applying #{sym} to all instances"
+            end
+
+            group.send(sym) # XXX "everybody"
+          end
+        end
+
+        # XXX this mess is similar to above, but generates the call_name as well.
+        # e.g.: "authorize_egress" comes from @egress[:authorize], and each
+        # inner array in the data structure is expanded as args in the call.
+        [:authorize, :revoke].each do |part|
+          [:ingress, :egress].each do |type|
+            if send(type) and ary = send(type)[part] and ary.kind_of?(Array) and !ary.empty?
+              if type == :egress and !group.vpc?
+                raise ArgumentError, "egress filtering was configured for #{group.id}/#{group.name} and it is not a VPC security group"
+              end
+
+              ary.each do |rule|
+                if_debug(3) do
+                  puts "group #{group.id}/#{group.name}: applying #{part}_#{type} with values #{rule.inspect}"
+                end
+
+                group.send("#{part}_#{type}", *rule.flatten)
+              end
+            end
+          end
+        end
+      end
+
+      #
+      # Queries the API for all instances, returns any that are not in the
+      # terminated state.
+      #
+      def find_secgroup_running_instances(group)
+        group.instances.select { |i| i.status != :terminated }
+      end
+
+      #
+      # Searches for a group by its name, returns the AWS::EC2::SecurityGroup
+      # instance.
+      #
+      def find_group_by_name(group_name)
+        return nil unless group_name
+        ec2.security_groups.filter('group-name', [group_name.to_s]).first
+      end
+
+      #
+      # Only used during shutdown; returns the group if it exists, otherwise
+      # logs and returns false. Intended to isolate idempotent function.
+      #
+      def load_group_for_shutdown
+        unless group_id
+          prov_name = furnish_group_name
+          this_group_name = group_name
+
+          if_debug(2) do
+            puts "group id for #{this_group_name} did not exist during provisioner shutdown for #{prov_name || "unknown"} -- did this get provisioned?"
+          end
+
+          return false
+        end
+
+        group = ec2.security_groups[group_id]
+
+        unless group and group.exists?
+          this_group_name = group_name
+
+          if_debug(2) do
+            puts "group #{this_group_name} did not exist during shutdown; not attempting to deprovision it and returning success"
+          end
+
+          return false
+        end
+
+        return group
+      end
+
+      #
+      # only used during shutdown; if kill_instances is true, terminates all
+      # the non-terminated machines in the group so it can be deleted. Will
+      # keep trying this until all machines are in a terminated state.
+      #
+      def terminate_instances_for_shutdown(group)
+        running_instances = find_secgroup_running_instances(group)
+
+        if running_instances.count > 0
+          if kill_instances
+            if_debug(3) do
+              puts "group #{group.id}/#{group.name} is flagged to kill instances in its group on deprovision"
+            end
+
+            until (instances = find_secgroup_running_instances(group)).empty?
+              if_debug(1) do
+                puts "Trying to destroy security group #{group.name}, but instances are still bound to it."
+                puts instances.map(&:id).inspect
+                puts "Terminating instances, sleeping, and trying again."
+              end
+
+              instances.each do |i|
+                i.terminate rescue nil
+              end
+
+              sleep 10
+            end
+          else
+            raise "instances #{running_instances.map(&:id).inspect} are still running for group #{group.id}/#{group.name} and kill_instances is off. Cannot continue."
+          end
+        end
+      end
+
+      #
+      # Provision a security group. Group name generation happens here if
+      # #group_name is set to :auto, if the group already exists, it will be
+      # used and then rules will be applied to it. Returns (and sets on the
+      # object) the group_id as returned by EC2.
+      #
+      def startup(args={})
+        # FIXME VPC as argument?
+
+        @group_name = generate_group_name if group_name == :auto
+
+        begin
+          group =
+            ec2.security_groups.create(
+              group_name,
+              :description => description,
+              :vpc => vpc
+            )
+        rescue ::AWS::EC2::Errors::InvalidGroup::Duplicate => e
+          group = find_group_by_name(group_name)
+          raise e unless group
+
+          my_name = furnish_group_name # yay instance_eval
+
+          if_debug(1) do
+            puts "#{group.id}/#{group.name} already existed during #{my_name || "ungrouped"} provision -- applying rules and continuing"
+          end
+        end
+
+        apply_group_rules(group)
+        @group_id = group.id
+
+        return({ :security_group_ids => (args[:security_group_ids] || []) + [group_id] })
+      end
+
+      #
+      # Deprovision a security group. If the group does not exist, immediately
+      # returns a true result.
+      #
+      # If #kill_instances is true, it will terminate (and wait for the
+      # termination to succeed) any machines that are using the security group
+      # that are not yet terminated.
+      #
+      # After deleting the group via the API, it will return true if the group
+      # no longer exists.
+      #
+      def shutdown(args={})
+        return { } unless group = load_group_for_shutdown
+
+        terminate_instances_for_shutdown(group)
+        group.delete
+
+        return group.exists? ? false : { }
+      end
+
+      #
+      # Report method as required by Furnish: yields the name of the group,
+      # it's group id, and any vpc information if VPC is in use.
+      #
+      def report
+        vpc_str = vpc ? " vpc_id: #{vpc}" : ""
+        ["name: #{group_name}, id: #{group_id}#{vpc_str}"]
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,61 @@
+require 'bundler/setup'
+
+if ENV["COVERAGE"]
+  require 'simplecov'
+  SimpleCov.start do
+    add_filter '/test/'
+  end
+end
+
+unless File.exist?(".acknowledged")
+  $stderr.puts <<-'EOF'
+   /.--------------.\
+  //                \\
+ //                  \\
+|| .-..----. .-. .--. ||
+||( ( '-..-'|.-.||.-.|||
+|| \ \  ||  || ||||_||||
+||._) ) ||  \'-'/||-' ||
+ \\'-'  `'   `-' `'  //
+  \\                //
+   \\______________//
+    '--------------'
+
+  These tests have no mocks -- they provision real AWS services (machines,
+  security groups, etc) and tear them down.
+
+  It's your job to set your AWS environment variables to an environment that's
+  safe, and bound to a credit card you're comfortable spending money on.
+  Likewise, if the tests fail, there's a high likelihood you'll need to clean
+  machines up manually.
+
+  If you're ok with this, `touch .acknowledged` in the root of this repository
+  and run the tests again.
+
+  EOF
+
+  exit 1
+end
+
+eval File.read('.aws-env') if File.exist?('.aws-env')
+
+unless ENV["AWS_ACCESS_KEY_ID"] and ENV["AWS_SECRET_ACCESS_KEY"]
+  if ENV["AWS_ACCESS_KEY"] and ENV["AWS_SECRET_KEY"]
+    # XXX the above vars are the "new school" and so if we have them and not
+    # the "old school", copy them.
+    ENV["AWS_ACCESS_KEY_ID"]      = ENV["AWS_ACCESS_KEY"]
+    ENV["AWS_SECRET_ACCESS_KEY"]  = ENV["AWS_SECRET_KEY"]
+  else
+    $stderr.puts <<-EOF
+    These tests cannot run without AWS environment variables set.
+
+    You can create a file in the root of the repository called .aws-env that
+    will be loaded as a ruby file to set these values.
+    EOF
+
+    exit 1
+  end
+end
+
+require 'furnish/test'
+require 'minitest/autorun'

data/test/test_aws_base_class.rb

@@ -0,0 +1,30 @@
+require 'helper'
+require 'furnish/provisioners/aws'
+
+class TestAWSProvisioner < Furnish::RunningSchedulerTestCase
+  def setup
+    @klass = Furnish::Provisioner::AWS
+    super
+  end
+
+  def test_constructor
+    assert_raises(ArgumentError, "arguments must be a hash") { @klass.new }
+    assert_raises(ArgumentError, "AWS credentials must be provided to the provisioner.") { @klass.new({}) }
+
+    # XXX should not raise
+    obj = @klass.new(:access_key => "qwe123", :secret_key => "password1")
+
+    assert_equal("qwe123", obj.access_key, "access key is available from object after set in constructor")
+    assert_equal("password1", obj.secret_key, "secret key is available from object after set in constructor")
+  end
+
+  def test_ec2
+    obj = @klass.new(:access_key => "qwe123", :secret_key => "password1", :region => "us-east-1")
+    assert_kind_of(AWS::EC2::Region, obj.ec2)
+    refute_nil(obj.region, "region is non-nil")
+    assert_equal(obj.region, obj.ec2.name, "region propogates to ec2 object")
+
+    obj.region = "us-west-1"
+    assert_equal(obj.region, obj.ec2.name, "region propogates to ec2 object after being changed")
+  end
+end