furnish-aws 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MDdmMmNkNDA4ZTc1NzZiNTQ0MWY0NmRjMTg2ZGZlOTUxOTQ2ZjU0Yw==
+  data.tar.gz: !binary |-
+    MzkwNjE1M2Q5YzAyNDgzMGI5MWRhODg1ZmJjN2I1MGY2N2E0ZGNiMw==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NDUyYTljYTBlMmI3YjRlNTI3ZTA2NjkyMDU5ZGE3MzRkZDMyMzYwMTE0YTI5
+    YTVkMTYwZTExODNmYWEyNGZjNDg0NGVjNDkzZDAyZjUzMmE5MWU4MmQxMjdi
+    OTZkMzkyNmVlN2ViZTM1ZjAyZGE2N2YyYTQ1Y2JiNTliOTJmZDY=
+  data.tar.gz: !binary |-
+    YjMzMGE5NTNhMzAwNWUyYWZmYTdmNTk4ODk1MzlmZTNjZmEwZjdmYTZhMDMy
+    MzI3NWVhZGEyZTNkN2FhNDFiMTY4NmYwMzhlOGY2MzRiMDZhNTNjMDg5Yzg2
+    YjJiYWIyZjY4OTYxY2JkMjMzN2Y4ZmMwMDQ3MmU0NWYzNWY5Zjg=

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.acknowledged
+.aws-env
+html

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in furnish-aws.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,10 @@
+# vim: ft=ruby
+guard 'minitest' do
+  # with Minitest::Unit
+  watch(%r!^test/(.*)\/?test_(.*)\.rb!)
+  watch(%r!^test/helper\.rb!) { "test" }
+end
+
+guard 'rake', :run_on_all => false, :task => 'rdoc_cov' do
+  watch(%r!^lib/(.*)([^/]+)\.rb!)
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Erik Hollensbe
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+# AWS provisioners for Furnish
+
+Several AWS-related provisioners for Furnish:
+
+* EC2 Instances
+* Security Groups
+
+## Usage
+
+For usage, see the documentation for each Provisioner class. For general
+Furnish usage, see the [Furnish documentation](http://rubydoc.info/gems/furnish/).
+
+## Testing
+
+Tests do not use mocks. They are built in a way to do a minimum amount of API
+traffic, but traffic does happen, machines and other resources get provisioned,
+and yes, your card will be charged as a result of that.
+
+Acknowledging that and still willing to test, there are two things you need to
+do to get tests to run:
+
+* `touch .acknowledged` in the root of the repo - this more or less indicates
+  you read the above.
+* open `.aws-env` and put some ruby in it that sets your AWS credentials. This
+  will be evaluated at harness time. This is to ensure you don't do something
+  stupid to production.
+
+e.g.:
+
+```ruby
+ENV["AWS_ACCESS_KEY_ID"]="aaaaah"
+ENV["AWS_SECRET_ACCESS_KEY"]="it's a space herpe"
+```
+
+If you don't do these the test suite will not run. Both files are in
+`.gitignore` so you can feel comfortable knowing they won't be committed.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'furnish-aws'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install furnish-aws
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,32 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+require 'rdoc/task'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList["test/test_*.rb"]
+  t.verbose = true
+end
+
+RDoc::Task.new do |rdoc|
+  rdoc.title = "AWS provisioners for Furnish"
+  rdoc.main = "README.md"
+  rdoc.rdoc_files.include("README.md", "lib/**/*.rb")
+  rdoc.rdoc_files -= ["lib/furnish/aws/version.rb"]
+  if ENV["RDOC_COVER"]
+    rdoc.options << "-C"
+  end
+end
+
+desc "run tests with coverage report"
+task "test:coverage" do
+  ENV["COVERAGE"] = "1"
+  Rake::Task["test"].invoke
+end
+
+desc "run rdoc with coverage report"
+task :rdoc_cov do
+  # ugh
+  ENV["RDOC_COVER"] = "1"
+  ruby "-S rake rerdoc"
+end

data/furnish-aws.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'furnish/aws/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "furnish-aws"
+  gem.version       = Furnish::AWS::VERSION
+  gem.authors       = ["Erik Hollensbe"]
+  gem.email         = ["erik+github@hollensbe.org"]
+  gem.description   = %q{AWS provisioners for Furnish}
+  gem.summary       = %q{AWS provisioners for Furnish}
+  gem.homepage      = "https://github.com/chef-workflow/furnish-aws"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'furnish', '~> 0.1.0'
+  gem.add_dependency 'aws-sdk', '~> 1.8.5'
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'minitest', '~> 4.5.0'
+  gem.add_development_dependency 'guard-minitest'
+  gem.add_development_dependency 'guard-rake', '~> 0.0.8'
+  gem.add_development_dependency 'rdoc', '~> 4'
+  gem.add_development_dependency 'rb-fsevent'
+  gem.add_development_dependency 'simplecov'
+end

data/lib/furnish/aws/version.rb

@@ -0,0 +1,6 @@
+module Furnish
+  module AWS
+    # version of furnish-aws
+    VERSION = "0.0.1"
+  end
+end

data/lib/furnish/provisioners/aws.rb

@@ -0,0 +1,101 @@
+require 'aws'
+require 'furnish/logger'
+require 'furnish/provisioners/api'
+
+module Furnish # :nodoc:
+  module Provisioner # :nodoc:
+    #
+    # AWS base class, handles AWS API basics, argument checking, general
+    # utility methods.
+    #
+    # See attributes and #new for contraints that apply to all
+    # provisioners that inherit from it.
+    #
+    class AWS < API
+      include Furnish::Logger::Mixins
+
+      ##
+      # :attr: access_key
+      #
+      # The AWS access key ID. Required.
+      #
+      furnish_property :access_key,
+        "AWS access key ID",
+        String
+
+      ##
+      # :attr: secret_key
+      #
+      # The AWS secret key ID. Required.
+      #
+      furnish_property :secret_key,
+        "AWS secret key ID",
+        String
+
+      ##
+      # :attr: region
+      #
+      # The AWS region. Not required. The default is nil. Inheriting
+      # provisioners may affect the default and/or make it required.
+      #
+      furnish_property :region,
+        "AWS region -- default is nil, may be defaulted or required by subclasses.",
+        String
+
+      #
+      # Construct a new provisioner. Hash for arguments is required, values
+      # supplied will be injected into attributes. If a hash is not supplied,
+      # an ArgumentError is raised.
+      #
+      # The AWS #access_key and #secret_key are required, and also raise an
+      # ArgumentError unless supplied.
+      #
+      # In some provisioners, a region may also be required. In those, an...
+      # ArgumentError will be raise if it isn't supplied.
+      #
+      # For further constraints, please see provisioners that inherit from this
+      # class.
+      #
+      def initialize(args)
+        super
+        check_aws_settings
+      end
+
+      #
+      # Ensures #access_key and #secret_key are set. Used by initializer.
+      #
+      def check_aws_settings
+        unless access_key and secret_key
+          raise ArgumentError, "AWS credentials must be provided to the provisioner."
+        end
+      end
+
+      #
+      # Ensures the region is set. Is not used by default in the initializer,
+      # but is common to many provisioners.
+      #
+      def check_region
+        unless region
+          raise ArgumentError, "AWS region must be provided to the provisioner."
+        end
+      end
+
+      #
+      # constructs an AWS::EC2 object. if a region is supplied, constraints it
+      # to the region and a AWS::EC2::Region object will be returned.
+      #
+      # these have the same interface for most things, and things that require
+      # a region in AWS::EC2 are defaulted to us-east-1. See the aws-sdk
+      # documentation for more details.
+      #
+      def ec2
+        ec2 = ::AWS::EC2.new(
+          :access_key_id => access_key,
+          :secret_access_key => secret_key
+        )
+
+        return region ? ec2.regions[region] : ec2
+      end
+    end
+  end
+end

data/lib/furnish/provisioners/ec2.rb

@@ -0,0 +1,372 @@
+require 'furnish/provisioners/aws'
+require 'timeout'
+
+module Furnish # :nodoc:
+  module Provisioner # :nodoc:
+    #
+    # EC2 Instance Provisioner. See attributes and #new for argument
+    # descriptions.
+    #
+    # If security groups are supplied to this provisioner as a part of a
+    # provisioner group (in #startup's argument list), they are appended to any
+    # list of security group id's already provided. This allows you to
+    # provision a security group with the instances without knowing about
+    # either beforehand.
+    #
+    # On startup, IP addresses of the instances are returned to the next
+    # provisioner in the provisioner group.
+    #
+    # It would be in your best interest to familiarize yourself with some of
+    # aws-sdk's API when trying to understand some of these parameters. Here is
+    # some documentation:
+    #
+    # http://rdoc.info/gems/aws-sdk
+    #
+    class EC2 < AWS
+      ##
+      # :attr: region
+      #
+      # See Furnish::Provisioner::AWS#region. nil by default, required in this
+      # provisioner. If #availability_zone is supplied, the region will be
+      # extracted from what is set as the AZ.
+      #
+
+      ##
+      # :attr: provision_timeout
+      #
+      # Give up after this long for the requested instances to be in a running
+      # state. Default is 300 seconds.
+      #
+      furnish_property :provision_timeout,
+        "Give up after this long for the instance to come alive after requesting from the API. Default is 300 seconds.",
+        Integer
+
+      ##
+      # :attr: poll_interval
+      #
+      # Wait this long between instance status requests. Default is one second.
+      # Fractional values are OK.
+      #
+      furnish_property :poll_interval,
+        "Wait this long between instance status requests. Default is 1 second, fractional values OK.",
+        Numeric
+
+      ##
+      # :attr: key_name
+      #
+      # The SSH key name for EC2. Must already exists. Required. No Default.
+      #
+      furnish_property :key_name,
+        "EC2 SSH Key Name, must already exist. Required. No Default.",
+        String
+
+      ##
+      # :attr: availability_zone
+      #
+      # EC2 availability zone. If not provided, EC2 will pick based on the
+      # region. If provided, region will be determined from this value.
+      #
+      furnish_property :availability_zone,
+        "EC2 Availability Zone. If provided and no region supplied, region will be determined from this. If this is not supplied, AZ will be EC2's choosing based on the region.",
+        String
+
+      ##
+      # :attr: image_id
+      #
+      # The AMI identifier. Required. No default.
+      #
+      furnish_property :image_id,
+        "AMI Identifier. Required. No Default.",
+        String
+
+      ##
+      # :attr: kernel_id
+      #
+      # The AKI identifier. No default, EC2 picks based on the AMI if not
+      # provided.
+      #
+      furnish_property :kernel_id,
+        "AKI Identifier. If not supplied, left to EC2 to sort out.",
+        String
+
+      ##
+      # :attr: ramdisk_id
+      #
+      # The ARI identifier. No default, EC2 picks based on the AMI and possibly
+      # the AKI if not provided.
+      #
+      furnish_property :ramdisk_id,
+        "ARI Identifier. If not supplied, left to EC2 to sort out.",
+        String
+
+      ##
+      # :attr: count
+      #
+      # The number of instances to provision. Default is 1.
+      #
+      furnish_property :count,
+        "Number of instances to allocate. Required, Default is 1.",
+        Integer
+
+      ##
+      # :attr: instance_type
+      #
+      # The size or "flavor" of instance(s). Required, default is 'c1.medium'.
+      #
+      furnish_property :instance_type,
+        "Size or 'flavor' of instance(s). Required, default is 'c1.medium'."
+        String
+
+      ##
+      # :attr: security_group_ids
+      #
+      # Array of security group identifiers (not *group names*) that these
+      # instances will be bound to. Appends any incoming group id's from
+      # previous provisioners. At least one must exist at provisioning time, or
+      # the provision will fail.
+      #
+      furnish_property :security_group_ids,
+        "list of group identifiers (not names) that these instances will be bound to. Appends any incoming group id's from previous provisioners. At least one must exist at provisioning time.",
+        Array
+
+      ##
+      # :attr: block_device_mappings
+      #
+      # Mapping for block device information. Passed directly to
+      # AWS::EC2::InstanceCollection#create -- see its documentation for more
+      # information.
+      #
+      furnish_property :block_device_mappings,
+        "block device information. Passed directly to AWS::EC2::InstanceCollection#create's argument list. Not required, no default."
+
+      ##
+      # :attr: user_data
+      #
+      # Mapping for user data. Passed directly to
+      # AWS::EC2::InstanceCollection#create -- see its documentation for more
+      # information.
+      #
+      furnish_property :user_data,
+        "user data. Passed directly to AWS::EC2::InstanceCollection#create's argument list. Not required, no default."
+
+      ##
+      # :attr: monitoring_enabled
+      #
+      # Mapping for configuring monitoring. Passed directly to
+      # AWS::EC2::InstanceCollection#create -- see its documentation for more
+      # information.
+      #
+      furnish_property :monitoring_enabled,
+        "enable montioring. Passed directly to AWS::EC2::InstanceCollection#create's argument list. Not required, no default."
+
+
+      ##
+      #
+      # After provisioning, instance identifiers managed by this provisioner
+      # will be set here.
+      #
+      attr_reader   :instance_ids
+
+      ##
+      # Arguments that are passed through straight to
+      # AWS::EC2::InstanceCollection#create.
+      #
+      PASSTHROUGH_ATTRS = %w[
+        key_name
+        availability_zone
+        image_id
+        kernel_id
+        ramdisk_id
+        count
+        block_device_mappings
+        instance_type
+        user_data
+        security_group_ids
+        monitoring_enabled
+      ]
+
+      #
+      #
+      # Create a new EC2 Provisioner. Inherits the initializer from
+      # Furnish::Provisioner::AWS.new, see the documentation there for more
+      # information.
+      #
+      # Additionally, see the attribute listing for argument documentation and
+      # their defaults.
+      #
+      # Required arguments that do not have defaults. ArgumentError will be
+      # raised if they are not supplied at construction time:
+      #
+      # * #access_key (from Furnish::Provisioner::AWS)
+      # * #secret_key (from Furnish::Provisioner::AWS)
+      # * #instance_type
+      # * #image_id
+      # * #region (also see #availability_zone)
+      # * #key_name
+      #
+      # Additionally see #security_group_ids, #poll_interval and
+      # #provision_timeout for some detail on special logic surrounding the
+      # operation of this provisioner.
+      #
+      def initialize(args)
+        super
+
+        check_region
+        check_ec2_args
+
+        @count              ||= 1
+        @instance_type      ||= "c1.medium"
+        @security_group_ids ||= []
+        @provision_wait     ||= 300
+        @poll_interval      ||= 1
+        @instance_ids         = []
+      end
+
+      #
+      # Overload of Furnish::Provisioner::AWS#check_region that also deals with
+      # availability zones.
+      def check_region
+        if availability_zone and !region
+          @region = availability_zone.sub(/\D+$/, '')
+        end
+
+        if availability_zone and region and !availability_zone.start_with?(region)
+          raise ArgumentError,
+            "region and availability zone are both supplied and do not match: [r: #{region}, a: #{availability_zone}]"
+        end
+
+        super
+      end
+
+      #
+      # Validate additional required EC2-specific arguments like #key_name and
+      # #image_id.
+      #
+      def check_ec2_args
+        unless key_name
+          raise ArgumentError, "key_name is required for instance provision"
+        end
+
+        unless image_id
+          raise ArgumentError, "an AMI image_id must be provided"
+        end
+      end
+
+      #
+      # generates options to pass to AWS::EC2::InstanceCollection#create from
+      # our attributes.
+      #
+      def launch_options
+        options = { }
+
+        PASSTHROUGH_ATTRS.each { |x| options[x.to_sym] = send(x) if send(x) }
+
+        return options
+      end
+
+      #
+      # puts the return value from ec2.instances.create into a normal array no
+      # matter what it returns. See the method comments for more information.
+      #
+      def coerce_instances(instances)
+        # if count == 1, returns a single instance id.
+        instances = [instances] unless instances.kind_of?(Array)
+
+        # XXX there are versions of aws-sdk that respond to #kind_of?(Array)
+        # but aren't actually arrays. This will always give us an array.
+
+        tmp_instances = []
+        instances.each { |i| tmp_instances.push(i) }
+        return tmp_instances
+      end
+
+      #
+      # Polls the EC2 API waiting for instances to enter the state passed.
+      # Uses #poll_interval to determine how long to wait between status
+      # requests, and #provision_timeout to determine how long to wait before
+      # giving up.
+      #
+      def wait_for_instances(instances, state)
+        Timeout.timeout(provision_timeout) do
+          not_running = instances.dup
+
+          until not_running.empty?
+            instance = not_running.shift
+
+            status = instance.status rescue nil
+
+            if status
+              unless status == state
+                if_debug(3) do
+                  puts "instance #{instance.id} is not in #{state} state yet."
+                end
+
+                not_running.push(instance)
+              end
+            else
+              if_debug(3) do
+                puts "API server doesn't think #{instance.id} exists yet."
+              end
+
+              not_running.push(instance)
+            end
+
+            sleep poll_interval
+          end
+        end
+      rescue TimeoutError
+        raise "instances timed out waiting for ec2 API"
+      end
+
+      #
+      # Provision instance(s).
+      #
+      # If a security group id is supplied to this method, it will be
+      # permanently appended to #security_group_ids and used during instance
+      # creation.
+      #
+      # Regardless of the above behavior, if no security group ids exist, will
+      # raise RuntimeError (EC2 requires instances to be in at least one
+      # security group).
+      #
+      # Records the instance id's, waits for them to all enter the `:running`
+      # state, and then returns a list of their public IP addresses to the next
+      # provisioner.
+      #
+      def startup(args={})
+        if args[:security_group_ids]
+          @security_group_ids += args[:security_group_ids]
+        end
+
+        if security_group_ids.empty?
+          raise "no security groups supplied either at construction or provisioning time, cannot request instances."
+        end
+
+        instances = coerce_instances(ec2.instances.create(launch_options))
+        @instance_ids = instances.map(&:id)
+        wait_for_instances(instances, :running)
+
+        return({ :security_group_ids => @security_group_ids, :ips => Set[*instances.map(&:ip_address)], :ec2_instance_ids => @instance_ids })
+      end
+
+      #
+      # All instances are told to terminate, the method then waits for all of
+      # them to enter the terminated state, then returns true.
+      #
+      def shutdown(args={})
+        instances = instance_ids.map { |i| ec2.instances[i] }
+        instances.each { |i| i.terminate rescue nil }
+        wait_for_instances(instances, :terminated)
+
+        return { }
+      end
+
+      #
+      # Furnish reporter -- includes image id, number of servers, and instance id's.
+      #
+      def report
+        ["ami #{image_id}; #{count} servers; instance_ids: #{instance_ids.inspect}"]
+      end
+    end
+  end
+end