ironfan 4.10.3 → 4.10.4

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v4.10.4
+* Cleaning up specs to use local credentials, chef-zero (requires libssl): fixes #242
+* Added RDS support, see https://github.com/infochimps-labs/ironfan/pull/290 for details
+
 # v4.10.3
 * Tightening version constraints on excon, to avoid bad interaction between it & fog
 

data/Gemfile

@@ -34,4 +34,6 @@ group :test do
   gem 'guard-yard'
   gem 'ruby_gntp'
   gem 'ruby-debug19'
+  #
+  gem 'chef-zero'
 end

data/Gemfile.lock

@@ -22,6 +22,11 @@ GEM
       treetop (~> 1.4.9)
       uuidtools
       yajl-ruby (~> 1.1)
+    chef-zero (1.1.3)
+      hashie (~> 2.0)
+      mixlib-log (~> 1.3)
+      moneta (< 0.7.0)
+      puma (~> 2.0)
     coderay (1.0.9)
     columnize (0.3.6)
     configliere (0.4.18)
@@ -59,6 +64,7 @@ GEM
     guard-yard (2.1.0)
       guard (>= 1.1.0)
       yard (>= 0.7.0)
+    hashie (2.0.5)
     highline (1.6.19)
     ipaddress (0.8.0)
     jeweler (1.8.4)
@@ -107,6 +113,9 @@ GEM
       coderay (~> 1.0.5)
       method_source (~> 0.8)
       slop (~> 3.4)
+    puma (2.0.1)
+      rack (>= 1.1, < 2.0)
+    rack (1.5.2)
     rake (10.0.4)
     rb-fsevent (0.9.3)
     rb-inotify (0.9.0)
@@ -163,6 +172,7 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.0)
   chef (~> 10.16)
+  chef-zero
   excon (~> 0.21.0)
   fog (~> 1.2)
   formatador (~> 0.2)

data/README.md

@@ -23,7 +23,7 @@ Ironfan consists of the following Toolset:
   - knife plugins to orchestrate clusters of machines using simple commands like `knife cluster launch`
   - logic to coordinate truth among chef server and cloud providers.
 * [ironfan-pantry](https://github.com/infochimps-labs/ironfan-pantry): Our collection of industrial-strength, cloud-ready recipes for Hadoop, HBase, Cassandra, Elasticsearch, Zabbix and more.
-* [silverware cookbook](https://github.com/infochimps-labs/ironfan-homebase/tree/master/cookbooks/silverware): coordinate discovery of services ("list all the machines for `awesome_webapp`, that I might load balance them") and aspects ("list all components that write logs, that I might logrotate them, or that I might monitor the free space on their volumes".
+* [silverware cookbook](https://github.com/infochimps-labs/ironfan-pantry/tree/master/cookbooks/silverware): coordinate discovery of services ("list all the machines for `awesome_webapp`, that I might load balance them") and aspects ("list all components that write logs, that I might logrotate them, or that I might monitor the free space on their volumes".
 
 ### Documentation
 

data/VERSION

@@ -1 +1 @@
-4.10.3
+4.10.4

data/ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.10.3"
+  s.version = "4.10.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2013-05-17"
+  s.date = "2013-06-03"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [
@@ -40,6 +40,7 @@ Gem::Specification.new do |s|
     "ironfan.gemspec",
     "lib/chef/knife/bootstrap/centos6.2-ironfan.erb",
     "lib/chef/knife/bootstrap/chef-full-ironfan.erb",
+    "lib/chef/knife/bootstrap/rhel6.3-ironfan.erb",
     "lib/chef/knife/bootstrap/ubuntu10.04-ironfan.erb",
     "lib/chef/knife/bootstrap/ubuntu12.04-ironfan.erb",
     "lib/chef/knife/cluster_bootstrap.rb",
@@ -69,6 +70,7 @@ Gem::Specification.new do |s|
     "lib/ironfan/dsl/compute.rb",
     "lib/ironfan/dsl/ec2.rb",
     "lib/ironfan/dsl/facet.rb",
+    "lib/ironfan/dsl/rds.rb",
     "lib/ironfan/dsl/role.rb",
     "lib/ironfan/dsl/server.rb",
     "lib/ironfan/dsl/virtualbox.rb",
@@ -89,6 +91,9 @@ Gem::Specification.new do |s|
     "lib/ironfan/provider/ec2/machine.rb",
     "lib/ironfan/provider/ec2/placement_group.rb",
     "lib/ironfan/provider/ec2/security_group.rb",
+    "lib/ironfan/provider/rds.rb",
+    "lib/ironfan/provider/rds/machine.rb",
+    "lib/ironfan/provider/rds/security_group.rb",
     "lib/ironfan/provider/virtualbox.rb",
     "lib/ironfan/provider/virtualbox/machine.rb",
     "lib/ironfan/provider/vsphere.rb",
@@ -132,6 +137,7 @@ Gem::Specification.new do |s|
     "spec/fixtures/ec2/elb/snakeoil.key",
     "spec/fixtures/gunbai.rb",
     "spec/fixtures/gunbai_slice.json",
+    "spec/fixtures/knife/knife.rb",
     "spec/integration/minimal-chef-repo/chefignore",
     "spec/integration/minimal-chef-repo/environments/_default.json",
     "spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb",
@@ -155,7 +161,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.rubygems_version = "1.8.23"
   s.summary = "Infochimps' lightweight cloud orchestration toolkit, built on top of Chef."
-  s.test_files = ["spec/spec_helper/dummy_chef.rb", "spec/integration/spec_helper/launch_cluster.rb", "spec/integration/minimal-chef-repo/roles/systemwide.rb", "spec/integration/minimal-chef-repo/chefignore", "spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb", "spec/integration/minimal-chef-repo/knife/knife.rb", "spec/integration/minimal-chef-repo/environments/_default.json", "spec/integration/spec/simple_cluster_spec.rb", "spec/integration/spec/elb_build_spec.rb", "spec/integration/spec_helper.rb", "spec/test_config.rb", "spec/chef/cluster_bootstrap_spec.rb", "spec/chef/cluster_launch_spec.rb", "spec/fixtures/gunbai_slice.json", "spec/fixtures/ec2/elb/snakeoil.crt", "spec/fixtures/ec2/elb/snakeoil.key", "spec/fixtures/gunbai.rb", "spec/spec_helper.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/elb_spec.rb", "spec/ironfan/ec2/security_group_spec.rb"]
+  s.test_files = ["spec/spec_helper/dummy_chef.rb", "spec/integration/spec_helper/launch_cluster.rb", "spec/integration/minimal-chef-repo/roles/systemwide.rb", "spec/integration/minimal-chef-repo/chefignore", "spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb", "spec/integration/minimal-chef-repo/knife/knife.rb", "spec/integration/minimal-chef-repo/environments/_default.json", "spec/integration/spec/simple_cluster_spec.rb", "spec/integration/spec/elb_build_spec.rb", "spec/integration/spec_helper.rb", "spec/test_config.rb", "spec/chef/cluster_bootstrap_spec.rb", "spec/chef/cluster_launch_spec.rb", "spec/fixtures/gunbai_slice.json", "spec/fixtures/knife/knife.rb", "spec/fixtures/ec2/elb/snakeoil.crt", "spec/fixtures/ec2/elb/snakeoil.key", "spec/fixtures/gunbai.rb", "spec/spec_helper.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/elb_spec.rb", "spec/ironfan/ec2/security_group_spec.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/lib/chef/knife/bootstrap/rhel6.3-ironfan.erb

@@ -0,0 +1,179 @@
+bash <<'EOF' || echo "Chef bootstrap failed!"
+
+# This is the RHEL 6 bootstrap script from infochimps' ironfan. It is
+# based on opscode's bootstrap script, with the following important differences: 
+#
+# * installs ruby 1.9.2 (not 1.8.7) from source
+# * upgrades rubygems rather than installing from source
+# * pushes the computer identity into the first-boot.json
+# * installs the chef-client service and kicks off the first run of chef
+
+set -e
+
+<%= (@config[:verbosity].to_i > 1 ? 'set -v' : '') %>
+
+RUBY_VERSION=1.9.2-p290
+CHEF_VERSION=<%= bootstrap_version_string.gsub(/.*[\s=]/,"") %>
+
+mkdir -p /tmp/knife-bootstrap
+chmod 700 /tmp/knife-bootstrap 
+cd /tmp/knife-bootstrap
+
+<%= "export http_proxy=\"#{knife_config[:bootstrap_proxy]}\"" if knife_config[:bootstrap_proxy] -%>
+cat /etc/redhat-release
+
+date > /etc/box_build_time
+
+echo -e "`date` \n\n**** \n**** yum upgrade:\n****\n"
+yum upgrade --assumeyes
+
+echo -e "`date` \n\n**** \n**** Installing base packages:\n****\n"
+yum install --assumeyes make wget 
+yum install --assumeyes git rpm-build rpmdevtools gcc glibc-static zlib-devel libxml2-devel libxslt-devel openssl-devel telnet nc uuid-devel
+if [ ! -d runit-rpm ]; then git clone https://github.com/imeyer/runit-rpm.git; fi
+cd runit-rpm
+./build.sh
+yum install --assumeyes /root/rpmbuild/RPMS/x86_64/runit-*.rpm || true # TODO: Remove this shim
+cd -
+yum remove --assumeyes prelink
+yum clean all
+
+if [ ! -f /usr/bin/chef-client ]; then
+  echo -e "`date` \n\n**** \n**** Installing ruby version ${RUBY_VERSION}:\n****\n"
+
+  wget ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-${RUBY_VERSION}.tar.gz
+  tar xzf ruby-${RUBY_VERSION}.tar.gz
+  cd ruby-${RUBY_VERSION}
+  ./configure --with-ruby-version=${RUBY_VERSION} --prefix=/usr --program-suffix=${RUBY_VERSION}
+  make -j2
+  make install
+
+  alternatives \
+    --install /usr/bin/ruby ruby /usr/bin/ruby${RUBY_VERSION} 400 \
+    --slave   /usr/bin/ri   ri   /usr/bin/ri${RUBY_VERSION}       \
+    --slave   /usr/bin/irb  irb  /usr/bin/irb${RUBY_VERSION}      \
+    --slave   /usr/bin/erb  erb  /usr/bin/erb${RUBY_VERSION}      \
+    --slave   /usr/bin/gem  gem  /usr/bin/gem${RUBY_VERSION}      \
+    --slave   /usr/share/man/man1/ruby.1.gz ruby.1.gz             \
+    /usr/share/man/man1/ruby${RUBY_VERSION}.1
+
+  echo -e "`date` \n\n**** \n**** Updating rubygems:\n****\n"
+  gem install rubygems-update -v 1.8.5
+
+  echo -e "`date` \n\n**** \n**** Installing chef:\n****\n"
+  gem install net-ssh --no-rdoc --no-ri --version 2.2.2
+  gem install net-ssh-gateway --no-rdoc --no-ri --version 1.1.0
+  gem install net-ssh-multi --no-rdoc --no-ri --version 1.1
+  gem install ohai --no-rdoc --no-ri --version 6.14.0
+  gem install chef --no-rdoc --no-ri --version 10.16.4
+  # gems needed for the client.rb or so generically useful you want them at hand
+  gem install      --no-rdoc --no-ri extlib bundler json right_aws pry fog
+
+else # no chef-client
+  echo -e "`date` \n\n**** \n**** Chef is present -- skipping apt/ruby/chef installation\n****\n"
+fi # end ruby+chef install
+
+echo -e "`date` \n\n**** \n**** Knifing in the chef client config files:\n****\n"
+mkdir -p /etc/chef
+
+<%- if @config[:client_key] %>
+(
+cat <<'EOP'
+<%= @config[:client_key] %>
+EOP
+) > /tmp/knife-bootstrap/client.pem
+awk NF /tmp/knife-bootstrap/client.pem > /etc/chef/client.pem
+<%- else %>
+(
+cat <<'EOP'
+<%= validation_key %>
+EOP
+) > /tmp/knife-bootstrap/validation.pem
+awk NF /tmp/knife-bootstrap/validation.pem > /etc/chef/validation.pem
+<%- end %>
+
+<% if @chef_config[:encrypted_data_bag_secret] -%>
+(
+cat <<'EOP'
+<%= encrypted_data_bag_secret %>
+EOP
+) > /tmp/encrypted_data_bag_secret
+awk NF /tmp/encrypted_data_bag_secret > /etc/chef/encrypted_data_bag_secret
+rm /tmp/encrypted_data_bag_secret
+<% end -%>
+
+echo -e "`date` \n\n**** \n**** Nuking our temp files:\n****\n"
+
+cd /tmp
+rm -rf /tmp/knife-bootstrap
+
+echo -e "`date` \n\n**** \n**** Creating chef client script:\n****\n"
+
+(
+cat <<'EOP'
+<%= config_content %>
+<%= @config[:computer].chef_client_script_content %>
+EOP
+) > /etc/chef/client.rb
+
+(
+cat <<'EOP'
+<%= { "run_list" => @run_list, "cluster_name" => @config[:server].cluster_name, "facet_name" => @config[:server].facet_name, "facet_index" => @config[:server].index }.to_json %>
+EOP
+) > /etc/chef/first-boot.json
+
+# Ensure that EC2 images are recognized even inside VPC
+mkdir -p /etc/chef/ohai/hints/
+touch /etc/chef/ohai/hints/ec2.json
+
+
+echo -e "`date` \n\n**** \n**** Adding chef client nonce script:\n****\n"
+
+cat > /etc/init.d/chef-client-nonce <<'EOP'
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:           chef-client-nonce
+# Required-Start:     $remote_fs $network
+# Required-Stop:
+# Default-Start:      2 3 4 5
+# Default-Stop:       0 1 6
+# Short-Description:  Start a single chef-client run.
+### END INIT INFO
+#
+# Copyright (c) 2009-2010 Opscode, Inc, <legal@opscode.com>
+#
+# chef-client         Startup script for chef-client.
+# chkconfig: - 99 02
+# description: starts up chef-client once, at boot
+
+case "$1" in
+  start)
+    /usr/bin/chef-client -L /var/log/chef/client.log
+    exit $?
+    ;;
+  *)
+    echo "Usage: /etc/init.d/chef-client-nonce start" >&2
+    exit 1
+    ;;
+esac
+EOP
+
+mkdir -p /var/log/chef
+mkdir -p /etc/sv
+chmod +x /etc/init.d/chef-client-nonce 
+chkconfig --add chef-client-nonce
+chkconfig --del iptables
+chkconfig --del rh-cloud-firstboot
+chkconfig --add rh-cloud-firstboot
+
+rm /etc/sysconfig/rh-cloud-firstboot
+
+<%- if (@config[:bootstrap_runs_chef_client].to_s == 'true') || (@chef_config.knife[:bootstrap_runs_chef_client].to_s == 'true') %>
+sudo /etc/init.d/chef-client-nonce start
+<%- end %>
+
+
+
+echo -e "`date` \n\n**** \n**** Cleanup:\n****\n"
+cd /
+rm -r /tmp/knife-bootstrap

data/lib/chef/knife/cluster_launch.rb

@@ -95,7 +95,7 @@ class Chef
         # As each server finishes, configure it
         Ironfan.parallel(launched) do |computer|
           if (computer.is_a?(Exception)) then ui.warn "Error launching #{computer.inspect}; skipping after-launch tasks."; next; end
-          perform_after_launch_tasks(computer)
+          perform_after_launch_tasks(computer) if computer.machine.perform_after_launch_tasks?
         end
 
         if healthy?

data/lib/ironfan/dsl/cloud.rb

@@ -13,6 +13,7 @@ module Ironfan
             when :ec2        then Ec2
             when :virtualbox then VirtualBox
             when :vsphere    then Vsphere
+            when :rds        then Rds
             else raise "Unsupported cloud #{obj[:name]}"
             end
         end

data/lib/ironfan/dsl/rds.rb

@@ -0,0 +1,84 @@
+require 'digest/md5'
+
+module Ironfan
+  class Dsl
+
+    class Compute < Ironfan::Dsl
+      def rds(*attrs,&block)            cloud(:rds,*attrs,&block);      end
+    end
+
+    class Rds < Cloud
+      magic :autoupgrade,               :boolean,       :default => true
+      magic :availability_zones,        Array,          :default => ['us-east-1d']
+      magic :backup_retention,          Integer,        :default => 1
+      magic :charset,                   String,         :default => nil
+      magic :dbname,                    String,         :default => nil
+      magic :default_availability_zone, String,         :default => ->{ availability_zones.first }
+      magic :engine,                    String,         :default => ->{ engines.first }
+      magic :engines,                   Array,          :default => ["MySQL", "oracle-se1", "oracle-se", "oracle-ee", "sqlserver-ee", "sqlserver-se", "sqlserver-ex", "sqlserver-web"]
+      magic :flavor,                    String,         :default => "db.t1.micro"
+      magic :iops,                      Integer,        :default => 1000
+      magic :license_model,             String,         :default => nil
+      magic :multi_az,                  :boolean,       :default => false
+      magic :password,                  String,         :default => nil
+      magic :port,                      Integer,        :default => -> { default_port }      
+      magic :preferred_backup_window,    String,        :default => nil  # Format hh24:mi-hh24:mi.  Needs to be at least 30 minutes, UTC,
+      magic :preferred_maintenance_window, String,      :default => nil  # Format ddd:hh24:mi-ddd:hh24:mi.  Minimum 30 minutes, UTC
+      magic :provider,                  Whatever,       :default => Ironfan::Provider::Rds
+      magic :size,                      Integer,        :default => 50 # Size in GB
+      collection :security_groups,      Ironfan::Dsl::Rds::SecurityGroup, :key_method => :name
+      magic :username,                  String,         :default => nil
+      magic :version,                   String,         :default => nil
+      
+      def receive_provider(obj)
+        if obj.is_a?(String)
+          write_attribute :provider, Gorillib::Inflector.constantize(Gorillib::Inflector.camelize(obj.gsub(/\./, '/')))
+        else
+          super(obj)
+        end
+      end
+
+      # This is a dummy function to fill Ironfan's requirements.
+      def implied_volumes
+        []
+      end
+
+      def default_port
+        return 3306 if engine == "MySQL"
+        return 1521 if ["oracle-se1", "oracle-se", "oracle-ee"].include?(engine)
+        return 1433 if ["sqlserver-ee", "sqlserver-se", "sqlserver-ex", "sqlserver-web"].include?(engine)
+      end
+
+      def to_display(style,values={})
+        return values if style == :minimal
+
+        values["Engine"] =            engine
+        values["AZ"] =                default_availability_zone
+        return values if style == :default
+
+#        values["Public IP"] =        elastic_ip if elastic_ip
+        values
+      end
+
+      class SecurityGroup < Ironfan::Dsl
+        field :name,                    String
+        field :ec2_authorizations,      Array, :default => []
+        field :ip_authorizations,       Array, :default => []
+
+        def authorize_ip_range(cidr_ip = '0.0.0.0/0')
+          ip_authorizations << cidr_ip
+          ip_authorizations.compact!
+          ip_authorizations.uniq!
+        end
+
+        def authorize_ec2_group(ec2_security_group)
+          ec2_authorizations << ec2_security_group
+          ec2_authorizations.compact!
+          ec2_authorizations.uniq!
+        end
+      end # SecurityGroup
+
+    end
+  end
+end
+

data/lib/ironfan/headers.rb

@@ -31,6 +31,9 @@ module Ironfan
     end
     class VirtualBox < Cloud; end
     class Vsphere < Cloud; end
+    class Rds < Cloud
+      class SecurityGroup < Ironfan::Dsl; end
+    end
   end
 
   class Provider < Builder
@@ -62,7 +65,10 @@ module Ironfan
       class Machine < Ironfan::IaasProvider::Machine; end
       class Keypair < Ironfan::Provider::Resource; end
     end
-
+    class Rds < Ironfan::IaasProvider
+      class Machine < Ironfan::IaasProvider::Machine; end
+      class SecurityGroup < Ironfan::Provider::Resource; end
+    end
   end
 
 end

data/lib/ironfan/provider/ec2/machine.rb

@@ -77,6 +77,10 @@ module Ironfan
           adaptee.wait_for{ machine.stopping? or machine.stopped? }
         end
 
+        def perform_after_launch_tasks?
+          true
+        end
+
         def to_display(style,values={})
           # style == :minimal
           values["State"] =             state.to_sym

data/lib/ironfan/provider/rds/machine.rb

@@ -0,0 +1,225 @@
+module Ironfan
+  class Provider
+    class Rds
+
+      class Machine < Ironfan::IaasProvider::Machine
+        delegate :availability_zone, :created_at, :desstroy, :endpoint, :engine,
+          :engine_version, :destroy, :flavor_id, :id, :id=, :name, :state, :add_tags, 
+          :tags, 
+          :to => :adaptee
+
+        def self.shared?()      false;  end
+        def self.multiple?()    false;  end
+        def self.resource_type()        :machine;   end
+        def self.expected_ids(computer) [computer.server.full_name];   end
+
+        def name
+          return id if tags.empty?
+          tags["Name"] || tags["name"] || id
+        end
+
+        def public_hostname  ; dns_name ; end
+ 
+        def created?
+          not ['terminated', 'shutting-down'].include? state
+        end
+
+        def deleting?
+          state == "deleting"
+        end
+
+        def pending?
+          state == "pending"
+        end
+
+        def creating?
+          state == "creating"
+        end
+
+        def rebooting?
+          state == "rebooting"
+        end
+
+        def available?
+          state == "available"
+        end
+      
+        def stopped? 
+        end
+
+        def perform_after_launch_tasks?
+          false
+        end
+
+        def to_display(style,values={})
+          # style == :minimal
+          values["State"] =             state.to_sym
+          values["Endpoint"]  =         adaptee.endpoint["Address"]
+          values["Created On"] =        created_at.to_date
+          return values if style == :minimal
+
+          # style == :default
+          values["Flavor"] =            flavor_id
+          values["AZ"] =                availability_zone
+          return values if style == :default
+
+          # style == :expanded
+          values["Port"]  =         adaptee.endpoint["Port"]
+          values["Engine"]  =           engine
+          values["EngineVersion"] =     engine_version
+          values
+        end
+
+        def to_s
+          "<%-15s %-12s %-25s %-25s %-15s %-15s>" % [
+            self.class.handle, id, created_at, tags['name'], flavor_id, availability_zone]
+        end
+
+        #
+        # Discovery
+        #
+        def self.load!(cluster=nil)
+          Rds.connection.servers.each do |fs|
+            machine = new(:adaptee => fs)
+            if (not machine.created?)
+              next unless Ironfan.chef_config[:include_terminated]
+              remember machine, :append_id => "terminated:#{machine.id}"
+            elsif recall? machine.name
+              machine.bogus <<                 :duplicate_machines
+              recall(machine.name).bogus <<    :duplicate_machines
+              remember machine, :append_id => "duplicate:#{machine.id}"
+            else # never seen it
+              remember machine
+            end
+            Chef::Log.debug("Loaded #{machine}")
+          end
+        end
+
+        def receive_adaptee(obj)
+          obj = Rds.connection.servers.new(obj) if obj.is_a?(Hash)
+          super
+        end
+
+        # Find active machines that haven't matched, but should have,
+        #   make sure all bogus machines have a computer to attach to
+        #   for display purposes
+        def self.validate_resources!(computers)
+          recall.each_value do |machine|
+            next unless machine.users.empty? and machine.name
+            if machine.name.match("^#{computers.cluster.name}-")
+              machine.bogus << :unexpected_machine
+            end
+            next unless machine.bogus?
+            fake           = Ironfan::Broker::Computer.new
+            fake[:machine] = machine
+            fake.name      = machine.name
+            machine.users << fake
+            computers     << fake
+          end
+        end
+
+        #
+        # Manipulation
+        #
+        def self.create!(computer)
+          return if computer.machine? and computer.machine.created?
+          Ironfan.step(computer.name,"creating RDS machine... go grab some a beer or two.", :green)
+          #
+          errors = lint(computer)
+          if errors.present? then raise ArgumentError, "Failed validation: #{errors.inspect}" ; end
+          #
+          launch_desc = launch_description(computer)
+          launch_desc[:id] = computer.name
+          Chef::Log.debug(JSON.pretty_generate(launch_desc))
+
+          Ironfan.safely do
+            fog_server = Rds.connection.servers.create(launch_desc)
+            machine = Machine.new(:adaptee => fog_server)
+            computer.machine = machine
+            remember machine, :id => computer.name
+
+            Ironfan.step(fog_server.id,"waiting for machine to be ready", :gray)
+            Ironfan.tell_you_thrice     :name           => fog_server.id,
+                                        :problem        => "server unavailable",
+                                        :error_class    => Fog::Errors::Error do
+              fog_server.wait_for { ready? }
+            end
+          end
+
+          # tag the computer correctly
+          tags = {
+            'cluster' =>      computer.server.cluster_name,
+            'facet' =>        computer.server.facet_name,
+            'index' =>        computer.server.index,
+            'name' =>         computer.name,
+            'Name' =>         computer.name,
+          }
+          Rds.ensure_tags(tags, computer.machine)
+        end
+
+        # @returns [Hash{String, Array}] of 'what you did wrong' => [relevant, info]
+        def self.lint(computer)
+          cloud = computer.server.cloud(:rds)
+          info  = [computer.name, cloud.inspect]
+          errors = {}
+          errors["No Password"] = info if cloud.password.empty?
+          errors["No Username"] = info if cloud.username.empty?
+          errors["Size"]        = info if cloud.size < 5
+          errors
+        end
+
+        def self.launch_description(computer)
+          cloud = computer.server.cloud(:rds)
+          user_data_hsh =               {
+            :chef_server =>             Chef::Config[:chef_server_url],
+            :node_name =>               computer.name,
+            :organization =>            Chef::Config[:organization],
+            :cluster_name =>            computer.server.cluster_name,
+            :facet_name =>              computer.server.facet_name,
+            :facet_index =>             computer.server.index,
+          }
+
+
+          # main machine info
+          # note that Fog does not actually create tags when it creates a
+          #  server; they and permanence are applied during sync
+          description = {
+            :allocated_storage            => cloud.size,
+            :auto_minor_version_upgrade   => cloud.autoupgrade,
+            :availability_zone            => cloud.multi_az ? nil : cloud.default_availability_zone,
+            :backup_retention_period      => cloud.backup_retention,
+            :db_name                      => cloud.dbname,
+            :engine                       => cloud.engine,
+            :engine_version               => cloud.version,
+            :flavor_id                    => cloud.flavor,
+            :license_model                => cloud.license_model,
+            :master_username              => cloud.username,
+            :multi_az                     => cloud.multi_az, 
+            :password                     => cloud.password,
+            :port                         => cloud.port,
+            :preferred_backup_window      => cloud.preferred_backup_window,
+            :preferred_maintenance_window => cloud.preferred_maintenance_window,
+            :security_group_names         => cloud.security_groups.keys,
+            :user_data                    => JSON.pretty_generate(user_data_hsh),
+#            :charset                    => cloud.charset, # Not supported in FOG?
+#            :iops                       => cloud.iops, # Not supported in FOG?
+          }
+
+          description
+        end
+
+        def self.destroy!(computer)
+          return unless computer.machine?
+          forget computer.machine.name
+          computer.machine.destroy
+          computer.machine.reload            # show the node as shutting down
+        end
+
+        def self.save!(computer)
+          return unless computer.machine?
+        end
+      end
+
+    end
+  end
+end

data/lib/ironfan/provider/rds/security_group.rb

@@ -0,0 +1,128 @@
+module Ironfan
+  class Provider
+    class Rds
+
+      class SecurityGroup < Ironfan::Provider::Resource
+
+        delegate :id, :authorize_port_range,
+          :to => :adaptee
+
+        def self.shared?()      true;   end
+        def self.multiple?()    true;   end
+        def self.resource_type()        :security_group;   end
+        def self.expected_ids(computer)
+          return unless computer.server
+          rds = computer.server.cloud(:rds)
+          rds.security_groups.keys.uniq
+        end
+
+        def name()
+          adaptee.id
+        end
+
+        #
+        # Discovery
+        #
+        def self.load!(cluster=nil)
+          Rds.connection.security_groups.reject { |raw| raw.blank? }.each do |raw|
+            sg = SecurityGroup.new(:adaptee => raw)
+            remember(sg)
+            Chef::Log.debug("Loaded #{sg}: #{sg.inspect}")
+          end
+        end
+
+        def receive_adaptee(obj)
+          obj = Rds.connection.security_groups.new(obj) if obj.is_a?(Hash)
+          super
+        end
+
+        def to_s
+            return "<%-15s %s>" % [ self.class.handle, id ]
+        end
+
+        #
+        # Manipulation
+        #
+
+        def self.prepare!(computers)
+
+          # Create any groups that don't yet exist, and ensure any authorizations
+          # that are required for those groups
+          cluster_name             = nil
+          groups_to_create         = [ ]
+          ec2_authorizations       = [ ]
+          ip_authorizations        = [ ]
+
+          # First, deduce the list of all groups to which at least one instance belongs
+          # We'll use this later to decide whether to create groups, or authorize access,
+          # using a VPC security group or an EC2 security group.
+
+          computers.select { |computer| Rds.applicable computer }.each do |computer|
+            cloud           = computer.server.cloud(:rds)
+            cluster_name    = computer.server.cluster_name
+
+            # Iterate over all of the security group information, keeping track of
+            # any groups that must exist and any authorizations that must be ensured
+            cloud.security_groups.values.each do |dsl_group|
+              groups_to_create << dsl_group.name
+              ip_authorizations << dsl_group.ip_authorizations.map do |ip|
+                { 
+                  :grantor => dsl_group.name, 
+                  :grantee => ip,
+                }
+              end
+
+              ec2_authorizations << dsl_group.ec2_authorizations.map do |ec2|
+                {
+                  :grantor => dsl_group.name, 
+                  :grantee => ec2,
+                }
+              end
+            end
+          end
+
+          groups_to_create   = groups_to_create.flatten.uniq.reject { |group| recall? group.to_s }.sort
+          ip_authorizations  = ip_authorizations.flatten.uniq.sort { |a,b| a[:grantor] <=> b[:grantor] }
+          ec2_authorizations = ec2_authorizations.flatten.uniq.sort { |a,b| a[:grantor] <=> b[:grantor] }
+
+          Ironfan.step(cluster_name, "creating security groups", :blue) unless groups_to_create.empty?
+          groups_to_create.each do |group|
+            Ironfan.step(group, "creating #{group} security group", :blue)
+            begin
+              Rds.connection.create_db_security_group(group,"Ironfan created group #{group}")
+            rescue Fog::Compute::AWS::Error => e # InvalidPermission.Duplicate
+              Chef::Log.info("ignoring security group error: #{e}")
+            end
+          end
+
+          # Re-load everything so that we have a @@known list of security groups to manipulate
+          load! unless groups_to_create.empty?
+
+          # Now make sure that all required authorizations are present
+          Ironfan.step(cluster_name, "ensuring security group permissions", :blue) unless (ec2_authorizations.empty? or ip_authorizations.empty?)
+          ip_authorizations.each do |auth|
+            message = " ensuring access from #{auth[:grantor]} to #{auth[:grantee]} "
+            Ironfan.step(auth[:grantor], message, :blue)
+            begin 
+              Rds.connection.authorize_db_security_group_ingress(auth[:grantor], { "CIDRIP" => auth[:grantee] })
+            rescue Fog::AWS::RDS::AuthorizationAlreadyExists => e
+              Chef::Log.info("ignoring security group error: #{e}")
+            end
+          end
+
+          ec2_authorizations.each do |auth|
+            message = " ensuring access from #{auth[:grantor]} to #{auth[:grantee]} "
+            Ironfan.step(auth[:grantor], message, :blue)
+            aws_account, security_group = auth[:grantee].split('/')
+            begin 
+              Rds.connection.authorize_db_security_group_ingress(auth[:grantor], { "EC2SecurityGroupName" => security_group, "EC2SecurityGroupOwnerId" => aws_account })
+            rescue Fog::AWS::RDS::AuthorizationAlreadyExists => e
+              Chef::Log.info("ignoring security group error: #{e}")
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/ironfan/provider/rds.rb

@@ -0,0 +1,53 @@
+module Ironfan
+  class Provider
+
+    class Rds < Ironfan::IaasProvider
+      self.handle = :rds
+
+      def self.resources
+        [ Machine, SecurityGroup ]
+      end
+
+      #
+      # Utility functions
+      #
+      def self.connection
+        @@connection ||=  Fog::AWS::RDS.new(self.aws_credentials)
+      end
+
+      def self.iam
+        credentials = self.aws_credentials
+        credentials.delete(:region)
+        @@iam ||= Fog::AWS::IAM.new(credentials)
+      end
+
+      def self.aws_account_id()
+        Chef::Config[:knife][:aws_account_id]
+      end
+
+      # Ensure that a fog object (machine, volume, etc.) has the proper tags on it
+      def self.ensure_tags(tags, fog)
+        tags.delete_if {|k, v| fog.tags[k] == v.to_s  rescue false }
+        return if tags.empty?
+
+        Ironfan.step(fog.id, "tagging with #{tags.inspect}", :green)
+        fog.add_tags(tags)
+      end
+
+      def self.applicable(computer)
+        computer.server and computer.server.clouds.include?(:rds)
+      end
+
+      private
+
+      def self.aws_credentials
+        return {
+          :aws_access_key_id     => Chef::Config[:knife][:aws_access_key_id],
+          :aws_secret_access_key => Chef::Config[:knife][:aws_secret_access_key],
+          :region                => Chef::Config[:knife][:region]
+        }
+      end
+
+    end
+  end
+end

data/lib/ironfan/provider/vsphere/machine.rb

@@ -78,6 +78,10 @@ module Ironfan
           adaptee.PowerOffVM_Task.wait_for_completion
         end
 
+        def perform_after_launch_tasks?
+          true
+        end
+
         def self.ip_settings(settings, hostname)
           # FIXME: A lot of this is required if using a static IP
           # Heavily referenced Serengeti's FOG here 

data/lib/ironfan/provider.rb

@@ -15,6 +15,7 @@ module Ironfan
           when :ec2         then Ec2
           when :vsphere     then Vsphere
           when :virtualbox  then VirtualBox
+          when :rds         then Rds
           else raise "Unsupported provider #{obj[:name]}"
           end
       end

data/lib/ironfan/requirements.rb

@@ -20,6 +20,7 @@ require 'ironfan/dsl/volume'
 require 'ironfan/dsl/cloud'
 require 'ironfan/dsl/ec2'
 require 'ironfan/dsl/vsphere'
+require 'ironfan/dsl/rds'
 
 
 # Providers for specific resources
@@ -47,6 +48,9 @@ require 'ironfan/provider/vsphere'
 require 'ironfan/provider/vsphere/machine'
 require 'ironfan/provider/vsphere/keypair'
 
+require 'ironfan/provider/rds'
+require 'ironfan/provider/rds/machine'
+require 'ironfan/provider/rds/security_group'
 
 # Broker classes to coordinate DSL expectations and provider resources
 require 'ironfan/broker'

data/spec/fixtures/knife/knife.rb

@@ -0,0 +1,5 @@
+username            'testmonkey'
+ec2_key_dir         "."
+log_level           :info
+node_name           username
+client_key          "#{username}.pem"

data/spec/spec_helper.rb

@@ -12,9 +12,11 @@ Pathname.register_paths(
   fixtures:    [:code, 'spec', 'fixtures'],
   )
 
-RSpec.configure do |config|
+RSpec.configure do |cfg|
   def ironfan_go!
-    Chef::Knife.new.configure_chef
+    k = Chef::Knife.new
+    k.config[:config_file] = Pathname.path_to(:fixtures, 'knife/knife.rb')
+    k.configure_chef
     Chef::Config.instance_eval do
       knife.merge!({
           :aws_access_key_id => 'access_key',
@@ -26,7 +28,11 @@ RSpec.configure do |config|
     require 'ironfan'
 
     Ironfan.ui          = Chef::Knife::UI.new(STDOUT, STDERR, STDIN, {})
-    Ironfan.chef_config = { :verbosity => 0 }
+    Ironfan.chef_config = k.config
     Ironfan.cluster_path
   end
 end
+
+require 'chef_zero/server'
+server = ChefZero::Server.new(port: 4000)
+server.start_background

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ironfan
 version: !ruby/object:Gem::Version
-  version: 4.10.3
+  version: 4.10.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-17 00:00:00.000000000 Z
+date: 2013-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -251,6 +251,7 @@ files:
 - ironfan.gemspec
 - lib/chef/knife/bootstrap/centos6.2-ironfan.erb
 - lib/chef/knife/bootstrap/chef-full-ironfan.erb
+- lib/chef/knife/bootstrap/rhel6.3-ironfan.erb
 - lib/chef/knife/bootstrap/ubuntu10.04-ironfan.erb
 - lib/chef/knife/bootstrap/ubuntu12.04-ironfan.erb
 - lib/chef/knife/cluster_bootstrap.rb
@@ -280,6 +281,7 @@ files:
 - lib/ironfan/dsl/compute.rb
 - lib/ironfan/dsl/ec2.rb
 - lib/ironfan/dsl/facet.rb
+- lib/ironfan/dsl/rds.rb
 - lib/ironfan/dsl/role.rb
 - lib/ironfan/dsl/server.rb
 - lib/ironfan/dsl/virtualbox.rb
@@ -300,6 +302,9 @@ files:
 - lib/ironfan/provider/ec2/machine.rb
 - lib/ironfan/provider/ec2/placement_group.rb
 - lib/ironfan/provider/ec2/security_group.rb
+- lib/ironfan/provider/rds.rb
+- lib/ironfan/provider/rds/machine.rb
+- lib/ironfan/provider/rds/security_group.rb
 - lib/ironfan/provider/virtualbox.rb
 - lib/ironfan/provider/virtualbox/machine.rb
 - lib/ironfan/provider/vsphere.rb
@@ -343,6 +348,7 @@ files:
 - spec/fixtures/ec2/elb/snakeoil.key
 - spec/fixtures/gunbai.rb
 - spec/fixtures/gunbai_slice.json
+- spec/fixtures/knife/knife.rb
 - spec/integration/minimal-chef-repo/chefignore
 - spec/integration/minimal-chef-repo/environments/_default.json
 - spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb
@@ -375,7 +381,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4135598770723135786
+      hash: -3042039387970317693
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -403,6 +409,7 @@ test_files:
 - spec/chef/cluster_bootstrap_spec.rb
 - spec/chef/cluster_launch_spec.rb
 - spec/fixtures/gunbai_slice.json
+- spec/fixtures/knife/knife.rb
 - spec/fixtures/ec2/elb/snakeoil.crt
 - spec/fixtures/ec2/elb/snakeoil.key
 - spec/fixtures/gunbai.rb