ironfan 4.2.1 → 4.2.2

data/CHANGELOG.md

@@ -1,5 +1,14 @@
+# v4.2.2: @mrflip rocks the house
+* Terminated machines are not bogus (fixes #165)
+* Ignore deleting, deleted, or errored volumes in discovery
+* Rescue on duplicate security groups; don't die logging on full 'load!'
+* Changed key_pair to keypair, as per the DSL
+* Specs for scripts (but not really)
+* Model cleanup -- now mostly round-trip to JSON
+* knife cluster bootstrap sets Chef::Config[:environment] (fixes #148)
+
 # v4.2.1: @nickmarden rocks the house
-* More correct merging of cluster and facet objects, with specs
+* Correct merging of cluster and facet objects, with specs (fixes #158)
 * Circumvent memory bloat by resolving just once
 
 # v4.2.0:

data/VERSION

@@ -1 +1 @@
-4.2.1
+4.2.2

data/ironfan.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.2.1"
+  s.version = "4.2.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
@@ -77,14 +77,18 @@ Gem::Specification.new do |s|
     "lib/ironfan/provider/ec2.rb",
     "lib/ironfan/provider/ec2/ebs_volume.rb",
     "lib/ironfan/provider/ec2/elastic_ip.rb",
-    "lib/ironfan/provider/ec2/key_pair.rb",
+    "lib/ironfan/provider/ec2/keypair.rb",
     "lib/ironfan/provider/ec2/machine.rb",
     "lib/ironfan/provider/ec2/placement_group.rb",
     "lib/ironfan/provider/ec2/security_group.rb",
     "lib/ironfan/provider/virtualbox.rb",
     "lib/ironfan/provider/virtualbox/machine.rb",
     "lib/ironfan/requirements.rb",
+    "spec/chef/cluster_bootstrap_spec.rb",
+    "spec/fixtures/gunbai.rb",
+    "spec/fixtures/gunbai_slice.json",
     "spec/ironfan/cluster_spec.rb",
+    "spec/ironfan/ec2/cloud_provider.rb",
     "spec/ironfan/ec2/cloud_provider_spec.rb",
     "spec/ironfan/ec2/security_group_spec.rb",
     "spec/spec_helper.rb",
@@ -97,7 +101,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.rubygems_version = "1.8.24"
   s.summary = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
-  s.test_files = ["spec/spec_helper/dummy_chef.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/security_group_spec.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
+  s.test_files = ["spec/spec_helper/dummy_chef.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/security_group_spec.rb", "spec/ironfan/ec2/cloud_provider.rb", "spec/chef/cluster_bootstrap_spec.rb", "spec/fixtures/gunbai_slice.json", "spec/fixtures/gunbai.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/lib/chef/knife/cluster_bootstrap.rb

@@ -59,6 +59,7 @@ class Chef
       end
 
       def perform_execution(target)
+        reconcile_chef_config(target)
         # Execute across all servers in parallel
         Ironfan.parallel(target.values) {|computer| run_bootstrap(computer)}
 #         threads = target.servers.map{ |server| Thread.new(server) { |svr| run_bootstrap(svr, svr.public_hostname) } }
@@ -69,6 +70,18 @@ class Chef
         confirm_or_exit("Are you absolutely certain that you want to perform this action? (Type 'Yes' to confirm) ", 'Yes')
       end
 
+    protected
+
+      def reconcile_chef_config(target)
+        environments = target.environments
+        if environments.length > 1
+          ui.error "You cannot bootstrap machines in multiple chef environments: got #{environments.inspect} from #{target.map(&:name)}"
+          ui.error "Re-run this command on each subgroup of machines that share an environment"
+          raise StandardError, "Cannot bootstrap multiple chef environments"
+        end
+        Chef::Config[:environment] = environments.first
+      end
+
     end
   end
 end

data/lib/chef/knife/cluster_pry.rb

@@ -45,9 +45,12 @@ class Chef
         cluster = target.cluster
 
         ui.info("")
-        ui.info("You are in a cluster. There is a sign overhead reading '#{ui.color(@name_args.first, :yellow)}'.")
-        ui.info("Next to you a burly man in a greasy apron sharpens his cleaver, and a lissom princess performs treacherous origami.")
-        ui.info("It is Pitch Dark. You are likely to be eaten by a grue.") if target.select(&:running?).empty?
+        ui.info([
+            ui.color("You are in a cluster. There is a sign overhead reading '", :magenta),
+            ui.color(@name_args.first, :yellow, :bold),
+            ui.color("'.\nNext to you a burly man in a greasy apron sharpens his cleaver, \nand a lissom princess performs treacherous origami.", :magenta)
+            ].join)
+        ui.info(ui.color("It is Pitch Dark. You are likely to be eaten by a grue.", :black, :bold)) if target.select(&:running?).empty?
 
         # Commands to try:
         #   nn = Chef::Node.load('node-name')

data/lib/chef/knife/cluster_show.rb

@@ -35,6 +35,7 @@ class Chef
         :boolean     => true
 
       def run
+        with_verbosity(1){ config[:include_terminated] }
         load_ironfan
         die(banner) if @name_args.empty?
         configure_dry_run

data/lib/gorillib/resolution.rb

@@ -4,7 +4,7 @@ require 'gorillib/hash/deep_merge'
 
 module Gorillib
 
-  # Make a clean deep-copy of the value, via gorillib semantics if 
+  # Make a clean deep-copy of the value, via gorillib semantics if
   #   possible, otherwise via marshalling
   def self.deep_copy(value)
     case
@@ -36,7 +36,7 @@ module Gorillib
             else
               write_attribute(collection_field_name, coll)
             end
-          rescue StandardError => err ; err.polish("#{self.class} #{collection_field_name} collection on #{args}'") rescue nil ; raise ; end
+          rescue StandardError => err ; err.polish("#{self.class} #{collection_field_name} collection on #{coll}'") rescue nil ; raise ; end
         end
       end
     end
@@ -44,13 +44,13 @@ module Gorillib
   end
 
   # The attribute :underlay provides an object (preferably another
-  #   Gorillib::Model or the like) that will resolve stacked 
-  #   defaults. If fields are declared with a :resolver, it will 
+  #   Gorillib::Model or the like) that will resolve stacked
+  #   defaults. If fields are declared with a :resolver, it will
   #   apply that call in preference the default rules (self.field
   #   -> underlay.field -> self.field.default )
   #
-  # To provide resolve cleanly without read-write loops destroying 
-  #   the separation of concerns, the resolve mechanism has been 
+  # To provide resolve cleanly without read-write loops destroying
+  #   the separation of concerns, the resolve mechanism has been
   #   broken from the regular read-write accessors.
   #
   module Resolution
@@ -59,9 +59,9 @@ module Gorillib
     attr_accessor :underlay
 
     # Return a fully-resolved copy of this object. All objects
-    #   referenced will be clean deep_copies, and will lack the 
+    #   referenced will be clean deep_copies, and will lack the
     #   :underlay accessor. This is by design, to prevent self-
-    #   referential loops (parent->collection->child->owner) 
+    #   referential loops (parent->collection->child->owner)
     #   when deep_coping.
     def resolve
       result = self.class.new
@@ -91,7 +91,7 @@ module Gorillib
       end
       result
     end
-    
+
     def resolve_value(value)
       return if value.nil?
       return value.resolve if value.respond_to? :resolve

data/lib/ironfan/broker/computer.rb

@@ -4,9 +4,9 @@ module Ironfan
     class Computer < Builder
 
       field :server,            Ironfan::Dsl::Server
-      collection :resources,    Whatever
-      collection :drives,       Ironfan::Broker::Drive
-      collection :providers,    Ironfan::Provider
+      collection :resources,    Ironfan::Provider::Resource, :key_method => :name
+      collection :drives,       Ironfan::Broker::Drive,      :key_method => :name
+      collection :providers,    Whatever,                    :key_method => :name
       delegate :[], :[]=, :include?, :delete, :to => :resources
 
       # Only used for bogus servers
@@ -21,6 +21,7 @@ module Ironfan
         volumes =               server.volumes.values
         volumes +=              server.implied_volumes
         volumes.each { |v| self.drive v.name, :volume => v }
+      rescue StandardError => err ; err.polish("#{self.class} on #{args}'") rescue nil ; raise
       end
 
       def name
@@ -128,10 +129,6 @@ module Ironfan
         @chef_client_script_content = Ironfan.safely{ File.read(script_filename) }
       end
 
-      def ssh_identity_file
-        resources[:key_pair].key_filename
-      end
-
 #       def ensure_resource(type)
 #         if type.multiple?
 #           existing = resources[type.resource_id]
@@ -182,6 +179,8 @@ module Ironfan
         self[:node]= value
       end
 
+      def environment ; server && server.environment ; end
+
       #
       # client
       #
@@ -200,9 +199,21 @@ module Ironfan
         self[:machine] = value
       end
       def dns_name         ; machine? ? machine.dns_name : nil ; end
-      def ssh_user         ; (server && server.selected_cloud) ? server.selected_cloud.ssh_user : nil ; end
       def bootstrap_distro ; (server && server.selected_cloud) ? server.selected_cloud.bootstrap_distro   : nil ; end
 
+      def keypair
+        resources[:keypair]
+      end
+
+      def ssh_user         ; (server && server.selected_cloud) ? server.selected_cloud.ssh_user : nil ; end
+
+      def ssh_identity_file
+        if    keypair                         then keypair.key_filename
+        elsif server && server.selected_cloud then server.selected_cloud.ssh_identity_file(self)
+        else nil
+        end
+      end
+
       #
       # Status flags
       #
@@ -248,6 +259,14 @@ module Ironfan
         running? || node?
       end
 
+      def receive_providers(objs)
+        objs = objs.map do |obj|
+          if obj.is_a?(String) then obj = Gorillib::Inflector.constantize(Gorillib::Inflector.camelize(obj.gsub(/\./, '/'))) ; end
+          obj
+        end
+        super(objs)
+      end
+
       def to_s
         "<#{self.class}(server=#{server}, resources=#{resources && resources.inspect_compact}, providers=#{providers && providers.inspect_compact})>"
       end
@@ -300,6 +319,10 @@ module Ironfan
         Ironfan.parallel(values) {|c| c.stop }
       end
 
+      def environments
+        map{|comp| comp.environment }.uniq
+      end
+
       #
       # Utility
       #

data/lib/ironfan/broker.rb

@@ -8,10 +8,11 @@ module Ironfan
   end
 
   class Broker < Builder
-    # Take in a Dsl::Cluster, return Computers populated with
-    #   all discovered resources that correlate, plus bogus computers
-    #   corresponding to
+    # Take in a Dsl::Cluster; return Computers populated with all discovered
+    #   resources that correlate; computers corresponding to partial or
+    #   unrecognizable resources are labeled as bogus.
     def discover!(cluster)
+
       # Get fully resolved servers, and build Computers using them
       computers = Computers.new(:cluster => cluster)
       #
@@ -22,6 +23,7 @@ module Ironfan
       end
       #
       Ironfan.step cluster.name, "Reconciling DSL and provider information", :cyan
+
       computers.correlate
       computers.validate
       #
@@ -39,5 +41,4 @@ module Ironfan
     end
 
   end
-
 end

data/lib/ironfan/dsl/cloud.rb

@@ -4,13 +4,17 @@ module Ironfan
     class Cloud < Ironfan::Dsl
       magic :default_cloud,           :boolean,       :default => false
 
-      # Factory out to subclasses 
-      def self.receive(obj,&block)
-        obj[:_type] = case obj[:name]
-          when        :ec2;           Ec2
-          when        :virtualbox;    VirtualBox
-          else;       raise "Unsupported cloud #{obj[:name]}"
-        end unless native?(obj)
+      # Factory out to subclasses
+      def self.receive(obj, &block)
+        if obj.is_a?(Hash)
+          obj = obj.symbolize_keys
+          obj[:_type] ||=
+            case obj[:name]
+            when :ec2        then Ec2
+            when :virtualbox then VirtualBox
+            else raise "Unsupported cloud #{obj[:name]}"
+            end
+        end
         super
       end
 
@@ -18,4 +22,4 @@ module Ironfan
     end
 
   end
-end
+end

data/lib/ironfan/dsl/compute.rb

@@ -1,14 +1,24 @@
 module Ironfan
   class Dsl
 
+    class RunListItem < Hash
+      Gorillib::Factory.register_factory(self, [self])
+      def name
+        self[:name]
+      end
+      def self.receive(hsh)
+        new.merge!(hsh.symbolize_keys)
+      end
+    end
+
     class Compute < Ironfan::Dsl
       @@run_list_rank = 0
       field      :name,         String
 
       # Resolve each of the following as a merge of their container's attributes and theirs
-      collection :run_list_items, Hash,                    :resolver => :merge_resolve
-      collection :clouds,       Ironfan::Dsl::Cloud,       :resolver => :merge_resolve
-      collection :volumes,      Ironfan::Dsl::Volume,      :resolver => :merge_resolve
+      collection :run_list_items, RunListItem,             :resolver => :merge_resolve, :key_method => :name
+      collection :clouds,       Ironfan::Dsl::Cloud,       :resolver => :merge_resolve, :key_method => :name
+      collection :volumes,      Ironfan::Dsl::Volume,      :resolver => :merge_resolve, :key_method => :name
 
       # Resolve these normally (overriding on each layer)
       magic      :environment,  Symbol,                    :default => :_default
@@ -81,4 +91,4 @@ module Ironfan
     end
 
   end
-end
+end

data/lib/ironfan/dsl/ec2.rb

@@ -15,17 +15,17 @@ module Ironfan
       magic :flavor,                    String,         :default => 't1.micro'
       magic :image_id,                  String
       magic :image_name,                String
-      magic :keypair,                   Whatever
+      magic :keypair,                   String
       magic :monitoring,                String
       magic :mount_ephemerals,          Hash,           :default => {}
       magic :permanent,                 :boolean,       :default => false
       magic :placement_group,           String
-      magic :provider,                  Ironfan::Provider,      :default => Ironfan::Provider::Ec2
+      magic :provider,                  Whatever,       :default => Ironfan::Provider::Ec2
       magic :public_ip,                 String
       magic :region,                    String,         :default => ->{ default_region }
       magic :ssh_user,                  String,         :default => ->{ image_info[:ssh_user] }
       magic :ssh_identity_dir,          String,         :default => ->{ Chef::Config.ec2_key_dir }
-      collection :security_groups,      Ironfan::Dsl::Ec2::SecurityGroup
+      collection :security_groups,      Ironfan::Dsl::Ec2::SecurityGroup, :key_method => :name
       magic :subnet,                    String
       magic :validation_key,            String,         :default => ->{ IO.read(Chef::Config.validation_key) rescue '' }
       magic :vpc,                       String
@@ -102,6 +102,14 @@ module Ironfan
         result
       end
 
+      def receive_provider(obj)
+        if obj.is_a?(String)
+          write_attribute :provider, Gorillib::Inflector.constantize(Gorillib::Inflector.camelize(obj.gsub(/\./, '/')))
+        else
+          super(obj)
+        end
+      end
+
       class SecurityGroup < Ironfan::Dsl
         field :name,                    String
         field :group_authorized,        Array, :default => []

data/lib/ironfan/dsl/virtualbox.rb

@@ -1,8 +1,8 @@
 module Ironfan
-  module Dsl
+  class Dsl
 
     class VirtualBox < Cloud
     end
 
   end
-end
+end

data/lib/ironfan/headers.rb

@@ -46,7 +46,7 @@ module Ironfan
       class EbsVolume < Ironfan::Provider::Resource; end
       class ElasticIp < Ironfan::Provider::Resource; end
       class Machine < Ironfan::IaasProvider::Machine; end
-      class KeyPair < Ironfan::Provider::Resource; end
+      class Keypair < Ironfan::Provider::Resource; end
       class PlacementGroup < Ironfan::Provider::Resource; end
       class SecurityGroup < Ironfan::Provider::Resource; end
     end

data/lib/ironfan/provider/ec2/ebs_volume.rb

@@ -51,6 +51,7 @@ module Ironfan
         def self.load!(cluster=nil)
           Ec2.connection.volumes.each do |vol|
             next if vol.blank?
+            next if %w[deleting deleted error].include?(vol.state.to_s)
             ebs = EbsVolume.new(:adaptee => vol)
             # Already have a volume by this name
             if recall? ebs.name
@@ -64,6 +65,11 @@ module Ironfan
           end
         end
 
+        def receive_adaptee(obj)
+          obj = Ec2.connection.volumes.new(obj) if obj.is_a?(Hash)
+          super
+        end
+
         def on_correlate(computer)
           drive = computer.drive(drivename)
           drive.disk = self

data/lib/ironfan/provider/ec2/elastic_ip.rb

@@ -3,8 +3,7 @@ module Ironfan
     class Ec2
 
       class ElasticIp < Ironfan::Provider::Resource
-
       end
     end
   end
-end
+end

data/lib/ironfan/provider/ec2/{key_pair.rb → keypair.rb}

@@ -2,20 +2,22 @@ module Ironfan
   class Provider
     class Ec2
 
-      class KeyPair < Ironfan::Provider::Resource
+      class Keypair < Ironfan::Provider::Resource
         delegate :_dump, :collection, :collection=, :connection,
             :connection=, :destroy, :fingerprint, :fingerprint=, :identity,
             :identity=, :name, :name=, :new_record?, :public_key,
             :public_key=, :reload, :requires, :requires_one, :save,
             :symbolize_keys, :wait_for, :writable?, :write,
           :to => :adaptee
-        field :key_filename,    String,
-            :default => ->{ "#{KeyPair.key_dir}/#{name}.pem" }
 
-        def self.shared?()      true;   end
-        def self.multiple?()    false;  end
-        def self.resource_type()        :key_pair;   end
-        def self.expected_ids(computer)   [computer.server.cluster_name];   end
+        field :key_filename,    String, :default => ->{ "#{Keypair.key_dir}/#{name}.pem" }
+
+        def self.shared?       ; true     ; end
+        def self.multiple?     ; false    ; end
+        def self.resource_type ; :keypair ; end
+        def self.expected_ids(computer)
+          [computer.server.cluster_name]
+        end
 
         def private_key
           File.open(key_filename, "rb").read
@@ -39,6 +41,11 @@ module Ironfan
           end
         end
 
+        def receive_adaptee(obj)
+          obj = Ec2.connection.key_pairs.new(obj) if obj.is_a?(Hash)
+          super
+        end
+
         #
         # Manipulation
         #
@@ -63,6 +70,7 @@ module Ironfan
           warn "Please set 'ec2_key_dir' in your knife.rb. Will use #{dir} as a default"
           dir
         end
+
       end
 
     end

data/lib/ironfan/provider/ec2/machine.rb

@@ -34,7 +34,6 @@ module Ironfan
             :vpc_id=, :wait_for,
           :to => :adaptee
 
-
         def self.shared?()      false;  end
         def self.multiple?()    false;  end
 #        def self.resource_type()        Ironfan::IaasProvider::Machine;   end
@@ -46,7 +45,8 @@ module Ironfan
           tags["Name"] || tags["name"] || id
         end
 
-        def public_hostname()           dns_name;      end
+        def public_hostname  ; dns_name ; end
+        def keypair          ; key_pair ; end
 
         def created?
           not ['terminated', 'shutting-down'].include? state
@@ -104,20 +104,26 @@ module Ironfan
         def self.load!(cluster=nil)
           Ec2.connection.servers.each do |fs|
             machine = new(:adaptee => fs)
-            if recall? machine.name
+            if (not machine.created?)
+              next unless Ironfan.chef_config[:include_terminated]
+              remember machine, :append_id => "terminated:#{machine.id}"
+            elsif recall? machine.name
               raise 'duplicate'
               machine.bogus <<                 :duplicate_machines
               recall(machine.name).bogus <<    :duplicate_machines
               remember machine, :append_id => "duplicate:#{machine.id}"
-            elsif machine.created?
+            else # never seen it
               remember machine
-            else
-              remember machine, :append_id => "terminated:#{machine.id}"
             end
             Chef::Log.debug("Loaded #{machine}")
           end
         end
 
+        def receive_adaptee(obj)
+          obj = Ec2.connection.servers.new(obj) if obj.is_a?(Hash)
+          super
+        end
+
         # Find active machines that haven't matched, but should have,
         #   make sure all bogus machines have a computer to attach to
         #   for display purposes
@@ -167,7 +173,7 @@ module Ironfan
             'name' =>         computer.name,
             'Name' =>         computer.name,
           }
-          Ec2.ensure_tags(tags,computer.machine)
+          Ec2.ensure_tags(tags, computer.machine)
 
           # register the new volumes for later save!, and tag appropriately
           computer.machine.volumes.each do |v|

data/lib/ironfan/provider/ec2/security_group.rb

@@ -32,6 +32,11 @@ module Ironfan
           end
         end
 
+        def receive_adaptee(obj)
+          obj = Ec2.connection.security_groups.new(obj) if obj.is_a?(Hash)
+          super
+        end
+
         def to_s
           if ip_permissions.present?
             perm_str = ip_permissions.map{|perm|
@@ -63,7 +68,12 @@ module Ironfan
           Ironfan.step(computer.server.cluster_name, "creating security groups", :blue)
           groups.each do |group|
             Ironfan.step(group, "  creating #{group} security group", :blue)
-            Ec2.connection.create_security_group(group.to_s,"Ironfan created group #{group}")
+            begin
+              Ec2.connection.create_security_group(group.to_s,"Ironfan created group #{group}")
+            rescue Fog::Compute::AWS::Error => e # InvalidPermission.Duplicate
+              Chef::Log.info("ignoring security group error: #{e}")
+              sleep 0.5  # quit racing so hard
+            end
           end
           load! # Get the native groups via reload
         end
@@ -134,7 +144,7 @@ module Ironfan
           begin
             fog_group.authorize_port_range(range,options)
           rescue Fog::Compute::AWS::Error => e      # InvalidPermission.Duplicate
-            Chef::Log.debug("ignoring #{e}")
+            Chef::Log.info("ignoring #{e}")
           end
         end
       end

data/lib/ironfan/provider/ec2.rb

@@ -5,7 +5,7 @@ module Ironfan
       self.handle = :ec2
 
       def self.resources
-        [ Machine, EbsVolume, KeyPair, SecurityGroup ]
+        [ Machine, EbsVolume, Keypair, SecurityGroup ]
       end
 
       #

data/lib/ironfan/provider.rb

@@ -6,13 +6,17 @@ module Ironfan
   class Provider < Builder
     class_attribute :handle
 
-    def self.receive(obj,&block)
-      obj[:_type] = case obj[:name]
-        when    :chef;          Chef
-        when    :ec2;           Ec2
-        when    :virtualbox;    VirtualBox
-        else;   raise "Unsupported provider #{obj[:name]}"
-      end unless native?(obj)
+    def self.receive(obj, &block)
+      if obj.is_a?(Hash)
+        obj = obj.symbolize_keys
+        obj[:_type] =
+          case obj[:name]
+          when :chef        then Chef
+          when :ec2         then Ec2
+          when :virtualbox  then VirtualBox
+          else raise "Unsupported provider #{obj[:name]}"
+          end
+      end
       super
     end
 
@@ -50,6 +54,11 @@ module Ironfan
 
       def self.handle ; name.to_s.gsub(/.*::/,'').to_sym ; end
 
+      def self.receive(obj)
+        obj = obj.symbolize_keys if obj.is_a?(Hash)
+        super(obj)
+      end
+
       #
       # Flags
       #

data/lib/ironfan/requirements.rb

@@ -2,11 +2,9 @@
 require 'gorillib/builder'
 require 'gorillib/resolution'
 
-
 # Pre-declaration of class hierarchy
 require 'ironfan/headers'
 
-
 # DSL for cluster descriptions
 require 'ironfan/dsl'
 require 'ironfan/builder'
@@ -34,7 +32,7 @@ require 'ironfan/provider/chef/role'
 require 'ironfan/provider/ec2'
 require 'ironfan/provider/ec2/ebs_volume'
 require 'ironfan/provider/ec2/machine'
-require 'ironfan/provider/ec2/key_pair'
+require 'ironfan/provider/ec2/keypair'
 require 'ironfan/provider/ec2/placement_group'
 require 'ironfan/provider/ec2/security_group'
 

data/lib/ironfan.rb

@@ -29,7 +29,7 @@ module Ironfan
     raise 'missing block' unless block_given?
     results = []
     [targets].flatten.each_with_index.map do |target, idx|
-      sleep(0.1) # avoid hammering with simultaneous requests
+      sleep(0.25) # avoid hammering with simultaneous requests
       Thread.new(target) do |target|
         results[idx] = safely(target.inspect) do
           yield target
@@ -80,19 +80,19 @@ module Ironfan
   # doesn't define the requested cluster.
   #
   # @return [Ironfan::Cluster] the requested cluster
-  def self.load_cluster(cluster_name)
-    cluster = cluster_name.to_sym
-    raise ArgumentError, "Please supply a cluster name" if cluster_name.to_s.empty?
-    return @@clusters[cluster] if @@clusters[cluster]
+  def self.load_cluster(name)
+    name = name.to_sym
+    raise ArgumentError, "Please supply a cluster name" if name.to_s.empty?
+    return @@clusters[name] if @@clusters[name]
 
-    cluster_file = cluster_filenames[cluster_name] or die("Couldn't find a definition for #{cluster_name} in cluster_path: #{cluster_path.inspect}")
+    cluster_file = cluster_filenames[name] or raise("Couldn't find a definition for #{name} in cluster_path: #{cluster_path.inspect}")
 
     Chef::Log.info("Loading cluster #{cluster_file}")
 
     require cluster_file
-    unless @@clusters[cluster] then  die("#{cluster_file} was supposed to have the definition for the #{cluster_name} cluster, but didn't") end
+    unless @@clusters[name] then  die("#{cluster_file} was supposed to have the definition for the #{name} cluster, but didn't") end
 
-    @@clusters[cluster]
+    @@clusters[name]
   end
 
   #
@@ -105,7 +105,7 @@ module Ironfan
     cluster_path.each do |cp_dir|
       Dir[ File.join(cp_dir, '*.rb') ].each do |filename|
         cluster_name = File.basename(filename).gsub(/\.rb$/, '')
-        @cluster_filenames[cluster_name] ||= filename
+        @cluster_filenames[cluster_name.to_sym] ||= filename
       end
     end
     @cluster_filenames
@@ -149,7 +149,11 @@ module Ironfan
   end
 
   def self.substep(name, desc)
-    step(name, "  - #{desc}", :gray) if chef_config[:verbosity] >= 1
+    step(name, "  - #{desc}", :gray) if verbosity >= 1
+  end
+
+  def self.verbosity
+    chef_config[:verbosity].to_i
   end
 
   # Output a TODO to the logs if you've switched on pestering

data/spec/chef/cluster_bootstrap_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+ironfan_go!
+require 'chef/knife/cluster_bootstrap'
+
+describe Chef::Knife::ClusterBootstrap do
+  let(:cluster) do
+    Ironfan.load_cluster(:gunbai)
+  end
+
+  let(:target) do
+    Ironfan.broker.discover!(cluster)
+  end
+
+  let(:computers) do
+    Ironfan::Broker::Computers.receive(
+      MultiJson.load(
+        File.open(Pathname.path_to(:fixtures, 'gunbai_slice.json'))))
+  end
+
+  subject do
+    described_class.new(slice)
+  end
+
+  context 'getting slice' do
+    before do
+      subject.stub(:relevant?){ true }
+      subject.stub(:run_bootstrap)
+      subject.config[:yes] = true
+    end
+    context 'full slice' do
+      let(:slice){ ['gunbai'] }
+      it 'fails if there are multiple environments' do
+        expect{ subject.run }.to raise_error("Cannot bootstrap multiple chef environments")
+      end
+    end
+    context 'partial slice' do
+      let(:slice){ ['gunbai-hub'] }
+      it 'runs' do
+        subject.should_receive(:run_bootstrap).once
+        subject.run
+      end
+    end
+
+  end
+
+  # it 'loads computers from json' do
+  #   computers.length.should == 2
+  #   computers.first.server.fullname.should == 'gunbai-hub-0'
+  # end
+end

data/spec/fixtures/gunbai.rb

@@ -0,0 +1,24 @@
+Ironfan.cluster 'gunbai' do
+  cloud(:ec2) do
+    permanent           false
+    availability_zones ['us-east-1d']
+    flavor              't1.micro'
+    backing             'ebs'
+    image_name          'natty'
+    bootstrap_distro    'ubuntu10.04-ironfan'
+    chef_client_script  'client.rb'
+    mount_ephemerals
+  end
+
+  environment           :dev
+
+  role                  :ssh
+  cloud(:ec2).security_group(:ssh).authorize_port_range(22..22)
+
+  facet :hub do
+  end
+
+  facet :spoke do
+    environment           :other
+  end
+end

data/spec/fixtures/gunbai_slice.json

@@ -0,0 +1,330 @@
+[
+  {
+    "server": {
+      "name": "0",
+      "run_list_items": [
+        {
+          "name": "role[ssh]",
+          "rank": 1,
+          "placement": "normal"
+        },
+        {
+          "name": "role[gunbai_cluster]",
+          "rank": 2,
+          "placement": "last"
+        },
+        {
+          "name": "role[gunbai_hub]",
+          "rank": 3,
+          "placement": "last"
+        }
+      ],
+      "clouds": [
+        {
+          "name": "ec2",
+          "default_cloud": false,
+          "availability_zones": [
+            "us-east-1d"
+          ],
+          "backing": "ebs",
+          "bits": 64,
+          "bootstrap_distro": "ubuntu10.04-ironfan",
+          "chef_client_script": "client.rb",
+          "default_availability_zone": "us-east-1d",
+          "flavor": "t1.micro",
+          "image_id": null,
+          "image_name": "natty",
+          "keypair": null,
+          "monitoring": null,
+          "mount_ephemerals": {
+
+          },
+          "permanent": false,
+          "placement_group": null,
+          "provider": "Ironfan::Provider::Ec2",
+          "public_ip": null,
+          "region": "us-east-1",
+          "ssh_user": "ubuntu",
+          "ssh_identity_dir": "/Users/flip/ics/sysadmin/enterprise-homebase/knife/credentials/ec2_keys",
+          "security_groups": [
+            {
+              "name": "ssh",
+              "group_authorized": [
+
+              ],
+              "group_authorized_by": [
+
+              ],
+              "range_authorizations": [
+                [
+                  "22..22",
+                  "0.0.0.0/0",
+                  "tcp"
+                ]
+              ],
+              "_type": "ironfan.dsl.ec2.security_group"
+            }
+          ],
+          "subnet": null,
+          "validation_key": "-----BEGIN RSA PRIVATE KEY-----\nSNIP\n-----END RSA PRIVATE KEY-----\n",
+          "vpc": null,
+          "_type": "ironfan.dsl.ec2"
+        }
+      ],
+      "volumes": [
+
+      ],
+      "environment": "dev",
+      "use_cloud": null,
+      "cluster_role": {
+        "name": "gunbai_cluster",
+        "override_attributes": {
+
+        },
+        "default_attributes": {
+
+        },
+        "_type": "ironfan.dsl.role"
+      },
+      "facet_role": {
+        "name": "gunbai_hub",
+        "override_attributes": {
+
+        },
+        "default_attributes": {
+
+        },
+        "_type": "ironfan.dsl.role"
+      },
+      "cluster_name": "gunbai",
+      "facet_name": "hub",
+      "_type": "ironfan.dsl.server"
+    },
+    "resources": [
+      {
+        "adaptee": {"name":"gunbai","fingerprint":"a2:9a:15:b3:bc:95:68:99:2c:91:43:9c:3b:6b:99:00:11:e9:b2:5e"},
+        "bogus": [
+
+        ],
+        "key_filename": "/Users/flip/ics/sysadmin/enterprise-homebase/knife/credentials/ec2_keys/gunbai.pem",
+        "_type": "ironfan.provider.ec2.keypair"
+      },
+      {
+        "adaptee": {"ip_permissions":[{"groups":[],"ipRanges":[{"cidrIp":"0.0.0.0/0"}],"ipProtocol":"tcp","fromPort":22,"toPort":22}],"ip_permissions_egress":[],"owner_id":"689981200839","group_id":"sg-a0f174c9","name":"ssh","description":"cluster_chef+generated+group+ssh"},
+        "bogus": [
+
+        ],
+        "ensured": false,
+        "_type": "ironfan.provider.ec2.security_group"
+      }
+    ],
+    "drives": [
+      {
+        "node": {
+
+        },
+        "disk": null,
+        "volume": {
+          "name": "root",
+          "attachable": "ebs",
+          "availability_zone": null,
+          "create_at_launch": false,
+          "device": "/dev/sda1",
+          "formattable": false,
+          "fstype": "ext4",
+          "in_raid": false,
+          "keep": false,
+          "mount_dump": null,
+          "mount_pass": null,
+          "mount_options": "defaults,nouuid,noatime",
+          "mount_point": "/",
+          "mountable": true,
+          "size": null,
+          "volume_id": null,
+          "resizable": false,
+          "snapshot_id": null,
+          "snapshot_name": null,
+          "tags": {
+
+          },
+          "_type": "ironfan.dsl.volume"
+        },
+        "name": "root",
+        "_type": "ironfan.broker.drive"
+      }
+    ],
+    "providers": [
+      "Ironfan::Provider::ChefServer",
+      "Ironfan::Provider::Ec2"
+    ],
+    "name": null,
+    "bogus": [
+
+    ],
+    "_type": "ironfan.broker.computer"
+  },
+  {
+    "server": {
+      "name": "0",
+      "run_list_items": [
+        {
+          "name": "role[ssh]",
+          "rank": 1,
+          "placement": "normal"
+        },
+        {
+          "name": "role[gunbai_cluster]",
+          "rank": 4,
+          "placement": "last"
+        },
+        {
+          "name": "role[gunbai_spoke]",
+          "rank": 5,
+          "placement": "last"
+        }
+      ],
+      "clouds": [
+        {
+          "name": "ec2",
+          "default_cloud": false,
+          "availability_zones": [
+            "us-east-1d"
+          ],
+          "backing": "ebs",
+          "bits": 64,
+          "bootstrap_distro": "ubuntu10.04-ironfan",
+          "chef_client_script": "client.rb",
+          "default_availability_zone": "us-east-1d",
+          "flavor": "t1.micro",
+          "image_id": null,
+          "image_name": "natty",
+          "keypair": null,
+          "monitoring": null,
+          "mount_ephemerals": {
+
+          },
+          "permanent": false,
+          "placement_group": null,
+          "provider": "Ironfan::Provider::Ec2",
+          "public_ip": null,
+          "region": "us-east-1",
+          "ssh_user": "ubuntu",
+          "ssh_identity_dir": "/Users/flip/ics/sysadmin/enterprise-homebase/knife/credentials/ec2_keys",
+          "security_groups": [
+            {
+              "name": "ssh",
+              "group_authorized": [
+
+              ],
+              "group_authorized_by": [
+
+              ],
+              "range_authorizations": [
+                [
+                  "22..22",
+                  "0.0.0.0/0",
+                  "tcp"
+                ]
+              ],
+              "_type": "ironfan.dsl.ec2.security_group"
+            }
+          ],
+          "subnet": null,
+          "validation_key": "-----BEGIN RSA PRIVATE KEY-----\nSNIP\n-----END RSA PRIVATE KEY-----\n",
+          "vpc": null,
+          "_type": "ironfan.dsl.ec2"
+        }
+      ],
+      "volumes": [
+
+      ],
+      "environment": "dev",
+      "use_cloud": null,
+      "cluster_role": {
+        "name": "gunbai_cluster",
+        "override_attributes": {
+
+        },
+        "default_attributes": {
+
+        },
+        "_type": "ironfan.dsl.role"
+      },
+      "facet_role": {
+        "name": "gunbai_spoke",
+        "override_attributes": {
+
+        },
+        "default_attributes": {
+
+        },
+        "_type": "ironfan.dsl.role"
+      },
+      "cluster_name": "gunbai",
+      "facet_name": "spoke",
+      "_type": "ironfan.dsl.server"
+    },
+    "resources": [
+      {
+        "adaptee": {"name":"gunbai","fingerprint":"a2:3b:9a:15:b3:bc:95:68:99:9e:2c:91:43:9c:6b:a5:6f:e9:b2:5e"},
+        "bogus": [
+
+        ],
+        "key_filename": "/Users/flip/ics/sysadmin/enterprise-homebase/knife/credentials/ec2_keys/gunbai.pem",
+        "_type": "ironfan.provider.ec2.keypair"
+      },
+      {
+        "adaptee": {"ip_permissions":[{"groups":[],"ipRanges":[{"cidrIp":"0.0.0.0/0"}],"ipProtocol":"tcp","fromPort":22,"toPort":22}],"ip_permissions_egress":[],"owner_id":"689981200839","group_id":"sg-a0f174c9","name":"ssh","description":"cluster_chef+generated+group+ssh"},
+        "bogus": [
+
+        ],
+        "ensured": false,
+        "_type": "ironfan.provider.ec2.security_group"
+      }
+    ],
+    "drives": [
+      {
+        "node": {
+
+        },
+        "disk": null,
+        "volume": {
+          "name": "root",
+          "attachable": "ebs",
+          "availability_zone": null,
+          "create_at_launch": false,
+          "device": "/dev/sda1",
+          "formattable": false,
+          "fstype": "ext4",
+          "in_raid": false,
+          "keep": false,
+          "mount_dump": null,
+          "mount_pass": null,
+          "mount_options": "defaults,nouuid,noatime",
+          "mount_point": "/",
+          "mountable": true,
+          "size": null,
+          "volume_id": null,
+          "resizable": false,
+          "snapshot_id": null,
+          "snapshot_name": null,
+          "tags": {
+
+          },
+          "_type": "ironfan.dsl.volume"
+        },
+        "name": "root",
+        "_type": "ironfan.broker.drive"
+      }
+    ],
+    "providers": [
+      "Ironfan::Provider::ChefServer",
+      "Ironfan::Provider::Ec2"
+    ],
+    "name": null,
+    "bogus": [
+
+    ],
+    "_type": "ironfan.broker.computer"
+  }
+]

data/spec/ironfan/ec2/cloud_provider.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+require 'ironfan'
+
+describe Ironfan::Dsl::Cluster do
+  let (:cluster) do
+    Ironfan.cluster "sparky" do
+
+      cloud(:ec2) do
+        security_group(:ssh).authorize_port_range 22..22
+        flavor 't1.micro'
+      end
+
+      facet :web do
+        instances 3
+      end
+
+    end
+  end
+
+  describe 'web facet server resolution' do
+    before { @facet = cluster.facets.values.first }
+    subject { @facet }
+    its(:name) { should eql "web" }
+
+    it 'should have the right number of servers' do
+      @facet.servers.length.should == 3
+    end
+
+    it 'should have one cloud provider, EC2' do
+      @facet.servers[0].clouds.keys.should == [ :ec2 ]
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -2,6 +2,31 @@ $:.unshift File.expand_path('../../lib', __FILE__)
 require 'chef'
 require 'chef/knife'
 require 'fog'
-
 Fog.mock!
 Fog::Mock.delay = 0
+
+require 'gorillib/pathname'
+
+Pathname.register_paths(
+  code:        File.expand_path('../..', __FILE__),
+  fixtures:    [:code, 'spec', 'fixtures'],
+  )
+
+RSpec.configure do |config|
+  def ironfan_go!
+    Chef::Knife.new.configure_chef
+    Chef::Config.instance_eval do
+      knife.merge!({
+          :aws_access_key_id => 'access_key',
+          :aws_secret_access_key => 'secret',
+        })
+      cluster_path Pathname.path_to(:fixtures).to_s
+    end
+
+    require 'ironfan'
+
+    Ironfan.ui          = Chef::Knife::UI.new(STDOUT, STDERR, STDIN, {})
+    Ironfan.chef_config = { :verbosity => 0 }
+    Ironfan.cluster_path
+  end
+end

metadata

@@ -2,7 +2,7 @@
 name: ironfan
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 4.2.1
+  version: 4.2.2
 platform: ruby
 authors: 
 - Infochimps
@@ -203,14 +203,18 @@ files:
 - lib/ironfan/provider/ec2.rb
 - lib/ironfan/provider/ec2/ebs_volume.rb
 - lib/ironfan/provider/ec2/elastic_ip.rb
-- lib/ironfan/provider/ec2/key_pair.rb
+- lib/ironfan/provider/ec2/keypair.rb
 - lib/ironfan/provider/ec2/machine.rb
 - lib/ironfan/provider/ec2/placement_group.rb
 - lib/ironfan/provider/ec2/security_group.rb
 - lib/ironfan/provider/virtualbox.rb
 - lib/ironfan/provider/virtualbox/machine.rb
 - lib/ironfan/requirements.rb
+- spec/chef/cluster_bootstrap_spec.rb
+- spec/fixtures/gunbai.rb
+- spec/fixtures/gunbai_slice.json
 - spec/ironfan/cluster_spec.rb
+- spec/ironfan/ec2/cloud_provider.rb
 - spec/ironfan/ec2/cloud_provider_spec.rb
 - spec/ironfan/ec2/security_group_spec.rb
 - spec/spec_helper.rb
@@ -230,7 +234,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -2791351356463943124
+      hash: 3459217144726680577
       segments: 
       - 0
       version: "0"
@@ -252,5 +256,9 @@ test_files:
 - spec/ironfan/cluster_spec.rb
 - spec/ironfan/ec2/cloud_provider_spec.rb
 - spec/ironfan/ec2/security_group_spec.rb
+- spec/ironfan/ec2/cloud_provider.rb
+- spec/chef/cluster_bootstrap_spec.rb
+- spec/fixtures/gunbai_slice.json
+- spec/fixtures/gunbai.rb
 - spec/spec_helper.rb
 - spec/test_config.rb