ironfan 4.0.9 → 4.1.0

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# v4.1.0: several bug-fixes and code cleanup
+* Splat the args to DSL::Volume.snapshot_id so we can call it properly (fixes #161)
+* cloud(:ec2) lets you declare bitness (fixes #147)
+* Better logging; errors within Ironfan.parallel don't crash the world (pull #167)
+* Like any good hitman, knife cluster kill should 'take care of' errant clients (pull #168)
+
 # v4.0.9:
 * Making bootstrap work again (fixes #159)
 

data/VERSION

@@ -1 +1 @@
-4.0.9
+4.1.0

data/ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.0.9"
+  s.version = "4.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-09-20"
+  s.date = "2012-09-24"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [

data/lib/chef/knife/cluster_launch.rb

@@ -79,10 +79,12 @@ class Chef
         ui.info("")
         section("Launching computers", :green)
         display(target)
-        target.launch
+        launched = target.launch
 
         # As each server finishes, configure it
-        Ironfan.parallel(target.values) do |computer|
+        Ironfan.parallel(launched) do |computer|
+          if (computer.is_a?(Exception)) then ui.warn "Error launching #{computer.inspect}; skipping after-launch tasks."; next; end
+          Ironfan.step(computer.name, 'launching', :white)
           perform_after_launch_tasks(computer)
         end
         # progressbar_for_threads(watcher_threads)
@@ -91,12 +93,14 @@ class Chef
       end
 
       def perform_after_launch_tasks(computer)
+        Ironfan.step(computer.name, 'waiting for ready', :white)
         # Wait for machine creation on amazon side
 #         server.fog_server.wait_for{ ready? }
         computer.machine.wait_for{ ready? }
 
         # Try SSH
         unless config[:dry_run]
+          Ironfan.step(computer.name, 'trying ssh', :white)
 #           nil until tcp_test_ssh(server.fog_server.dns_name){ sleep @initial_sleep_delay ||= 10  }
           nil until tcp_test_ssh(computer.machine.dns_name){ sleep @initial_sleep_delay ||= 10  }
         end
@@ -107,11 +111,14 @@ class Chef
 
         # Attach volumes, etc
 #         server.sync_to_cloud
+        Ironfan.step(computer.name, 'final provisioning', :white)
         computer.save
 
         # Run Bootstrap
-        Chef::Log.warn "UNTESTED --bootstrap"
-        run_bootstrap(computer) if config[:bootstrap]
+        if config[:bootstrap]
+          Chef::Log.warn "UNTESTED --bootstrap"
+          run_bootstrap(computer)
+        end
       end
 
       def tcp_test_ssh(hostname)

data/lib/chef/knife/cluster_show.rb

@@ -42,19 +42,48 @@ class Chef
         # Load the cluster/facet/slice/whatever
         target = get_slice(* @name_args)
 
+        dump_command_config
+        dump_chef_config
         #
-        # Dump entire contents of objects if -VV flag given
-        #
-        if config[:verbosity] >= 2
-          target.each do |computer|
-            Chef::Log.debug( "Computer #{computer.name}: #{JSON.pretty_generate(computer.to_wire)}" )
-          end
+        target.each do |computer|
+          dump_computer(computer)
         end
 
         # Display same
         display(target)
+      end
+
+    protected
+
+      def dump_computer(computer)
+        with_verbosity 1 do
+          dump("Computer #{computer.name} (#{computer.class})", computer.to_wire)
+        end
+      end
 
+      def dump_command_config
+        with_verbosity 2 do
+          Chef::Log.info( ["", "*"*50, "", "Command Config", ""].join("\n") )
+          dump("Command config", self.config)
+        end
       end
+
+      def dump_chef_config
+        with_verbosity 2 do
+          chef_config_hash = Hash[Chef::Config.keys.map{|key| [key, Chef::Config[key]]}]
+          dump("Chef Config", chef_config_hash)
+        end
+      end
+
+      def dump(title, hsh)
+        Chef::Log.info( ["", "*"*50, "", "#{title}: ", ""].join("\n") )
+        Chef::Log.info( MultiJson.dump(hsh, pretty: true ) )
+      end
+
+      def with_verbosity(num)
+        yield if config[:verbosity] >= num
+      end
+
     end
   end
 end

data/lib/chef/knife/ironfan_knife_common.rb

@@ -37,9 +37,13 @@ module Ironfan
         ui.warn("Please specify server slices joined by dashes and not separate args:\n\n  knife cluster #{sub_command} #{slice_string}\n\n")
       end
       cluster_name, facet_name, slice_indexes = slice_string.split(/[\s\-]/, 3)
-      ui.info("Inventorying servers in #{predicate_str(cluster_name, facet_name, slice_indexes)}")
-      cluster = Ironfan.load_cluster(cluster_name)
+      desc = predicate_str(cluster_name, facet_name, slice_indexes)
+      #
+      ui.info("Inventorying servers in #{desc}")
+      cluster   = Ironfan.load_cluster(cluster_name)
       computers =  broker.discover! cluster
+      Chef::Log.info("Inventoried #{computers.size} computers")
+      #
       computers.slice(facet_name, slice_indexes)
     end
 
@@ -80,7 +84,7 @@ module Ironfan
     def display(target, display_style=nil, &block)
       display_style ||= (config[:verbosity] == 0 ? :default : :expanded)
 #       target.display(ui, display_style, &block)
-      broker.display(target,display_style)
+      broker.display(target, display_style)
     end
 
     #
@@ -115,25 +119,24 @@ module Ironfan
     end
 
     def bootstrapper(computer)
-      server = computer.server
-      cloud = server.selected_cloud
-      hostname = computer.machine.dns_name
-
+      server   = computer.server
+      hostname = computer.dns_name
+      #
       bootstrap = Chef::Knife::Bootstrap.new
       bootstrap.config.merge!(config)
-
+      #
       bootstrap.name_args               = [ hostname ]
       bootstrap.config[:computer]       = computer
       bootstrap.config[:server]         = server
       bootstrap.config[:run_list]       = server.run_list
-      bootstrap.config[:ssh_user]       = config[:ssh_user]       || cloud.ssh_user
+      bootstrap.config[:ssh_user]       = config[:ssh_user]       || computer.ssh_user
       bootstrap.config[:attribute]      = config[:attribute]
       bootstrap.config[:identity_file]  = config[:identity_file]  || computer.ssh_identity_file
-      bootstrap.config[:distro]         = config[:distro]         || cloud.bootstrap_distro
+      bootstrap.config[:distro]         = config[:distro]         || computer.bootstrap_distro
       bootstrap.config[:use_sudo]       = true unless config[:use_sudo] == false
       bootstrap.config[:chef_node_name] = server.fullname
       bootstrap.config[:client_key]     = ( computer.client.private_key rescue nil )
-
+      #
       bootstrap
     end
 
@@ -143,14 +146,11 @@ module Ironfan
         ui.info "Skipping: bootstrap #{computer.name} with #{JSON.pretty_generate(bs.config)}"
         return
       end
-      begin
+      #
+      Ironfan.step(computer.name, "Running bootstrap")
+      Chef::Log.info("Bootstrapping:\n  Computer #{computer}\n  Bootstrap config #{bs.config}")
+      Ironfan.safely([computer, bs.config].inspect) do
         bs.run
-      rescue StandardError => e
-        ui.warn e
-        ui.warn e.backtrace
-        ui.warn ""
-        ui.warn computer.inspect
-        ui.warn ""
       end
     end
 

data/lib/ironfan/broker/computer.rb

@@ -2,16 +2,16 @@ module Ironfan
   class Broker
 
     class Computer < Builder
+
+      field :server,            Ironfan::Dsl::Server
       collection :resources,    Whatever
       collection :drives,       Ironfan::Broker::Drive
       collection :providers,    Ironfan::Provider
-      field :server,            Ironfan::Dsl::Server
-      delegate :[],:[]=,:include?,:delete,
-          :to =>                :resources
+      delegate :[], :[]=, :include?, :delete, :to => :resources
 
       # Only used for bogus servers
       field :name,              String
-      field :bogus,             Array,          :default => []
+      field :bogus,             Array,  :default => []
 
       def initialize(*args)
         super
@@ -61,14 +61,21 @@ module Ironfan
         target_resources = chosen_resources(options)
         resources.each do |res|
           next unless target_resources.include? res.class
-          res.destroy unless res.shared?
+          descriptor = "#{res.class} named #{res.name}"
+          if res.shared?
+            Chef::Log.debug("Not killing shared resource #{descriptor}")
+          else
+            Ironfan.step(self.name, "Killing #{descriptor}")
+            res.destroy
+          end
         end
       end
 
       def launch
         ensure_dependencies
-        providers[:iaas].machine_class.create! self
+        iaas_provider.machine_class.create! self
         save
+        self
       end
 
       def stop
@@ -87,6 +94,9 @@ module Ironfan
         chosen_resources(options).each {|res| res.save! self}
       end
 
+      def chef_provider ; providers[:chef] ; end
+      def iaas_provider ; providers[:iaas] ; end
+
       #
       # Utility
       #
@@ -105,7 +115,7 @@ module Ironfan
 
       def ensure_dependencies
         chosen_resources.each do |res|
-#           ensure_resource res unless res < Ironfan::IaasProvider::Machine
+          # ensure_resource res unless res < Ironfan::IaasProvider::Machine
           res.create! self unless res < Ironfan::IaasProvider::Machine
         end
       end
@@ -126,7 +136,7 @@ module Ironfan
 #         if type.multiple?
 #           existing = resources[type.resource_id]
 #         else
-#           
+#
 #         end
 #       end
 
@@ -165,21 +175,33 @@ module Ironfan
       def bogus
         resources.values.map(&:bogus).flatten
       end
+      def node
+        self[:node]
+      end
+      def node=(value)
+        self[:node]= value
+      end
+
+      #
+      # client
+      #
       def client
         self[:client]
       end
+      def private_key ; client? ? client.private_key : nil ; end
+
+      #
+      # Machine
+      #
       def machine
         self[:machine]
       end
       def machine=(value)
         self[:machine] = value
       end
-      def node
-        self[:node]
-      end
-      def node=(value)
-        self[:node]= value
-      end
+      def dns_name         ; machine? ? machine.dns_name : nil ; end
+      def ssh_user         ; (server && server.selected_cloud) ? server.selected_cloud.ssh_user : nil ; end
+      def bootstrap_distro ; (server && server.selected_cloud) ? server.selected_cloud.bootstrap_distro   : nil ; end
 
       #
       # Status flags
@@ -197,7 +219,7 @@ module Ironfan
         not machine.nil?
       end
       def killable?
-        not permanent? and (node? or created?)
+        not permanent? and (node? || client? || created?)
       end
       def launchable?
         not bogus? and not created?
@@ -220,14 +242,22 @@ module Ironfan
         machine? and machine.stopped?
       end
 
+      # @return [Boolean] true if machine is likely to be reachable by ssh
+      def sshable?
+        # if there's any hope of success, give it a try.
+        running? || node?
+      end
+
+      def to_s
+        "<#{self.class}(server=#{server}, resources=#{resources && resources.inspect_compact}, providers=#{providers && providers.inspect_compact})>"
+      end
     end
 
     class Computers < Gorillib::ModelCollection
-      self.item_type    = Computer
-      self.key_method   = :object_id
-      delegate :first, :map, :any?,
-          :to           => :values
-      attr_accessor     :cluster
+      self.item_type  = Computer
+      self.key_method = :object_id
+      delegate        :first, :map, :any?, :to => :values
+      attr_accessor   :cluster
 
       def initialize(*args)
         super
@@ -236,19 +266,19 @@ module Ironfan
         create_expected!
       end
 
-      # 
+      #
       # Discovery
       #
       def correlate
-        values.each {|c| c.correlate }
+        values.each{|c| c.correlate }
       end
 
       def validate
         providers = values.map {|c| c.providers.values}.flatten
         computers = self
 
-        values.each {|c| c.validate }
-        providers.each {|p| p.validate computers }
+        values.each{|c|    c.validate }
+        providers.each{|p| p.validate computers }
       end
 
       #
@@ -258,16 +288,16 @@ module Ironfan
         Ironfan.parallel(values) {|c| c.kill(options) }
       end
       def launch
-        Ironfan.parallel(values) {|c| c. launch }
+        Ironfan.parallel(values) {|c| c.launch }
       end
       def save(options={})
-        Ironfan.parallel(values) {|c| c. save(options) }
+        Ironfan.parallel(values) {|c| c.save(options) }
       end
       def start
-        Ironfan.parallel(values) {|c| c. start }
+        Ironfan.parallel(values) {|c| c.start }
       end
       def stop
-        Ironfan.parallel(values) {|c| c. stop }
+        Ironfan.parallel(values) {|c| c.stop }
       end
 
       #
@@ -307,6 +337,7 @@ module Ironfan
         end
         result
       end
+
       def build_slice_array(slice_indexes)
         return [] if slice_indexes.nil?
         raise "Bad slice_indexes: #{slice_indexes}" if slice_indexes =~ /[^0-9\.,]/
@@ -317,7 +348,11 @@ module Ironfan
       def joined_names
         values.map(&:name).join(", ").gsub(/, ([^,]*)$/, ' and \1')
       end
+
+      def to_s
+        "#{self.class}[#{values.map(&:name).join(",")}]"
+      end
     end
 
   end
-end
+end

data/lib/ironfan/broker.rb

@@ -1,6 +1,6 @@
 # This module is intended to read in a cluster DSL description, and broker
 #   out to the various cloud providers, to control instance life-cycle and
-#   handle provider-specific amenities (SecurityGroup, Volume, etc.) for 
+#   handle provider-specific amenities (SecurityGroup, Volume, etc.) for
 #   them.
 module Ironfan
   def self.broker
@@ -10,15 +10,21 @@ module Ironfan
   class Broker < Builder
     # Take in a Dsl::Cluster, return Computers populated with
     #   all discovered resources that correlate, plus bogus computers
-    #   corresponding to 
+    #   corresponding to
     def discover!(cluster)
       # Get fully resolved servers, and build Computers using them
       computers = Computers.new(:cluster => cluster.resolve)
-      providers = computers.map{|c| c.providers.values}.flatten.uniq
-
-      providers.each {|p| p.load cluster }
+      #
+      providers = computers.map{|c| c.providers.values }.flatten.uniq
+      providers.each do |provider|
+        Ironfan.step cluster.name, "Loading #{provider.handle}", :cyan
+        provider.load cluster
+      end
+      #
+      Ironfan.step cluster.name, "Reconciling DSL and provider information", :cyan
       computers.correlate
       computers.validate
+      #
       computers
     end
 
@@ -34,4 +40,4 @@ module Ironfan
 
   end
 
-end
+end

data/lib/ironfan/deprecated.rb

@@ -2,7 +2,7 @@ module Ironfan
   def self.deprecated call, replacement=nil
     correction = ", " if replacement
     ui.error "The '#{call}' statement is deprecated#{correction} (in #{caller(2).first.inspect}). It looks like you are using an outdated DSL definition: please see https://github.com/infochimps-labs/ironfan/wiki/Upgrading-to-v4 for all the necessary upgrade steps."
-    exit(1)
+    raise StandardError, "Deprecated call to #{call} - #{replacement}", caller
   end
 
   class Dsl
@@ -11,7 +11,7 @@ module Ironfan
         Ironfan.deprecated 'defaults'
       end
     end
-    
+
     class Compute
       def cloud(provider=nil)
         if provider.nil?

data/lib/ironfan/dsl/ec2.rb

@@ -8,38 +8,46 @@ module Ironfan
     class Ec2 < Cloud
       magic :availability_zones,        Array,          :default => ['us-east-1d']
       magic :backing,                   String,         :default => 'ebs'
-      magic :bootstrap_distro,          String
+      magic :bits,                      Integer,        :default => ->{ flavor_info[:bits] }
+      magic :bootstrap_distro,          String,         :default => ->{ image_info[:bootstrap_distro] }
       magic :chef_client_script,        String
-      magic :default_availability_zone, String,         :default => ->{availability_zones.first}
+      magic :default_availability_zone, String,         :default => ->{ availability_zones.first }
       magic :flavor,                    String,         :default => 't1.micro'
-      magic :image_name,                String
       magic :image_id,                  String
+      magic :image_name,                String
       magic :keypair,                   Whatever
-      magic :mount_ephemerals,          Hash,           :default => {}
       magic :monitoring,                String
+      magic :mount_ephemerals,          Hash,           :default => {}
       magic :permanent,                 :boolean,       :default => false
-      magic :public_ip,                 String
       magic :placement_group,           String
-      magic :region,                    String,         :default => -> do
-        default_availability_zone.gsub(/^(\w+-\w+-\d)[a-z]/, '\1') if default_availability_zone
-      end
-      collection :security_groups,      Ironfan::Dsl::Ec2::SecurityGroup
-      magic :ssh_user,                  String,         :default => 'ubuntu'
+      magic :provider,                  Ironfan::Provider,      :default => Ironfan::Provider::Ec2
+      magic :public_ip,                 String
+      magic :region,                    String,         :default => ->{ default_region }
+      magic :ssh_user,                  String,         :default => ->{ image_info[:ssh_user] }
       magic :ssh_identity_dir,          String,         :default => ->{ Chef::Config.ec2_key_dir }
-      magic :ssh_identity_file,         String #, :default => "#{keypair}.pem"
+      collection :security_groups,      Ironfan::Dsl::Ec2::SecurityGroup
       magic :subnet,                    String
       magic :validation_key,            String,         :default => ->{ IO.read(Chef::Config.validation_key) rescue '' }
       magic :vpc,                       String
 
-      magic :provider,                  Ironfan::Provider,      :default => Ironfan::Provider::Ec2
-
-      def ec2_image_info
-        keys = [region, flavor_info[:bits], backing, image_name]
-        Chef::Config[:ec2_image_info][ keys ] || {}
+      def image_info
+        bit_str = "#{self.bits.to_i}-bit" # correct for legacy image info.
+        keys = [region, bit_str, backing, image_name]
+        info = Chef::Config[:ec2_image_info][ keys ]
+        ui.warn("Can't find image for #{[region, bits.to_s, backing, image_name].inspect}") if info.blank?
+        return info || {}
       end
 
       def image_id
-        result = read_attribute(:image_id) || ec2_image_info[:image_id]
+        result = read_attribute(:image_id) || image_info[:image_id]
+      end
+
+      def ssh_key_name(computer)
+        keypair ? keypair.to_s : computer.server.cluster_name
+      end
+
+      def default_region
+        default_availability_zone ? default_availability_zone.gsub(/^(\w+-\w+-\d)[a-z]/, '\1') : nil
       end
 
       def to_display(style,values={})
@@ -127,20 +135,22 @@ end
 
 Chef::Config[:ec2_flavor_info] ||= {}
 Chef::Config[:ec2_flavor_info].merge!({
-  't1.micro'    => { :price => 0.02,  :bits => '64-bit', :ram =>    686, :cores => 1, :core_size => 0.25, :inst_disks => 0, :inst_disk_size =>    0, :ephemeral_volumes => 0 },
-  'm1.small'    => { :price => 0.08,  :bits => '64-bit', :ram =>   1740, :cores => 1, :core_size => 1,    :inst_disks => 1, :inst_disk_size =>  160, :ephemeral_volumes => 1 },
-  'm1.medium'   => { :price => 0.165, :bits => '32-bit', :ram =>   3840, :cores => 2, :core_size => 1,    :inst_disks => 1, :inst_disk_size =>  410, :ephemeral_volumes => 1 },
-  'c1.medium'   => { :price => 0.17,  :bits => '32-bit', :ram =>   1740, :cores => 2, :core_size => 2.5,  :inst_disks => 1, :inst_disk_size =>  350, :ephemeral_volumes => 1 },
-  'm1.large'    => { :price => 0.32,  :bits => '64-bit', :ram =>   7680, :cores => 2, :core_size => 2,    :inst_disks => 2, :inst_disk_size =>  850, :ephemeral_volumes => 2 },
-  'm2.xlarge'   => { :price => 0.45,  :bits => '64-bit', :ram =>  18124, :cores => 2, :core_size => 3.25, :inst_disks => 1, :inst_disk_size =>  420, :ephemeral_volumes => 1 },
-  'c1.xlarge'   => { :price => 0.64,  :bits => '64-bit', :ram =>   7168, :cores => 8, :core_size => 2.5,  :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 4 },
-  'm1.xlarge'   => { :price => 0.66,  :bits => '64-bit', :ram =>  15360, :cores => 4, :core_size => 2,    :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 4 },
-  'm2.2xlarge'  => { :price => 0.90,  :bits => '64-bit', :ram =>  35020, :cores => 4, :core_size => 3.25, :inst_disks => 2, :inst_disk_size =>  850, :ephemeral_volumes => 2 },
-  'm2.4xlarge'  => { :price => 1.80,  :bits => '64-bit', :ram =>  70041, :cores => 8, :core_size => 3.25, :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 4 },
-  'cc1.4xlarge' => { :price => 1.30,  :bits => '64-bit', :ram =>  23552, :cores => 8, :core_size => 4.19, :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 2, :placement_groupable => true, :virtualization => 'hvm' },
-  'cc1.8xlarge' => { :price => 2.40,  :bits => '64-bit', :ram =>  61952, :cores =>16, :core_size => 5.50, :inst_disks => 8, :inst_disk_size => 3370, :ephemeral_volumes => 4, :placement_groupable => true, :virtualization => 'hvm' },
-  'cg1.4xlarge' => { :price => 2.10,  :bits => '64-bit', :ram =>  22528, :cores => 8, :core_size => 4.19, :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 2, :placement_groupable => true, :virtualization => 'hvm' },
-})
+    # 32-or-64: m1.small, m1.medium, t1.micro, c1.medium
+    't1.micro'    => { :price => 0.02,  :bits => 64, :ram =>    686, :cores => 1, :core_size => 0.25, :inst_disks => 0, :inst_disk_size =>    0, :ephemeral_volumes => 0 },
+    'm1.small'    => { :price => 0.08,  :bits => 64, :ram =>   1740, :cores => 1, :core_size => 1,    :inst_disks => 1, :inst_disk_size =>  160, :ephemeral_volumes => 1 },
+    'm1.medium'   => { :price => 0.165, :bits => 32, :ram =>   3840, :cores => 2, :core_size => 1,    :inst_disks => 1, :inst_disk_size =>  410, :ephemeral_volumes => 1 },
+    'c1.medium'   => { :price => 0.17,  :bits => 32, :ram =>   1740, :cores => 2, :core_size => 2.5,  :inst_disks => 1, :inst_disk_size =>  350, :ephemeral_volumes => 1 },
+    #
+    'm1.large'    => { :price => 0.32,  :bits => 64, :ram =>   7680, :cores => 2, :core_size => 2,    :inst_disks => 2, :inst_disk_size =>  850, :ephemeral_volumes => 2 },
+    'm2.xlarge'   => { :price => 0.45,  :bits => 64, :ram =>  18124, :cores => 2, :core_size => 3.25, :inst_disks => 1, :inst_disk_size =>  420, :ephemeral_volumes => 1 },
+    'c1.xlarge'   => { :price => 0.64,  :bits => 64, :ram =>   7168, :cores => 8, :core_size => 2.5,  :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 4 },
+    'm1.xlarge'   => { :price => 0.66,  :bits => 64, :ram =>  15360, :cores => 4, :core_size => 2,    :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 4 },
+    'm2.2xlarge'  => { :price => 0.90,  :bits => 64, :ram =>  35020, :cores => 4, :core_size => 3.25, :inst_disks => 2, :inst_disk_size =>  850, :ephemeral_volumes => 2 },
+    'm2.4xlarge'  => { :price => 1.80,  :bits => 64, :ram =>  70041, :cores => 8, :core_size => 3.25, :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 4 },
+    'cc1.4xlarge' => { :price => 1.30,  :bits => 64, :ram =>  23552, :cores => 8, :core_size => 4.19, :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 2, :placement_groupable => true, :virtualization => 'hvm' },
+    'cc1.8xlarge' => { :price => 2.40,  :bits => 64, :ram =>  61952, :cores =>16, :core_size => 5.50, :inst_disks => 8, :inst_disk_size => 3370, :ephemeral_volumes => 4, :placement_groupable => true, :virtualization => 'hvm' },
+    'cg1.4xlarge' => { :price => 2.10,  :bits => 64, :ram =>  22528, :cores => 8, :core_size => 4.19, :inst_disks => 4, :inst_disk_size => 1690, :ephemeral_volumes => 2, :placement_groupable => true, :virtualization => 'hvm' },
+  })
 
 Chef::Config[:ec2_image_info] ||= {}
 Chef::Config[:ec2_image_info].merge!({

data/lib/ironfan/dsl/server.rb

@@ -28,7 +28,17 @@ module Ironfan
         values["Env"] =         environment
         values
       end
+
+      # we should always show up in owners' inspect string
+      def inspect_compact ; inspect ; end
+
+      # @returns [Hash{String, Array}] of 'what you did wrong' => [relevant, info]
+      def lint
+        errors = []
+        errors['missing cluster/facet/server'] = [cluster_name, facet_name, name] unless (cluster_name && facet_name && name)
+        errors
+      end
     end
 
   end
-end
+end

data/lib/ironfan/dsl/volume.rb

@@ -29,8 +29,8 @@ module Ironfan
         :blank_xfs       => 'snap-d9c1edb1',
       })
 
-      def snapshot_id(id)
-        Chef::Log.warn("CODE SMELL: EBS specific information in Dsl::Volume::VOLUME_IDS")
+      def snapshot_id(*)
+        Ironfan.todo("CODE SMELL: EBS specific information in Dsl::Volume::VOLUME_IDS")
         super || VOLUME_IDS[snapshot_name]
       end
     end

data/lib/ironfan/provider/chef/client.rb

@@ -3,11 +3,11 @@ module Ironfan
     class ChefServer
 
       class Client < Ironfan::Provider::Resource
-        delegate :add_to_index, :admin, :cdb_destroy, :cdb_save, 
-            :class_from_file, :couchdb, :couchdb=, :couchdb_id, :couchdb_id=, 
-            :couchdb_rev, :couchdb_rev=, :create, :create_keys, 
-            :delete_from_index, :destroy, :from_file, :index_id, :index_id=, 
-            :index_object_type, :name, :public_key, :save, :set_or_return, 
+        delegate :add_to_index, :admin, :cdb_destroy, :cdb_save,
+            :class_from_file, :couchdb, :couchdb=, :couchdb_id, :couchdb_id=,
+            :couchdb_rev, :couchdb_rev=, :create, :create_keys,
+            :delete_from_index, :destroy, :from_file, :index_id, :index_id=,
+            :index_object_type, :name, :public_key, :save, :set_or_return,
             :to_hash, :validate, :with_indexer_metadata,
           :to => :adaptee
         field :key_filename,    String,
@@ -18,6 +18,11 @@ module Ironfan
           self.adaptee ||= Chef::ApiClient.new
         end
 
+        def to_s
+          "<%-15s %-23s %s>" % [
+            self.class.handle, name, key_filename]
+        end
+
         def self.shared?()              false;  end
         def self.multiple?()            false;  end
         def self.resource_type()        :client;                        end
@@ -40,13 +45,16 @@ module Ironfan
         # Discovery
         #
         def self.load!(cluster=nil)
+          Ironfan.substep(cluster.name, "chef clients")
           nameq = "name:#{cluster.name}-* OR clientname:#{cluster.name}-*"
-          ChefServer.search(:client, nameq) do |client|
-            register client unless client.blank?
+          ChefServer.search(:client, nameq) do |raw|
+            next unless raw.present?
+            client = register(raw)
+            Chef::Log.debug("Loaded #{client}")
           end
         end
 
-        # 
+        #
         # Manipulation
         #
         def self.create!(computer)

data/lib/ironfan/provider/chef/node.rb

@@ -30,6 +30,11 @@ module Ironfan
           self.adaptee ||= Chef::Node.new
         end
 
+        def to_s
+          "<%-15s %-23s %s>" % [
+            self.class.handle, name, run_list]
+        end
+
         def self.shared?()              false;  end
         def self.multiple?()            false;  end
 #         def self.resource_type()        self;   end
@@ -83,11 +88,11 @@ module Ironfan
         # Discovery
         #
         def self.load!(cluster=nil)
+          Ironfan.substep(cluster.name, "nodes")
           ChefServer.search(:node,"name:#{cluster.name}-*") do |raw|
-            next if raw.blank?
-            node = Node.new
-            node.adaptee = raw
-            remember node
+            next unless raw.present?
+            node = register(raw)
+            Chef::Log.debug("Loaded #{node}")
           end
         end
 

data/lib/ironfan/provider/chef/role.rb

@@ -38,13 +38,20 @@ module Ironfan
           self
         end
 
-        # 
+        def to_s
+          "<%-15s %-23s %s>" % [
+            self.class.handle, name, run_list]
+        end
+
+        #
         # Discovery
         #
         def self.load!(cluster)
+          Ironfan.substep(cluster.name, "roles")
           ChefServer.search(:role,"name:#{cluster.name}_*") do |raw|
-            next if raw.blank?
-            remember Role.new(:adaptee => raw)
+            next unless raw.present?
+            role = register(raw)
+            Chef::Log.debug("Loaded #{role}")
           end
         end
 

data/lib/ironfan/provider/chef.rb

@@ -2,7 +2,8 @@ module Ironfan
   class Provider
 
     class ChefServer < Ironfan::Provider
-      
+      self.handle = :chef
+
       def self.resources
         [ Client, Node, Role ]
       end
@@ -25,4 +26,4 @@ module Ironfan
     end
 
   end
-end
+end

data/lib/ironfan/provider/ec2/ebs_volume.rb

@@ -16,6 +16,11 @@ module Ironfan
           :to => :adaptee
         field :dsl_volume,        Ironfan::Dsl::Volume
 
+        def to_s
+          "<%-15s %-12s %-25s %-32s %-10s %-12s %-15s %-5s %s:%s>" % [
+            self.class.handle, id, created_at, tags['name'], state, device, tags['mount_point'], size, server_id, attached_at ]
+        end
+
         def self.shared?()      true;   end
         def self.multiple?()    true;   end
         def self.resource_type()        :ebs_volume;   end
@@ -44,6 +49,7 @@ module Ironfan
         # Discovery
         #
         def self.load!(cluster=nil)
+          Ironfan.substep(cluster.name, "volumes")
           Ec2.connection.volumes.each do |vol|
             next if vol.blank?
             ebs = EbsVolume.new(:adaptee => vol)
@@ -55,6 +61,7 @@ module Ironfan
             else
               remember ebs
             end
+            Chef::Log.debug("Loaded #{ebs}")
           end
         end
 
@@ -100,4 +107,4 @@ module Ironfan
 
     end
   end
-end
+end

data/lib/ironfan/provider/ec2/key_pair.rb

@@ -25,12 +25,18 @@ module Ironfan
           File.open(key_filename, "w", 0600){|f| f.print( body ) }
         end
 
+        def to_s
+          "<%-15s %-12s>" % [self.class.handle, name]
+        end
+
         #
         # Discovery
         #
         def self.load!(cluster=nil)
+          Ironfan.substep(cluster && cluster.name, "keypairs")
           Ec2.connection.key_pairs.each do |keypair|
             register keypair unless keypair.blank?
+            Chef::Log.debug("Loaded <%-15s %s>" % [handle, keypair.name])
           end
         end
 
@@ -62,4 +68,4 @@ module Ironfan
 
     end
   end
-end
+end

data/lib/ironfan/provider/ec2/machine.rb

@@ -93,10 +93,16 @@ module Ironfan
           keypair = cloud.keypair || computer.server.cluster_name
         end
 
+        def to_s
+          "<%-15s %-12s %-25s %-25s %-15s %-15s %-12s %-12s %s:%s>" % [
+            self.class.handle, id, created_at, tags['name'], private_ip_address, public_ip_address, flavor_id, availability_zone, key_name, groups.join(',') ]
+        end
+
         #
         # Discovery
         #
         def self.load!(cluster=nil)
+          Ironfan.substep(cluster.name, "machines")
           Ec2.connection.servers.each do |fs|
             machine = new(:adaptee => fs)
             if recall? machine.name
@@ -109,6 +115,7 @@ module Ironfan
             else
               remember machine, :append_id => "terminated:#{machine.id}"
             end
+            Chef::Log.debug("Loaded #{machine}")
           end
         end
 
@@ -122,11 +129,11 @@ module Ironfan
               machine.bogus << :unexpected_machine
             end
             next unless machine.bogus?
-            fake =                Ironfan::Broker::Computer.new
-            fake[:machine] =      machine
-            fake.name =           machine.name
-            machine.users <<      fake
-            computers <<          fake
+            fake           = Ironfan::Broker::Computer.new
+            fake[:machine] = machine
+            fake.name      = machine.name
+            machine.users << fake
+            computers     << fake
           end
         end
 
@@ -134,11 +141,14 @@ module Ironfan
         # Manipulation
         #
         def self.create!(computer)
-          Chef::Log.warn("CODE SMELL: overly large method: #{caller}")
+          Ironfan.todo("CODE SMELL: overly large method: #{caller}")
           return if computer.machine? and computer.machine.created?
           Ironfan.step(computer.name,"creating cloud machine", :green)
-          # lint_fog
-          launch_desc = fog_launch_description(computer)
+          #
+          errors = lint(computer)
+          if errors.present? then raise ArgumentError, "Failed validation: #{errors.inspect}" ; end
+          #
+          launch_desc = launch_description(computer)
           Chef::Log.debug(JSON.pretty_generate(launch_desc))
 
           Ironfan.safely do
@@ -162,7 +172,7 @@ module Ironfan
 
           # register the new volumes for later save!, and tag appropriately
           computer.machine.volumes.each do |v|
-            Chef::Log.warn "CODE SMELL: Machine is too familiar with EbsVolume problems"
+            Ironfan.todo "CODE SMELL: Machine is too familiar with EbsVolume problems"
             ebs_vol = Ec2::EbsVolume.register v
             drive = computer.drives.values.select do |drive|
               drive.volume.device == ebs_vol.device
@@ -178,30 +188,43 @@ module Ironfan
             Ec2.ensure_tags(tags,ebs_vol)
           end
         end
-        def self.fog_launch_description(computer)
-          cloud =                       computer.server.cloud(:ec2)
+
+        # @returns [Hash{String, Array}] of 'what you did wrong' => [relevant, info]
+        def self.lint(computer)
+          cloud = computer.server.cloud(:ec2)
+          info  = [computer.name, cloud.inspect]
+          errors = {}
+          server_errors = computer.server.lint
+          errors["Unhappy Server"] = server_errors if server_errors.present?
+          errors["No AMI found"] = info if cloud.image_id.blank?
+          errors['Missing client']      = info            unless computer.client?
+          errors['Missing private_key'] = computer.client unless computer.private_key
+          errors
+        end
+
+        def self.launch_description(computer)
+          cloud = computer.server.cloud(:ec2)
           user_data_hsh =               {
             :chef_server =>             Chef::Config[:chef_server_url],
-            #:validation_client_name =>  Chef::Config[:validation_client_name],
+            # :validation_client_name => Chef::Config[:validation_client_name],
             #
             :node_name =>               computer.name,
             :organization =>            Chef::Config[:organization],
             :cluster_name =>            computer.server.cluster_name,
             :facet_name =>              computer.server.facet_name,
             :facet_index =>             computer.server.index,
-            :client_key =>              computer[:client].private_key
+            :client_key =>              computer.private_key
           }
 
           # Fog does not actually create tags when it creates a server;
           #  they and permanence are applied during sync
-          keypair = cloud.keypair || computer.server.cluster_name
           description = {
             :image_id             => cloud.image_id,
             :flavor_id            => cloud.flavor,
             :vpc_id               => cloud.vpc,
             :subnet_id            => cloud.subnet,
             :groups               => cloud.security_groups.keys,
-            :key_name             => keypair.to_s,
+            :key_name             => cloud.ssh_key_name(computer),
             :user_data            => JSON.pretty_generate(user_data_hsh),
             :block_device_mapping => block_device_mapping(computer),
             :availability_zone    => cloud.default_availability_zone,
@@ -214,9 +237,10 @@ module Ironfan
           end
           description
         end
+
         # An array of hashes with dorky-looking keys, just like Fog wants it.
         def self.block_device_mapping(computer)
-          Chef::Log.warn "CODE SMELL: Machine is too familiar with EbsVolume problems"
+          Ironfan.todo "CODE SMELL: Machine is too familiar with EbsVolume problems"
           computer.drives.values.map do |drive|
             next if drive.disk  # Don't create any disc already satisfied
             volume = drive.volume or next
@@ -263,4 +287,3 @@ module Ironfan
     end
   end
 end
-

data/lib/ironfan/provider/ec2/placement_group.rb

@@ -11,14 +11,20 @@ module Ironfan
           self["groupName"]
         end
 
+        def to_s
+          "<%-15s %-12s %-12s>" % [ self.class.handle, '', name ]
+        end
+
         def self.load!(cluster)
+          Ironfan.substep(cluster.name, "placement groups")
           result = Ec2.connection.describe_placement_groups
           result.body["placementGroupSet"].each do |group|
             register group unless group.blank?
+            Chef::Log.debug("Loaded #{group.inspect}")
           end
         end
       end
 
     end
   end
-end
+end

data/lib/ironfan/provider/ec2/security_group.rb

@@ -24,8 +24,28 @@ module Ironfan
         # Discovery
         #
         def self.load!(cluster=nil)
-          Ec2.connection.security_groups.each do |sg|
-            remember SecurityGroup.new(:adaptee => sg) unless sg.blank?
+          Ironfan.substep(cluster.name, "security groups")
+
+          Ec2.connection.security_groups.each do |raw|
+            next if raw.blank?
+            sg = SecurityGroup.new(:adaptee => raw)
+            remember(sg)
+            Chef::Log.debug("Loaded #{sg}")
+          end
+        end
+
+        def to_s
+          if ip_permissions.present?
+            perm_str = ip_permissions.map{|perm|
+              "%s:%s-%s (%s | %s)" % [
+                perm['ipProtocol'], perm['fromPort'], perm['toPort'],
+                perm['groups'  ].map{|el| el['groupName'] }.join(','),
+                perm['ipRanges'].map{|el| el['cidrIp']    }.join(','),
+              ]
+            }
+            return "<%-15s %-12s %-25s %s>" % [ self.class.handle, group_id, name, perm_str]
+          else
+            return "<%-15s %-12s %s>" % [ self.class.handle, group_id, name ]
           end
         end
 
@@ -35,7 +55,7 @@ module Ironfan
 
         def self.create!(computer)
           return unless Ec2.applicable computer
- 
+
           ensure_groups(computer)
           groups = self.expected_ids(computer)
           # Only handle groups that don't already exist
@@ -95,7 +115,7 @@ module Ironfan
           # Ensure the security_groups include those for cluster & facet
           # FIXME: This violates the DSL's immutability; it should be
           #   something calculated from within the DSL construction
-          Chef::Log.warn("CODE SMELL: violation of DSL immutability: #{caller}")
+          Ironfan.todo("CODE SMELL: violation of DSL immutability: #{caller}")
           cloud = computer.server.cloud(:ec2)
           c_group = cloud.security_group(computer.server.cluster_name)
           c_group.authorized_by_group(c_group.name)
@@ -123,4 +143,4 @@ module Ironfan
 
     end
   end
-end
+end

data/lib/ironfan/provider/ec2.rb

@@ -2,6 +2,7 @@ module Ironfan
   class Provider
 
     class Ec2 < Ironfan::IaasProvider
+      self.handle = :ec2
 
       def self.resources
         [ Machine, EbsVolume, KeyPair, SecurityGroup ]
@@ -18,7 +19,7 @@ module Ironfan
           :region                => Chef::Config[:knife][:region]
         })
       end
-      
+
       def self.aws_account_id()
         Chef::Config[:knife][:aws_account_id]
       end
@@ -44,4 +45,4 @@ module Ironfan
     end
 
   end
-end
+end

data/lib/ironfan/provider/virtualbox.rb

@@ -2,7 +2,8 @@ module Ironfan
   class Provider
 
     class VirtualBox < Ironfan::IaasProvider
+      self.handle = :virtualbox
     end
 
   end
-end
+end

data/lib/ironfan/provider.rb

@@ -1,9 +1,10 @@
 # Providers present a lightweight wrapper for various third-party services,
 #   such as Chef's node and client APIs, and Amazon's EC2 APIs. This allows
-#   Ironfan ask specialized questions (such as whether a given resource 
+#   Ironfan ask specialized questions (such as whether a given resource
 #   matches
 module Ironfan
   class Provider < Builder
+    class_attribute :handle
 
     def self.receive(obj,&block)
       obj[:_type] = case obj[:name]
@@ -42,9 +43,11 @@ module Ironfan
 
       def bogus?()                      !bogus.empty?;          end
 
+      def self.handle ; name.to_s.gsub(/.*::/,'').to_sym ; end
+
       #
       # Flags
-      # 
+      #
       # Non-shared resources live and die with the computer
       def self.shared?()                true;                   end
       # Can multiple instances of this resource be associated with the computer?
@@ -82,8 +85,9 @@ module Ironfan
 
       # Register and return the (adapted) object with the collection
       def self.register(native)
-        result = new(:adaptee => native)
-        remember result unless result.nil?
+        result = new(:adaptee => native) or return
+        remember result
+        result
       end
 
       def self.recall?(id)
@@ -135,4 +139,4 @@ module Ironfan
     end
   end
 
-end
+end

data/lib/ironfan.rb

@@ -25,12 +25,18 @@ module Ironfan
   def self.chef_config()    @chef_config      ; end
 
   # execute against multiple targets in parallel
-  def self.parallel(targets,options={})
+  def self.parallel(targets)
     raise 'missing block' unless block_given?
-    [targets].flatten.map do |target|
+    results = []
+    [targets].flatten.each_with_index.map do |target, idx|
       sleep(0.1) # avoid hammering with simultaneous requests
-      Thread.new(target) {|target| yield target }
+      Thread.new(target) do |target|
+        results[idx] = safely(target.inspect) do
+          yield target
+        end
+      end
     end.each(&:join) # wait for all the blocks to return
+    results
   end
 
   #
@@ -120,13 +126,15 @@ module Ironfan
   #     Ironfan.fog_connection.associate_address(self.fog_server.id, address)
   #   end
   #
-  def self.safely
+  def self.safely(info="")
     begin
       yield
-    rescue StandardError => boom
-      ui.info( boom )
-      Chef::Log.error( boom )
-      Chef::Log.error( boom.backtrace.join("\n") )
+    rescue StandardError => err
+      ui.warn("Error running #{info}:")
+      ui.warn(err)
+      Chef::Log.error( err )
+      Chef::Log.error( err.backtrace.join("\n") )
+      return err
     end
   end
 
@@ -137,6 +145,15 @@ module Ironfan
     ui.info("  #{"%-15s" % (name.to_s+":")}\t#{ui.color(desc.to_s, *style)}")
   end
 
+  def self.substep(name, desc)
+    step(name, "  - #{desc}", :gray) if chef_config[:verbosity] >= 1
+  end
+
+  # Output a TODO to the logs if you've switched on pestering
+  def self.todo(*args)
+    Chef::Log.debug(*args) if Chef::Config[:show_todo]
+  end
+
   #
   # Utility to do mock out a step during a dry-run
   #
@@ -152,8 +169,8 @@ module Ironfan
     chef_config[:dry_run]
   end
 
-  # Intentionally skipping an implied step 
+  # Intentionally skipping an implied step
   def self.noop(source,method,*params)
-    Chef::Log.debug("Nothing to do for #{source}.#{method}(#{params.join(',')}), skipping")
+    # Chef::Log.debug("#{method} is a no-op for #{source} -- skipping (#{params.join(',')})")
   end
 end

metadata

@@ -2,7 +2,7 @@
 name: ironfan
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 4.0.9
+  version: 4.1.0
 platform: ruby
 authors: 
 - Infochimps
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-09-20 00:00:00 Z
+date: 2012-09-24 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: chef
@@ -232,7 +232,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 2386087126154656208
+      hash: 3712152495684054514
       segments: 
       - 0
       version: "0"