ironfan 4.8.7 → 4.9.0

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# v4.9.0
+* initial vSphere support (thanks @brandonbell) - see lib/ironfan/dsl/vsphere and lib/ironfan/provider/vsphere for more info
+
 # v4.8.7
 * ec2/machine: fixing stop and start to wait for all valid end-states
 

data/Gemfile.lock

@@ -27,16 +27,16 @@ GEM
     configliere (0.4.18)
       highline (>= 1.5.2)
       multi_json (>= 1.1)
-    diff-lcs (1.2.1)
+    diff-lcs (1.2.2)
     erubis (2.7.0)
-    excon (0.19.3)
-    fog (1.9.0)
+    excon (0.20.1)
+    fog (1.10.1)
       builder
-      excon (~> 0.14)
+      excon (~> 0.20)
       formatador (~> 0.2.0)
       mime-types
       multi_json (~> 1.0)
-      net-scp (~> 1.0.4)
+      net-scp (~> 1.1)
       net-ssh (>= 2.1.3)
       nokogiri (~> 1.5.0)
       ruby-hmac
@@ -46,19 +46,19 @@ GEM
       configliere (>= 0.4.13)
       json
       multi_json (>= 1.1)
-    guard (1.6.2)
+    guard (1.7.0)
+      formatador (>= 0.2.4)
       listen (>= 0.6.0)
       lumberjack (>= 1.0.2)
       pry (>= 0.9.10)
-      terminal-table (>= 1.4.3)
       thor (>= 0.14.6)
-    guard-rspec (2.4.1)
+    guard-rspec (2.5.2)
       guard (>= 1.1)
       rspec (~> 2.11)
-    guard-yard (2.0.1)
+    guard-yard (2.1.0)
       guard (>= 1.1.0)
       yard (>= 0.7.0)
-    highline (1.6.15)
+    highline (1.6.16)
     ipaddress (0.8.0)
     jeweler (1.8.4)
       bundler (~> 1.0)
@@ -69,26 +69,26 @@ GEM
     linecache19 (0.5.12)
       ruby_core_source (>= 0.1.4)
     listen (0.7.3)
-    lumberjack (1.0.2)
+    lumberjack (1.0.3)
     method_source (0.8.1)
-    mime-types (1.21)
+    mime-types (1.22)
     mixlib-authentication (1.3.0)
       mixlib-log
     mixlib-cli (1.3.0)
     mixlib-config (1.1.2)
-    mixlib-log (1.4.1)
+    mixlib-log (1.6.0)
     mixlib-shellout (1.1.0)
     moneta (0.6.0)
-    multi_json (1.6.1)
-    net-scp (1.0.4)
-      net-ssh (>= 1.99.1)
-    net-ssh (2.6.5)
+    multi_json (1.7.2)
+    net-scp (1.1.0)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.6.6)
     net-ssh-gateway (1.2.0)
       net-ssh (>= 2.6.5)
     net-ssh-multi (1.1)
       net-ssh (>= 2.1.4)
       net-ssh-gateway (>= 0.99.0)
-    nokogiri (1.5.6)
+    nokogiri (1.5.9)
     ohai (6.16.0)
       ipaddress
       mixlib-cli
@@ -97,14 +97,14 @@ GEM
       mixlib-shellout
       systemu
       yajl-ruby
-    oj (2.0.7)
+    oj (2.0.10)
     polyglot (0.3.3)
     pry (0.9.12)
       coderay (~> 1.0.5)
       method_source (~> 0.8)
       slop (~> 3.4)
-    rake (10.0.3)
-    rdoc (4.0.0)
+    rake (10.0.4)
+    rdoc (4.0.1)
       json (~> 1.4)
     redcarpet (2.2.2)
     rest-client (1.6.7)
@@ -113,10 +113,10 @@ GEM
       rspec-core (~> 2.13.0)
       rspec-expectations (~> 2.13.0)
       rspec-mocks (~> 2.13.0)
-    rspec-core (2.13.0)
+    rspec-core (2.13.1)
     rspec-expectations (2.13.0)
       diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.13.0)
+    rspec-mocks (2.13.1)
     ruby-debug-base19 (0.11.25)
       columnize (>= 0.3.1)
       linecache19 (>= 0.5.11)
@@ -133,10 +133,9 @@ GEM
       multi_json (~> 1.0)
       simplecov-html (~> 0.7.1)
     simplecov-html (0.7.1)
-    slop (3.4.3)
+    slop (3.4.4)
     systemu (2.5.2)
-    terminal-table (1.4.5)
-    thor (0.17.0)
+    thor (0.18.1)
     treetop (1.4.12)
       polyglot
       polyglot (>= 0.3.1)

data/VERSION

@@ -1 +1 @@
-4.8.7
+4.9.0

data/config/ubuntu12.04-ironfan.erb

@@ -148,4 +148,4 @@ rm -r /tmp/knife-bootstrap
 updatedb
 
 echo -e "`date` \n\n**** \n**** Cluster Chef client bootstrap complete\n****\n"
-EOF
+EOF

data/ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.8.7"
+  s.version = "4.9.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2013-03-05"
+  s.date = "2013-04-09"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [
@@ -73,6 +73,7 @@ Gem::Specification.new do |s|
     "lib/ironfan/dsl/server.rb",
     "lib/ironfan/dsl/virtualbox.rb",
     "lib/ironfan/dsl/volume.rb",
+    "lib/ironfan/dsl/vsphere.rb",
     "lib/ironfan/headers.rb",
     "lib/ironfan/provider.rb",
     "lib/ironfan/provider/chef.rb",
@@ -90,6 +91,9 @@ Gem::Specification.new do |s|
     "lib/ironfan/provider/ec2/security_group.rb",
     "lib/ironfan/provider/virtualbox.rb",
     "lib/ironfan/provider/virtualbox/machine.rb",
+    "lib/ironfan/provider/vsphere.rb",
+    "lib/ironfan/provider/vsphere/keypair.rb",
+    "lib/ironfan/provider/vsphere/machine.rb",
     "lib/ironfan/requirements.rb",
     "notes/Future-development-proposals.md",
     "notes/Home.md",
@@ -149,7 +153,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://infochimps.com/labs"
   s.licenses = ["apachev2"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.24"
+  s.rubygems_version = "1.8.23"
   s.summary = "Infochimps' lightweight cloud orchestration toolkit, built on top of Chef."
   s.test_files = ["spec/spec_helper/dummy_chef.rb", "spec/integration/spec_helper/launch_cluster.rb", "spec/integration/minimal-chef-repo/roles/systemwide.rb", "spec/integration/minimal-chef-repo/environments/_default.json", "spec/integration/minimal-chef-repo/chefignore", "spec/integration/minimal-chef-repo/knife/knife.rb", "spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb", "spec/integration/spec/simple_cluster_spec.rb", "spec/integration/spec/elb_build_spec.rb", "spec/integration/spec_helper.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/elb_spec.rb", "spec/ironfan/ec2/security_group_spec.rb", "spec/chef/cluster_launch_spec.rb", "spec/chef/cluster_bootstrap_spec.rb", "spec/fixtures/gunbai_slice.json", "spec/fixtures/ec2/elb/snakeoil.key", "spec/fixtures/ec2/elb/snakeoil.crt", "spec/fixtures/gunbai.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
 

data/lib/chef/knife/bootstrap/ubuntu12.04-ironfan.erb

@@ -148,4 +148,4 @@ rm -r /tmp/knife-bootstrap
 updatedb
 
 echo -e "`date` \n\n**** \n**** Cluster Chef client bootstrap complete\n****\n"
-EOF
+EOF

data/lib/chef/knife/ironfan_knife_common.rb

@@ -9,6 +9,7 @@ module Ironfan
       require 'chef/node'
       require 'chef/api_client'
       require 'fog'
+      require 'rbvmomi'
     end
 
     def load_ironfan

data/lib/ironfan/dsl/cloud.rb

@@ -12,6 +12,7 @@ module Ironfan
             case obj[:name]
             when :ec2        then Ec2
             when :virtualbox then VirtualBox
+            when :vsphere    then Vsphere
             else raise "Unsupported cloud #{obj[:name]}"
             end
         end

data/lib/ironfan/dsl/vsphere.rb

@@ -0,0 +1,89 @@
+module Ironfan
+  class Dsl
+
+    class Compute < Ironfan::Dsl
+      def vsphere(*attrs,&block)
+        cloud(:vsphere, *attrs,&block)
+      end
+    end
+
+    class Vsphere < Cloud
+      magic :backing,                   String,         :default => ''
+      magic :bits,                      Integer,        :default => "64"
+      magic :bootstrap_distro,          String,         :default => 'ubuntu12.04-gems'
+      magic :chef_client_script,        String
+      magic :cluster,                   String
+      magic :cpus,                      String,         :default => "1" 
+      magic :datacenter,                String,         :default => ->{ default_datacenter }
+      magic :datastore,                 String
+      magic :dns_servers,		Array
+      magic :domain,                    String 
+      magic :default_datacenter,        String,         :default => ->{ vsphere_datacenters.first }
+      magic :gateway, 			Array
+      magic :image_name,                String
+      magic :ip, 	                String
+      magic :memory,                    String,         :default => "4" # Gigabytes
+      magic :provider,                  Whatever,       :default => Ironfan::Provider::Vsphere
+      magic :ssh_identity_dir,          String,         :default => ->{ Chef::Config.vsphere_key_dir }
+      magic :ssh_user,                  String,         :default => "root"
+      magic :subnet,			String
+      magic :template,                  String
+      magic :validation_key,            String,         :default => ->{ IO.read(Chef::Config.validation_key) rescue '' }
+      magic :virtual_disks,             Array,          :default => []
+      magic :vsphere_datacenters,       Array,          :default => ['New Datacenter']
+      magic :network, 			String,		:default => "VM Network"
+
+      def image_info
+        bit_str = "#{self.bits.to_i}-bit" # correct for legacy image info.
+        keys = [datacenter, bit_str, image_name]
+#        info = Chef::Config[:vsphere_image_info][ keys ]
+        info = nil
+        ui.warn("Can't find image for #{[datacenter, bit_str, image_name].inspect}") if info.blank?
+        return info || {}
+      end
+
+      def implied_volumes
+        result = []
+        # FIXME : This is really making assumptions
+        result << Ironfan::Dsl::Volume.new(:name => 'root') do
+            device              '/dev/sda1'
+            fstype              'ext4'
+            keep                false
+            mount_point         '/'
+        end
+        return result unless virtual_disks.length > 0
+
+        virtual_disks.each_with_index do |vd, idx|
+          dev, mnt = ["/dev/sd%s" %[(66 + idx).chr.downcase], idx == 0 ? "/mnt" : "/mnt#{idx}"] # WHAAAaaa 0 o ??? 
+          virtualdisk = Ironfan::Dsl::Volume.new(:name => "virtualdisk#{idx}") do
+            attachable          "VirtualDisk"
+            fstype              vd[:fs]
+            device              dev
+            mount_point         vd[:mount_point] || mnt
+            formattable         true
+            create_at_launch    true
+            mount_options       'defaults,noatime'
+            tags                vd[:tags] 
+          end
+          result << virtualdisk
+        end
+        result
+      end
+
+      def ssh_identity_file(computer)
+        cluster = computer.server.cluster_name
+        "%s/%s.pem" %[ssh_identity_dir, cluster]
+      end
+
+      def to_display(style,values={})
+        return values if style == :minimal
+
+        values["Datacenter"] =         datacenter
+        return values if style == :default
+
+        values
+      end
+
+    end
+  end
+end

data/lib/ironfan/headers.rb

@@ -30,6 +30,7 @@ module Ironfan
       class IamServerCertificate < Ironfan::Dsl; end
     end
     class VirtualBox < Cloud; end
+    class Vsphere < Cloud; end
   end
 
   class Provider < Builder
@@ -57,6 +58,11 @@ module Ironfan
     class VirtualBox < Ironfan::IaasProvider
       class Machine < Ironfan::IaasProvider::Machine; end
     end
+    class Vsphere < Ironfan::IaasProvider
+      class Machine < Ironfan::IaasProvider::Machine; end
+      class Keypair < Ironfan::Provider::Resource; end
+    end
+
   end
 
 end

data/lib/ironfan/provider/vsphere/keypair.rb

@@ -0,0 +1,74 @@
+module Ironfan
+  class Provider
+    class Vsphere
+
+      class Keypair < Ironfan::Provider::Resource
+        delegate :connection, :connection=, :name, :private_key,
+          :to => :adaptee
+#        delegate :_dump, :collection, :collection=, :connection,
+ #           :connection=, :destroy, :fingerprint, :fingerprint=, :identity,
+ #           :identity=, :name, :name=, :new_record?, :public_key,
+#            :public_key=, :reload, :requires, :requires_one, :save,
+#            :symbolize_keys, :wait_for, :writable?, :write,
+#          :to => :adaptee
+
+        field :key_filename,    String, :default => ->{ "#{Keypair.key_dir}/#{name}.pem" }
+
+        def self.shared?       ; true     ; end
+        def self.multiple?     ; false    ; end
+        def self.resource_type ; :keypair ; end
+
+        def self.expected_ids(computer)
+          [computer.server.cluster_name]
+        end
+
+        def private_key
+          puts "pkey"
+          File.open(key_filename, "rb").read
+        end
+
+        def self.public_key(computer)
+          key_filename = "%s/%s.pem" %[key_dir, computer.server.cluster_name]
+          key = OpenSSL::PKey::RSA.new(File.open(key_filename, "rb").read)
+          data = [ key.to_blob ].pack('m0')
+          "#{key.ssh_type} #{data}"
+        end
+ 
+        def self.create_private_key(key, name)
+          key_filename = "%s/%s.pem" %[key_dir, name]
+          File.open(key_filename, "w", 0600){|f| f.print( key.to_s ) }
+        end
+
+        def to_s
+          "<%-15s %-12s>" % [self.class.handle, name]
+        end
+
+        #
+        # Manipulation
+        #
+
+        def self.prepare!(computers)
+          return if computers.empty?
+          name = computers.values[0].server.cluster_name
+          return if recall? name
+          return if File.exists?("%s/%s.pem" %[key_dir, name])
+          Ironfan.step(name, "creating key pair for #{name}", :blue)
+          Dir.mkdir(key_dir) if !FileTest::directory?(key_dir)
+          create_private_key(OpenSSL::PKey::RSA.new(2048), name)
+        end
+
+        #
+        # Utility
+        #
+
+        def self.key_dir
+          return Chef::Config.vsphere_key_dir if Chef::Config.vsphere_key_dir
+          dir = "#{ENV['HOME']}/.chef/credentials/vshere_keys"
+          warn "Please set 'vsphere_key_dir' in your knife.rb. Will use #{dir} as a default"
+          dir
+        end
+
+      end
+    end
+  end
+end

data/lib/ironfan/provider/vsphere/machine.rb

@@ -0,0 +1,343 @@
+module Ironfan
+  class Provider
+    class Vsphere
+
+      class Machine < Ironfan::IaasProvider::Machine
+        delegate :config, :connection, :connection=, :disks, :Destroy_Task, :guest, :PowerOffVM_Task, 
+          :PowerOnVM_Task, :powerState, :ReconfigVM_Task, :runtime, 
+          :to => :adaptee
+
+        def self.shared?()      false;  end
+        def self.multiple?()    false;  end
+        def self.resource_type()        :machine;   end
+        def self.expected_ids(computer) [computer.server.full_name];   end 
+        
+        def name
+           return adaptee.config.name
+        end
+
+        def keypair
+        end
+
+        def vpc_id
+          return true
+        end
+
+        def dns_name
+          host = adaptee.guest.hostName 
+          domain = adaptee.guest.domainName
+          return host unless domain
+          return "%s.%s" %[host, domain]
+        end
+
+        def public_ip_address
+          # FIXME
+          return adaptee.guest.ipAddress
+        end
+
+        def private_ip_address
+          return adaptee.guest.ipAddress
+        end
+ 
+        def public_hostname
+          # FIXME
+          return nil
+        end
+
+        def destroy
+          adaptee.PowerOffVM_Task.wait_for_completion unless adaptee.runtime.powerState == "poweredOff"
+          adaptee.Destroy_Task.wait_for_completion
+          state = "destroyed"
+        end
+
+        def created?
+          ["poweredOn", "poweredOff"].include? adaptee.runtime.powerState
+        end
+
+        def wait_for
+          return true
+        end
+
+        def running?
+          adaptee.runtime.powerState == "poweredOn"
+          state = "poweredOn"
+        end
+
+        def stopped?
+          adaptee.runtime.powerState == "poweredOff"
+          state = "poweredOff"
+        end
+
+        def start
+          adaptee.PowerOnVM_Task.wait_for_completion
+        end
+
+        def stop
+          # TODO: Gracefully shutdown guest using adatee.ShutdownGuest ? 
+          # There is no "wait_for_completion" with this however...
+          adaptee.PowerOffVM_Task.wait_for_completion
+        end
+
+        def self.ip_settings(settings, hostname)
+          # FIXME: A lot of this is required if using a static IP
+          # Heavily referenced Serengeti's FOG here 
+
+          ip_settings = RbVmomi::VIM::CustomizationIPSettings.new(:ip => 
+            RbVmomi::VIM::CustomizationFixedIp(:ipAddress => settings[:ip]), :gateway => settings[:gateway], :subnetMask => settings[:subnet]) if settings[:ip]
+          ip_settings ||= RbVmomi::VIM::CustomizationIPSettings.new("ip" => RbVmomi::VIM::CustomizationDhcpIpGenerator.new())
+          ip_settings.dnsDomain = settings[:domain]
+          global_ip_settings = RbVmomi::VIM.CustomizationGlobalIPSettings
+          global_ip_settings.dnsServerList = settings[:dnsServers]
+          global_ip_settings.dnsSuffixList = [settings[:domain]]
+          hostname = RbVmomi::VIM::CustomizationFixedName.new(:name => hostname)
+          linux_prep = RbVmomi::VIM::CustomizationLinuxPrep.new( :domain => settings[:domain], :hostName => hostname )
+          adapter_mapping = [RbVmomi::VIM::CustomizationAdapterMapping.new("adapter" => ip_settings)]
+          spec = RbVmomi::VIM::CustomizationSpec.new( :identity => linux_prep,
+                                                      :globalIPSettings => global_ip_settings,
+                                                      :nicSettingMap => adapter_mapping )
+
+          return spec
+
+        end
+
+        def self.generate_clone_spec(src_config, dc, cpus, memory, datastore, virtual_disks, network, cluster)
+          # TODO : A lot of this should be moved to utilities in providers/vsphere.rb
+          rspec = Vsphere.get_rspec(dc, cluster)
+          rspec.datastore = datastore
+
+          clone_spec = RbVmomi::VIM.VirtualMachineCloneSpec(:location => rspec, :template => false, :powerOn => true)
+          clone_spec.config = RbVmomi::VIM.VirtualMachineConfigSpec(:deviceChange => Array.new, :extraConfig => nil)
+
+          network = Vsphere.find_network(network, dc)
+          card = src_config.hardware.device.find { |d| d.deviceInfo.label == "Network adapter 1" }
+          begin
+            switch_port = RbVmomi::VIM.DistributedVirtualSwitchPortConnection(
+              :switchUuid => network.config.distributedVirtualSwitch.uuid,
+              :portgroupKey => network.key
+            )
+            card.backing.port = switch_port
+          rescue
+            card.backing.deviceName = network.name
+          end
+
+ 	  network_spec = RbVmomi::VIM.VirtualDeviceConfigSpec(:device => card, :operation => "edit")
+          clone_spec.config.deviceChange.push network_spec
+
+          virtual_disks.each_with_index do |vd, idx|
+            size = vd[:size].to_i
+            label = vd[:label].to_i
+            key = 2001 + idx # key 2000 -> SCSI0, 2001 -> SCSI1...
+            filename = vd[:datastore] || datastore.name
+
+            disk = RbVmomi::VIM.VirtualDisk(
+              :key => key,
+              :capacityInKB => size * 1024 * 1024,
+              :controllerKey => 1000, # SCSI controller
+              :unitNumber => idx + 1,
+              :backing => RbVmomi::VIM.VirtualDiskFlatVer2BackingInfo(
+                :fileName => "[#{filename}]",
+                :diskMode => :persistent,
+                :thinProvisioned => true,
+                :datastore => Vsphere.find_ds(dc, vd[:datastore]) || datastore
+              ),
+              :deviceInfo => RbVmomi::VIM.Description(
+                :label => label, 
+                :summary => "%sGB" %[size]
+              ),
+            )  
+
+            disk_spec = {:operation => :add, :fileOperation => :create, :device => disk }
+            clone_spec.config.deviceChange.push disk_spec
+          end
+          
+          clone_spec.config.numCPUs  = Integer(cpus)
+          clone_spec.config.memoryMB = Integer(memory) * 1024
+ 
+          clone_spec
+        end
+
+        def self.create!(computer)
+          return if computer.machine? and computer.machine.created?
+          Ironfan.step(computer.name,"creating cloud machine", :green)
+
+          errors = lint(computer)
+          if errors.present? then raise ArgumentError, "Failed validation: #{errors.inspect}" ; end
+
+          # TODO: Pass launch_desc to a single function and let it do the rest... like fog does
+          launch_desc = launch_description(computer)
+          cpus          = launch_desc[:cpus]
+          datacenter    = launch_desc[:datacenter]
+          memory        = launch_desc[:memory]
+          template      = launch_desc[:template]
+          user_data     = launch_desc[:user_data]
+          virtual_disks = launch_desc[:virtual_disks]
+          network       = launch_desc[:network]
+          ip_settings   = launch_desc[:ip_settings]
+
+          datacenter    = Vsphere.find_dc(launch_desc[:datacenter])
+          raise "Couldn't find #{launch_desc[:datacenter]} datacenter" unless datacenter
+          cluster = Vsphere.find_in_folder(datacenter.hostFolder, RbVmomi::VIM::ClusterComputeResource, launch_desc[:cluster])
+          raise "Couldn't find #{launch_desc[:cluster]} cluster in #{datacenter}" unless cluster
+          datastore     = Vsphere.find_ds(datacenter, launch_desc[:datastore]) # FIXME: add in round robin choosing?
+          raise "Couldn't find #{launch_desc[:datastore]} datastore in #{datacenter}" unless datastore
+          src_folder    = datacenter.vmFolder
+          src_vm        = Vsphere.find_in_folder(src_folder, RbVmomi::VIM::VirtualMachine, template)
+          raise "Couldn't find template #{template} in #{datacenter}:#{cluster}" unless src_vm
+
+          Ironfan.safely do
+            clone_spec = generate_clone_spec(src_vm.config, datacenter, cpus, memory, datastore, virtual_disks, network, cluster)
+            clone_spec.customization = ip_settings(ip_settings, computer.name) if ip_settings[:ip]
+
+            vsphere_server = src_vm.CloneVM_Task(:folder => src_vm.parent, :name => computer.name, :spec => clone_spec)
+
+            state = vsphere_server.info.state # RbVmomi wait_for_completion stops world... 
+            while (state != 'error') and (state != 'success')
+              sleep 2
+              state = vsphere_server.info.state
+            end
+
+            # Will break if the VM isn't finished building.  
+            new_vm = Vsphere.find_in_folder(src_folder, RbVmomi::VIM::VirtualMachine, computer.name)
+            machine = Machine.new(:adaptee => new_vm)
+            computer.machine = machine
+            remember machine, :id => machine.name
+
+            Ironfan.step(computer.name,"pushing keypair", :green)
+            public_key = Vsphere::Keypair.public_key(computer)
+            # TODO - This probably belongs somewhere else.  This is how we'll enject the pubkey as well as any user information we may want
+            # This data is not persistent across reboots...
+            extraConfig = [{:key => 'guestinfo.pubkey', :value => public_key}, {:key => "guestinfo.user_data", :value => user_data}]
+            machine.ReconfigVM_Task(:spec => RbVmomi::VIM::VirtualMachineConfigSpec(:extraConfig => extraConfig)).wait_for_completion
+
+            # FIXME: This is extremelty fragile right now
+            if ip_settings[:ip]
+              Ironfan.step(computer.name,"waiting for ip customizations to complete", :green)
+              while machine.guest.ipAddress != ip_settings[:ip]
+                sleep 2
+              end
+            end
+
+          end
+        end
+
+        #
+        # Discovery
+        #
+        def self.load!(cluster=nil)
+          # FIXME:  Fix this one day when we have multiple "datacenters".
+          cloud = cluster.servers.values[0].cloud(:vsphere)
+          vm_folder = Vsphere.find_dc(cloud.datacenter).vmFolder
+          Vsphere.find_all_in_folder(vm_folder, RbVmomi::VIM::VirtualMachine).each do |fs|
+            machine = new(:adaptee => fs)
+            if recall? machine.name
+              machine.bogus <<                 :duplicate_machines
+              recall(machine.name).bogus <<    :duplicate_machines
+              remember machine, :append_id => "duplicate:#{machine.config.uuid}"
+            else # never seen it
+              remember machine
+            end
+            Chef::Log.debug("Loaded #{machine}")
+          end
+        end
+
+        def self.destroy!(computer)
+          return unless computer.machine?
+          forget computer.machine.name
+          computer.machine.destroy
+        end
+
+        def self.launch_description(computer)
+          cloud = computer.server.cloud(:vsphere)
+          user_data_hsh =               {
+            :chef_server 	=>      Chef::Config[:chef_server_url],
+            :client_key 	=>      computer.private_key,
+            :cluster_name 	=>      computer.server.cluster_name,
+            :facet_index 	=>      computer.server.index,
+            :facet_name 	=>      computer.server.facet_name,
+            :node_name	 	=>	computer.name,
+            :organization 	=>	Chef::Config[:organization],
+          }
+
+          ip_settings = {
+	    :dnsServers 	=> 	cloud.dns_servers,
+	    :domain 		=>	cloud.domain,
+            :gateway            =>      cloud.gateway,
+            :ip 		=>	cloud.ip,
+            :subnet		=> 	cloud.subnet,
+          }
+
+          description = {
+            :cpus		=> 	cloud.cpus,
+            :cluster		=>	cloud.cluster,
+            :datacenter		=>	cloud.datacenter,
+            :datastore		=>	cloud.datastore, 
+            :memory		=>	cloud.memory,
+            :network		=>	cloud.network, 
+            :template		=>	cloud.template,
+            :user_data		=>	JSON.pretty_generate(user_data_hsh),
+            :virtual_disks	=>	cloud.virtual_disks,
+            :ip_settings	=>	ip_settings
+          }
+          description
+        end
+
+        # @returns [Hash{String, Array}] of 'what you did wrong' => [relevant, info]
+        def self.lint(computer)
+          cloud = computer.server.cloud(:vsphere)
+          info  = [computer.name, cloud.inspect]
+          errors = {}
+          server_errors = computer.server.lint
+          errors["Unhappy Server"] = server_errors if server_errors.present?
+          errors["Datacenter"] = info if cloud.datacenter.blank?
+          errors["Template"] = info if cloud.template.blank?
+          errors["Datastore"] = info if cloud.datastore.blank?
+          errors['Missing client']      = info            unless computer.client?
+          errors['Missing private_key'] = computer.client unless computer.private_key
+          #
+          errors
+        end
+
+        def to_display(style,values={})
+          # style == :minimal
+          values["State"] =            runtime.powerState rescue "Terminated"
+          values["MachineID"] =        config.uuid rescue ""
+#          values["Public IP"] =         public_ip_address
+          values["Private IP"] =        guest.ipAddress rescue ""
+#          values["Created On"] =        created_at.to_date
+          return values if style == :minimal
+
+          # style == :default
+#          values["Flavor"] =            flavor_id
+#          values["AZ"] =                availability_zone
+          return values if style == :default
+
+          # style == :expanded
+#          values["Image"] =             image_id
+          values["Virtual Disks"] =      disks.map { |d| d.backing.fileName }.join(', ')
+#          values["SSH Key"] =           key_name
+          values
+        end
+
+        def ssh_key
+          keypair = cloud.keypair || computer.server.cluster_name
+        end
+
+        def self.validate_resources!(computers)
+          recall.each_value do |machine|
+            next unless machine.users.empty? and machine.name
+            if machine.name.match("^#{computers.cluster.name}-")
+              machine.bogus << :unexpected_machine
+            end
+            next unless machine.bogus?
+            fake           = Ironfan::Broker::Computer.new
+            fake[:machine] = machine
+            fake.name      = machine.name
+            machine.users << fake
+            computers     << fake
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/ironfan/provider/vsphere.rb

@@ -0,0 +1,79 @@
+module Ironfan
+  class Provider
+
+    class Vsphere < Ironfan::IaasProvider
+      self.handle = :vsphere
+
+      def self.resources()
+        [ Machine, Keypair]
+      end
+
+      #
+      # Utility functions
+      #
+
+      # Some methods cribbed and modified from https://github.com/ezrapagel/knife-vsphere, Ezra Pagel, Jesse Campbell
+      def self.connection
+        @@connection ||=  RbVmomi::VIM.connect(self.vsphere_credentials)
+      end
+
+      def self.find_ds(vsphere_dc, name = nil)
+        ds = vsphere_dc.datastore
+        return ds.grep(RbVmomi::VIM::Datastore).find { |d| d.name == name } if !name.nil?
+      end 
+
+      def self.find_dc(vsphere_dc)
+        connection.serviceInstance.find_datacenter(vsphere_dc)
+      end
+
+      def self.find_in_folder(folder, type, name)
+        folder.childEntity.grep(type).find { |o| o.name == name }
+      end
+
+      def self.get_rspec(dc, cluster)
+        if cluster
+          resource_pool = cluster.resourcePool
+        else
+          hosts = find_all_in_folder(dc.hostFolder, RbVmomi::VIM::ComputeResource).find { |o| o.class == RbVmomi::VIM::ComputeResource }
+          resource_pool = hosts.resourcePool || hosts.first.resourcePool
+        end
+        RbVmomi::VIM.VirtualMachineRelocateSpec(:pool => resource_pool)
+      end
+
+      def self.find_all_in_folder(folder, type)
+        if folder.instance_of?(RbVmomi::VIM::ClusterComputeResource)
+          folder = folder.resourcePool
+        end
+   
+        if folder.instance_of?(RbVmomi::VIM::ResourcePool)
+          folder.resourcePool.grep(type)
+        elsif folder.instance_of?(RbVmomi::VIM::Folder)
+          folder.childEntity.grep(type)
+        else
+          puts "Unknown type #{folder.class}, not enumerating"
+          nil
+        end
+      end
+
+      def self.find_network(network, dc)
+        baseEntity = dc.network
+        baseEntity.find { |f| f.name == network }
+      end
+
+      def self.get_pub_key(cluster)
+        puts Keypair.private_key
+      end
+
+      private
+      def self.vsphere_credentials
+        return {
+          :user                  => Chef::Config[:knife][:vsphere_username],
+          :port                  => Chef::Config[:knife][:vsphere_port] || 443,
+          :password              => Chef::Config[:knife][:vsphere_password],
+          :host                  => Chef::Config[:knife][:vsphere_server],
+          :insecure              => Chef::Config[:knife][:vsphere_insecure] || true
+        }
+      end
+    end
+  end
+end

data/lib/ironfan/provider.rb

@@ -13,6 +13,7 @@ module Ironfan
           case obj[:name]
           when :chef        then Chef
           when :ec2         then Ec2
+          when :vsphere     then Vsphere
           when :virtualbox  then VirtualBox
           else raise "Unsupported provider #{obj[:name]}"
           end

data/lib/ironfan/requirements.rb

@@ -19,6 +19,7 @@ require 'ironfan/dsl/volume'
 
 require 'ironfan/dsl/cloud'
 require 'ironfan/dsl/ec2'
+require 'ironfan/dsl/vsphere'
 
 
 # Providers for specific resources
@@ -42,6 +43,10 @@ require 'ironfan/provider/ec2/iam_server_certificate'
 require 'ironfan/provider/virtualbox'
 require 'ironfan/provider/virtualbox/machine'
 
+require 'ironfan/provider/vsphere'
+require 'ironfan/provider/vsphere/machine'
+require 'ironfan/provider/vsphere/keypair'
+
 
 # Broker classes to coordinate DSL expectations and provider resources
 require 'ironfan/broker'

metadata

@@ -1,148 +1,201 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: ironfan
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 4.9.0
   prerelease: 
-  version: 4.8.7
 platform: ruby
-authors: 
+authors:
 - Infochimps
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-03-05 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-04-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: chef
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: "10.16"
+      - !ruby/object:Gem::Version
+        version: '10.16'
   type: :runtime
   prerelease: false
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.16'
+- !ruby/object:Gem::Dependency
   name: fog
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: "1.2"
+      - !ruby/object:Gem::Version
+        version: '1.2'
   type: :runtime
   prerelease: false
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
   name: formatador
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: "0.2"
+      - !ruby/object:Gem::Version
+        version: '0.2'
   type: :runtime
   prerelease: false
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
   name: gorillib
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
+      - !ruby/object:Gem::Version
         version: 0.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+- !ruby/object:Gem::Dependency
   name: json
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
         version: 1.5.4
   type: :runtime
   prerelease: false
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.5.4
+- !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: "1.0"
+      - !ruby/object:Gem::Version
+        version: '1.0'
   type: :development
   prerelease: false
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   name: rake
-  requirement: &id007 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: "2.8"
+      - !ruby/object:Gem::Version
+        version: '2.8'
   type: :development
   prerelease: false
-  version_requirements: *id008
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
   name: yard
-  requirement: &id009 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0.7"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.7'
   type: :development
   prerelease: false
-  version_requirements: *id009
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
   name: redcarpet
-  requirement: &id010 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "2.1"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.1'
   type: :development
   prerelease: false
-  version_requirements: *id010
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
   name: oj
-  requirement: &id011 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "1.2"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.2'
   type: :development
   prerelease: false
-  version_requirements: *id011
-description: Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks.
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: Ironfan allows you to orchestrate not just systems but clusters of machines.
+  It includes a powerful layer on top of knife and a collection of cloud cookbooks.
 email: coders@infochimps.com
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE.md
 - README.md
-files: 
+files:
 - .gitignore
 - .rspec
 - .yardopts
@@ -199,6 +252,7 @@ files:
 - lib/ironfan/dsl/server.rb
 - lib/ironfan/dsl/virtualbox.rb
 - lib/ironfan/dsl/volume.rb
+- lib/ironfan/dsl/vsphere.rb
 - lib/ironfan/headers.rb
 - lib/ironfan/provider.rb
 - lib/ironfan/provider/chef.rb
@@ -216,6 +270,9 @@ files:
 - lib/ironfan/provider/ec2/security_group.rb
 - lib/ironfan/provider/virtualbox.rb
 - lib/ironfan/provider/virtualbox/machine.rb
+- lib/ironfan/provider/vsphere.rb
+- lib/ironfan/provider/vsphere/keypair.rb
+- lib/ironfan/provider/vsphere/machine.rb
 - lib/ironfan/requirements.rb
 - notes/Future-development-proposals.md
 - notes/Home.md
@@ -272,36 +329,34 @@ files:
 - spec/test_config.rb
 - tasks/chef_config.rake
 homepage: http://infochimps.com/labs
-licenses: 
+licenses:
 - apachev2
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: -3865228418823474445
-      segments: 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+      hash: 3090386787638060354
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Infochimps' lightweight cloud orchestration toolkit, built on top of Chef.
-test_files: 
+test_files:
 - spec/spec_helper/dummy_chef.rb
 - spec/integration/spec_helper/launch_cluster.rb
 - spec/integration/minimal-chef-repo/roles/systemwide.rb