chef-metal-fog 0.6.1 → 0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b5883afe070478a843245ccc9844ea0f355e9596
-  data.tar.gz: f3ea03361d9501d0860e32eadfbad2a0408164dc
+  metadata.gz: df1eb39ee54fce02bce795f16bcffb150a5a457b
+  data.tar.gz: 1c33c93f06ec8858fbd1f84c82850d351065936e
 SHA512:
-  metadata.gz: 580d1fc60224a67c4962f76c208ffa1691fb0b82b882e01769d624a066237fb2f4a413afb0591ca9562883373fe22dde212ebfd861374b772d63f031e7b12d3d
-  data.tar.gz: 83f6827b9ead2253391d5a409c32a869e40e2d4d31ab36cba8880f338e6ccac5149a14cd8732bd3129f08de1006d15ee4850b76d9a027f0ac9244a7e059b797c
+  metadata.gz: f669a9e485c513128afc5498932311c944d82eb185ccaba96c4fd5479be8c85c1cbefa2fd28432c2a69cb6045fea2e10c7cc0924973ce99548287921726001f4
+  data.tar.gz: 6660dc13199041cf38389da3820bff2b4572a0ba081b7531aff915e7cc464fea193c73662d75caf382a898b606011c45c07c38935ac70f567350c68717cf4406

data/lib/chef/provider/fog_key_pair.rb

@@ -32,6 +32,8 @@ class Chef::Provider::FogKeyPair < Chef::Provider::LWRPBase
     "#{new_resource.name} on #{new_driver.driver_url}"
   end
 
+  @@use_pkcs8 = nil # For Ruby 1.9 and below, PKCS can be run
+
   def create_key(action)
     if @should_create_directory
       Cheffish.inline_resource(self, action) do

data/lib/chef_metal_fog/fog_driver.rb

@@ -66,8 +66,8 @@ module ChefMetalFog
   #
   # - bootstrap_options: hash of options to pass to compute.servers.create
   # - is_windows: true if windows.  TODO detect this from ami?
-  # - create_timeout: the time to wait for the instance to boot to ssh (defaults to 600)
-  # - start_timeout: the time to wait for the instance to start (defaults to 600)
+  # - create_timeout: the time to wait for the instance to boot to ssh (defaults to 180)
+  # - start_timeout: the time to wait for the instance to start (defaults to 180)
   # - ssh_timeout: the time to wait for ssh to be available if the instance is detected as up (defaults to 20)
   # - ssh_username: username to use for ssh
   # - sudo: true to prefix all commands with "sudo"
@@ -155,6 +155,11 @@ module ChefMetalFog
     #   - log_level: :debug, :info, :warn, :error
     def initialize(driver_url, config)
       super(driver_url, config)
+      if config[:log_level] == :debug
+        Fog::Logger[:debug] = ::STDERR
+        Excon.defaults[:debug_request] = true
+        Excon.defaults[:debug_response] = true
+      end
     end
 
     def compute_options
@@ -170,7 +175,15 @@ module ChefMetalFog
     # converge, execute, file and directory.
     def allocate_machine(action_handler, machine_spec, machine_options)
       # If the server does not exist, create it
-      create_server(action_handler, machine_spec, machine_options)
+      create_servers(action_handler, { machine_spec => machine_options }, Chef::ChefFS::Parallelizer.new(0))
+      machine_spec
+    end
+
+    def allocate_machines(action_handler, specs_and_options, parallelizer)
+      create_servers(action_handler, specs_and_options, parallelizer) do |machine_spec, server|
+        yield machine_spec
+      end
+      specs_and_options.keys
     end
 
     def ready_machine(action_handler, machine_spec, machine_options)
@@ -251,46 +264,78 @@ module ChefMetalFog
       raise "unsupported fog provider #{provider} (please implement #creator)"
     end
 
-    def create_server(action_handler, machine_spec, machine_options)
-      if machine_spec.location
-        if machine_spec.location['driver_url'] != driver_url
-          raise "Switching a machine's driver from #{machine_spec.location['driver_url']} to #{driver_url} for is not currently supported!  Use machine :destroy and then re-create the machine on the new driver."
-        end
+    def create_servers(action_handler, specs_and_options, parallelizer, &block)
+      specs_and_servers = servers_for(specs_and_options.keys)
 
-        server = server_for(machine_spec)
+      # Get the list of servers which exist, segmented by their bootstrap options
+      # (we will try to create a set of servers for each set of bootstrap options
+      # with create_many)
+      by_bootstrap_options = {}
+      specs_and_options.each do |machine_spec, machine_options|
+        server = specs_and_servers[machine_spec]
         if server
           if %w(terminated archive).include?(server.state) # Can't come back from that
             Chef::Log.warn "Machine #{machine_spec.name} (#{server.id} on #{driver_url}) is terminated.  Recreating ..."
           else
-            return server
+            yield machine_spec, server if block_given?
+            next
           end
-        else
+        elsif machine_spec.location
           Chef::Log.warn "Machine #{machine_spec.name} (#{machine_spec.location['server_id']} on #{driver_url}) no longer exists.  Recreating ..."
         end
+
+        bootstrap_options = bootstrap_options_for(action_handler, machine_spec, machine_options)
+        by_bootstrap_options[bootstrap_options] ||= []
+        by_bootstrap_options[bootstrap_options] << machine_spec
       end
 
-      bootstrap_options = bootstrap_options_for(action_handler, machine_spec, machine_options)
+      # Create the servers in parallel
+      parallelizer.parallel_do(by_bootstrap_options) do |bootstrap_options, machine_specs|
+        machine_description = if machine_specs.size == 1
+          "machine #{machine_specs.first.name}"
+        else
+          "machines #{machine_specs.map { |s| s.name }.join(", ")}"
+        end
+        description = [ "creating #{machine_description} on #{driver_url}" ]
+        bootstrap_options.each_pair { |key,value| description << "  #{key}: #{value.inspect}" }
+        action_handler.report_progress description
 
-      description = [ "creating machine #{machine_spec.name} on #{driver_url}" ]
-      bootstrap_options.each_pair { |key,value| description << "  #{key}: #{value.inspect}" }
-      server = nil
-      action_handler.report_progress description
-      if action_handler.should_perform_actions
-        server = compute.servers.create(bootstrap_options)
-        machine_spec.location = {
-          'driver_url' => driver_url,
-          'driver_version' => ChefMetalFog::VERSION,
-          'server_id' => server.id,
-          'creator' => creator,
-          'allocated_at' => Time.now.to_i
-        }
-        machine_spec.location['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
-        %w(is_windows ssh_username sudo use_private_ip_for_ssh ssh_gateway).each do |key|
-          machine_spec.location[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
+        if action_handler.should_perform_actions
+          # Actually create the servers
+          create_many_servers(machine_specs.size, bootstrap_options, parallelizer) do |server|
+
+            # Assign each one to a machine spec
+            machine_spec = machine_specs.pop
+            machine_options = specs_and_options[machine_spec]
+            machine_spec.location = {
+              'driver_url' => driver_url,
+              'driver_version' => ChefMetalFog::VERSION,
+              'server_id' => server.id,
+              'creator' => creator,
+              'allocated_at' => Time.now.to_i
+            }
+            machine_spec.location['key_name'] = bootstrap_options[:key_name] if bootstrap_options[:key_name]
+            %w(is_windows ssh_username sudo use_private_ip_for_ssh ssh_gateway).each do |key|
+              machine_spec.location[key] = machine_options[key.to_sym] if machine_options[key.to_sym]
+            end
+            action_handler.performed_action "machine #{machine_spec.name} created as #{server.id} on #{driver_url}"
+
+            yield machine_spec, server if block_given?
+          end
+          
+          if machine_specs.size > 0
+            raise "Not all machines were created by create_many_machines!"
+          end
         end
       end
-      action_handler.performed_action "machine #{machine_spec.name} created as #{server.id} on #{driver_url}"
-      server
+    end
+
+    def create_many_servers(num_servers, bootstrap_options, parallelizer)
+      parallelizer.parallelize(1.upto(num_servers)) do |i|
+        server = compute.servers.create(bootstrap_options)
+        yield server if block_given?
+        server
+      end.to_a
     end
 
     def start_server(action_handler, machine_spec, server)
@@ -418,6 +463,21 @@ module ChefMetalFog
       end
     end
 
+    def servers_for(machine_specs)
+      result = {}
+      machine_specs.each do |machine_spec|
+        if machine_spec.location
+          if machine_spec.location['driver_url'] != driver_url
+            raise "Switching a machine's driver from #{machine_spec.location['driver_url']} to #{driver_url} for is not currently supported!  Use machine :destroy and then re-create the machine on the new driver."
+          end
+          result[machine_spec] = compute.servers.get(machine_spec.location['server_id'])
+        else
+          result[machine_spec] = nil
+        end
+      end
+      result
+    end
+
     @@metal_default_lock = Mutex.new
 
     def overwrite_default_key_willy_nilly(action_handler)

data/lib/chef_metal_fog/providers/aws.rb

@@ -26,12 +26,34 @@ module ChefMetalFog
         if !bootstrap_options[:key_name]
           bootstrap_options[:key_name] = overwrite_default_key_willy_nilly(action_handler)
         end
+        bootstrap_options.delete(:tags) # we handle these separately for performance reasons
+        bootstrap_options
+      end
 
-        bootstrap_options[:tags]  = default_tags(machine_spec, bootstrap_options[:tags] || {})
+      def create_servers(action_handler, specs_and_options, parallelizer)
+        super(action_handler, specs_and_options, parallelizer) do |machine_spec, server|
+          yield machine_spec, server if block_given?
 
-        bootstrap_options[:name] ||= machine_spec.name
+          machine_options = specs_and_options[machine_spec]
+          bootstrap_options = symbolize_keys(machine_options[:bootstrap_options] || {})
+          tags = default_tags(machine_spec, bootstrap_options[:tags] || {})
 
-        bootstrap_options
+          # Right now, not doing that in case manual tagging is going on
+          server_tags = server.tags || {}
+          extra_tags = tags.keys.select { |tag_name| !server_tags.has_key?(tag_name) }.to_a
+          different_tags = server_tags.select { |tag_name, tag_value| tags.has_key?(tag_name) && tags[tag_name] != tag_value }.to_a
+          if extra_tags.size > 0 || different_tags.size > 0
+            tags_description = [ "Update tags for #{machine_spec.name} on #{driver_url}" ]
+            tags_description += extra_tags.map { |tag| "  Add #{tag} = #{tags[tag].inspect}" }
+            tags_description += different_tags.map { |tag| "  Update #{tag.name} from #{tag.value.inspect} to #{tags[tag.name].inspect}"}
+            action_handler.perform_action tags_description do
+              # TODO should we narrow this down to just extra/different tags or
+              # is it OK to just pass 'em all?  Certainly easier to do the
+              # latter, and I can't think of a consequence for doing so offhand.
+              compute.create_tags(server.identity, tags)
+            end
+          end
+        end
       end
 
       def self.get_aws_profile(driver_options, aws_account_id)
@@ -209,6 +231,36 @@ module ChefMetalFog
 
         [result, id]
       end
+
+      def create_many_servers(num_servers, bootstrap_options, parallelizer)
+        # Create all the servers in one request if we have a version of fog that can do that
+        if compute.servers.respond_to?(:create_many)
+          servers = compute.servers.create_many(num_servers, num_servers, bootstrap_options)
+          if block_given?
+            parallelizer.parallel_do(servers) do |server|
+              yield server
+            end
+          end
+          servers
+        else
+          super
+        end
+      end
+
+      def servers_for(machine_specs)
+        # Grab all the servers in one request
+        instance_ids = machine_specs.map { |machine_spec| (machine_spec.location || {})['server_id'] }.select { |id| !id.nil? }
+        servers = compute.servers.all('instance-id' => instance_ids)
+        result = {}
+        machine_specs.each do |machine_spec|
+          if machine_spec.location
+            result[machine_spec] = servers.select { |s| s.id == machine_spec.location['server_id'] }.first
+          else
+            result[machine_spec] = nil
+          end
+        end
+        result
+      end
     end
   end
 end

data/lib/chef_metal_fog/providers/aws/credentials.rb

@@ -13,6 +13,9 @@ module ChefMetalFog
         include Enumerable
 
         def default
+          if @credentials.size == 0
+            raise "No credentials loaded!  Do you have a ~/.aws/config?"
+          end
           @credentials[ENV['AWS_DEFAULT_PROFILE'] || 'default'] || @credentials.first[1]
         end
 
@@ -30,15 +33,21 @@ module ChefMetalFog
 
         def load_ini(credentials_ini_file)
           inifile = IniFile.load(File.expand_path(credentials_ini_file))
-          inifile.each_section do |section|
-            if section =~ /^\s*profile\s+(.+)$/ || section =~ /^\s*(default)\s*/
-              profile_name = $1.strip
-              profile = inifile[section].inject({}) do |result, pair|
-                result[pair[0].to_sym] = pair[1]
-                result
+          if inifile
+            inifile.each_section do |section|
+              if section =~ /^\s*profile\s+(.+)$/ || section =~ /^\s*(default)\s*/
+                profile_name = $1.strip
+                profile = inifile[section].inject({}) do |result, pair|
+                  result[pair[0].to_sym] = pair[1]
+                  result
+                end
+                profile[:name] = profile_name
+                @credentials[profile_name] = profile
               end
-              profile[:name] = profile_name
-              @credentials[profile_name] = profile
+            end
+          else
+            # Get it to throw an error
+            File.open(File.expand_path(credentials_ini_file)) do
             end
           end
         end
@@ -55,7 +64,10 @@ module ChefMetalFog
         end
 
         def load_default
-          load_ini(ENV['AWS_CONFIG_FILE'] || File.expand_path('~/.aws/config'))
+          config_file = ENV['AWS_CONFIG_FILE'] || File.expand_path('~/.aws/config')
+          if File.file?(config_file)
+            load_ini(config_file)
+          end
         end
 
         def self.method_missing(name, *args, &block)

data/lib/chef_metal_fog/providers/cloudstack.rb

@@ -4,6 +4,10 @@ module ChefMetalFog
 
       ChefMetalFog::FogDriver.register_provider_class('CloudStack', ChefMetalFog::Providers::CloudStack)
 
+      def creator
+        ''
+      end
+
       def self.compute_options_for(provider, id, config)
         new_compute_options = {}
         new_compute_options[:provider] = provider

data/lib/chef_metal_fog/version.rb

@@ -1,3 +1,3 @@
 module ChefMetalFog
-  VERSION = '0.6.1'
+  VERSION = '0.7'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chef-metal-fog
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: '0.7'
 platform: ruby
 authors:
 - John Keiser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-18 00:00:00.000000000 Z
+date: 2014-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef