chef-metal 0.10.2 → 0.11.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af79447510d93b295a7d77a3223d670bdc569415
-  data.tar.gz: ac28faf8d8afa3026960b9997de94d9c4fc77369
+  metadata.gz: 544a8fd4658a8ad73c179ddd5f804440cb7f3f8e
+  data.tar.gz: ac8690681f379ae779bbc4b7b26d4c68281ec359
 SHA512:
-  metadata.gz: 49762ba3a7f0f71677b85d6f9657cdb05174c1d389ba96825bb218f9af7dd557f35ab515dd391865caba7dc20a6a3ddfbd12b1a3c566219bab1e6cda7d8248fa
-  data.tar.gz: 5412dfb1f503aa30259bf37c16a7f8e46dffc98c7434375fefaf208a3745af25208052d60a4982a6a809444eaaeb82211e6490fa5a9c52cced6a7347cf70c0a0
+  metadata.gz: d0f8d6f2f781e577898413a3bcc2cd6c9493dc592b11144ab1764b27f4c1fd5eca28c34571befb510a99790659feb03a0a7c5435cb244a55e262c762414ce041
+  data.tar.gz: 7575521c82702f2be0a9d82a840f71794d63ec3b417c371c8e0fa937452dbdaceb4a881d70f45123a8f73592a46524ce5ab37ef35daf8b69c22eb69fe64f91b1

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Chef Metal Changelog
 
+## 0.11.beta (5/23/2014)
+
+- New Driver interface (see docs/ and blogs/ directories for documentation)
+
 ## 0.10.2 (5/2/2014)
 
 - Fix crash with add_provisioner_options when provisioner_options is not yet set
@@ -72,11 +76,11 @@
 ## 0.5 (4/3/2014)
 
 * Provisioner interface changes designed to allow provisioners to be used outside of Chef (doubt@getchef.com)
-  * All Provisioner and Machine methods now take "action_handler" instead of "provider."  It uses the ActionHandler interface described in action_handler.rb.  In short:
-    - provider.run_context -> action_handler.recipe_context
-    - provider.updated_by_last_action(true) -> action_handler.updated!
-    - provider.converge_by -> action_handler.perform_action
-    - provider.cookbook_name -> provider.debug_name
+  * All Provisioner and Machine methods now take "action_handler" instead of "driver."  It uses the ActionHandler interface described in action_handler.rb.  In short:
+    - driver.run_context -> action_handler.recipe_context
+    - driver.updated_by_last_action(true) -> action_handler.updated!
+    - driver.converge_by -> action_handler.perform_action
+    - driver.cookbook_name -> driver.debug_name
   * Convergence strategy: delete_chef_objects() -> cleanup_convergence()
 * Ability to get back to a machine from a node (another Provisioner interface change) (doubt@getchef.com):
   * Provisioners must create a file named `chef_metal/provisioner_init/<scheme>_init.rb`.  It will be required when a node is encountered with that scheme.  It should call ChefMetal.add_registered_provisioner_class(<scheme>, <provisioner class name>).  For the provisioner_url `fog:AWS:21348723432`, the scheme is "fog" and the file is `chef_metalprovisioner_init/fog_init.rb`.  It should call `ChefMetal.add_registered_provisioner_class('fog', ChefMetal::Provisioner::FogProvisioner)`.

data/README.md

@@ -89,7 +89,7 @@ Provisioners each have their own repository.  Current provisioners:
 
 ### Vagrant
 
-chef-zero comes with a provisioner for Vagrant, an abstraction that covers VirtualBox, VMWare and other Virtual Machine providers. To run it, you can check out the sample recipes with:
+chef-zero comes with a provisioner for Vagrant, an abstraction that covers VirtualBox, VMWare and other Virtual Machine drivers. To run it, you can check out the sample recipes with:
 
 ```
 chef-client -z -o myapp::vagrant,myapp::linux,myapp::small
@@ -144,7 +144,7 @@ end
 
 ### Fog (EC2 and friends)
 
-chef-metal also comes with a [Fog](http://fog.io/) provisioner that handles provisioning to Amazon's EC2 and other cloud providers.  (Only EC2 has been tested so far.)  Before you begin, you will need to put your AWS credentials in ~/.aws/config in the format [mentioned in Option 1 here](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#d0e726).
+chef-metal also comes with a [Fog](http://fog.io/) provisioner that handles provisioning to Amazon's EC2 and other cloud drivers.  (Only EC2 has been tested so far.)  Before you begin, you will need to put your AWS credentials in ~/.aws/config in the format [mentioned in Option 1 here](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#d0e726).
 
 Once your credentials are in, basic usage looks like this:
 
@@ -159,7 +159,7 @@ ec2testdir = File.expand_path('~/ec2test')
 
 directory ec2testdir
 
-with_fog_ec2_provisioner # :provider => 'AWS'
+with_fog_ec2_provisioner # :driver => 'AWS'
 
 with_provisioner_options :image_id => 'ami-5ee70037'
 

data/bin/metal

@@ -134,22 +134,22 @@ class ChefMetal::Application < Chef::Application
       end
     when 'converge'
       each_machine(cli_arguments.shift) do |machine, provisioner|
-        machine.converge(action_handler, chef_server)
+        machine.converge(action_handler)
       end
     when 'delete'
       each_machine(cli_arguments.shift) do |machine, provisioner|
-        provisioner.delete_machine(action_handler, machine.node)
+        provisioner.delete_machine(action_handler, machine.machine_spec)
       end
     when 'update'
       each_machine(cli_arguments.shift) do |machine, provisioner|
-        machine = provisioner.acquire_machine(action_handler, machine.node)
+        machine = provisioner.acquire_machine(action_handler, machine.machine_spec)
         # TODO find out what files were uploaded so we can do it again
         machine.setup_convergence(action_handler)
-        machine.converge(action_handler, chef_server)
+        machine.converge(action_handler)
       end
     when 'stop'
       each_machine(cli_arguments.shift) do |machine, provisioner|
-        provisioner.stop_machine(action_handler, machine.node)
+        provisioner.stop_machine(action_handler, machine.machine_spec)
       end
     when 'cat'
       each_machine(cli_arguments.shift) do |machine, provisioner|
@@ -242,10 +242,6 @@ class ChefMetal::Application < Chef::Application
       Chef::RunContext.new(node, {},
         Chef::EventDispatch::Dispatcher.new(Chef::Formatters::Doc.new(STDOUT,STDERR)))
     end
-
-    def debug_name
-      'metal'
-    end
   end
 end
 

data/lib/chef/provider/machine.rb

@@ -1,11 +1,14 @@
 require 'chef/provider/lwrp_base'
 require 'chef/provider/chef_node'
 require 'openssl'
-require 'chef_metal/provider_action_handler'
+require 'chef_metal/chef_provider_action_handler'
+require 'chef_metal/chef_machine_spec'
 
 class Chef::Provider::Machine < Chef::Provider::LWRPBase
 
-  include ChefMetal::ProviderActionHandler
+  def action_handler
+    @action_handler ||= ChefMetal::ChefProviderActionHandler.new(self)
+  end
 
   use_inline_resources
 
@@ -13,59 +16,105 @@ class Chef::Provider::Machine < Chef::Provider::LWRPBase
     true
   end
 
-  action :create do
-    node_json = node_provider.new_json
-    node_json['normal']['provisioner_options'] = new_resource.provisioner_options
-    # Preserve provisioner_output, whatever the cost
-    node_json['normal']['provisioner_output'] = node_provider.current_json['normal']['provisioner_output']
-    machine = new_resource.provisioner.acquire_machine(self, node_json)
+  action :allocate do
+    new_driver.allocate_machine(action_handler, machine_spec, machine_options)
+    machine_spec.save(action_handler)
+  end
+
+  action :ready do
+    action_allocate
+    machine = current_driver.ready_machine(action_handler, machine_spec, machine_options)
+  end
+
+  action :setup do
+    machine = action_ready
+    begin
+      machine.setup_convergence(action_handler)
+      upload_files(machine)
+    ensure
+      machine.disconnect
+    end
+  end
+
+  action :converge do
+    machine = action_ready
     begin
-      machine.setup_convergence(self, new_resource)
+      machine.setup_convergence(action_handler)
       upload_files(machine)
       # If we were asked to converge, or anything changed, or if a converge has never succeeded, converge.
       if new_resource.converge || (new_resource.converge.nil? && resource_updated?) ||
-         !node_json['automatic'] || node_json['automatic'].size == 0
-        machine.converge(self, new_resource.chef_server)
+         !machine_spec.node['automatic'] || machine_spec.node['automatic'].size == 0
+        machine.converge(action_handler)
       end
     ensure
       machine.disconnect
     end
   end
 
-  action :converge do
-    node_json = node_provider.new_json
-    node_json['normal']['provisioner_options'] = new_resource.provisioner_options
-    # Preserve provisioner_output, whatever the cost
-    node_json['normal']['provisioner_output'] = node_provider.current_json['normal']['provisioner_output']
-    machine = new_resource.provisioner.connect_to_machine(node_json)
+  action :converge_only do
+    machine = run_context.chef_metal.connect_to_machine(machine_spec, machine_options)
     begin
-      machine.converge(self, new_resource.chef_server)
+      machine.converge(action_handler)
     ensure
       machine.disconnect
     end
   end
 
   action :stop do
-    node_json = node_provider.new_json
-    node_json['normal']['provisioner_options'] = new_resource.provisioner_options
-    # Preserve provisioner_output, whatever the cost
-    node_json['normal']['provisioner_output'] = node_provider.current_json['normal']['provisioner_output']
-    new_resource.provisioner.stop_machine(self, node_json)
+    if current_driver
+      current_driver.stop_machine(action_handler, machine_spec, machine_options)
+    end
+  end
+
+  action :destroy do
+    if current_driver
+      current_driver.destroy_machine(action_handler, machine_spec, machine_options)
+    end
+  end
+
+  def new_driver
+    run_context.chef_metal.driver_for(new_resource.driver)
   end
 
-  action :delete do
-    # Grab the node json by asking the provider for it
-    node_data = node_provider.current_json
+  def new_driver_config
+    run_context.chef_metal.driver_config_for(new_resource.driver)
+  end
 
-    # Destroy the machine
-    new_resource.provisioner.delete_machine(self, node_data)
+  def current_driver
+    if machine_spec.driver_url
+      run_context.chef_metal.driver_for_url(machine_spec.driver_url)
+    end
   end
 
-  attr_reader :node_provider
+  attr_reader :machine_spec
+
+  def machine_options
+    configs = []
+    configs << {
+      :convergence_options =>
+        [ :chef_server,
+          :allow_overwrite_keys,
+          :source_key, :source_key_path, :source_key_pass_phrase,
+          :private_key_options,
+          :ohai_hints,
+          :public_key_path, :public_key_format,
+          :admin, :validator
+        ].inject({}) do |result, key|
+          result[key] = new_resource.send(key)
+          result
+        end
+    }
+    configs << new_resource.machine_options if new_resource.machine_options
+    configs << new_driver_config[:machine_options] if new_driver_config[:machine_options]
+    Cheffish::MergedConfig.new(*configs)
+  end
 
   def load_current_resource
-    @node_provider = Chef::Provider::ChefNode.new(new_resource, run_context)
-    @node_provider.load_current_resource
+    node_driver = Chef::Provider::ChefNode.new(new_resource, run_context)
+    node_driver.load_current_resource
+    json = node_driver.new_json
+    json['normal']['metal'] = node_driver.current_json['normal']['metal']
+    @machine_spec = ChefMetal::ChefMachineSpec.new(json, new_resource.chef_server)
   end
 
   def self.upload_files(action_handler, machine, files)
@@ -87,6 +136,6 @@ class Chef::Provider::Machine < Chef::Provider::LWRPBase
   private
 
   def upload_files(machine)
-    Machine.upload_files(self, machine, new_resource.files)
+    Machine.upload_files(action_handler, machine, new_resource.files)
   end
 end

data/lib/chef/provider/machine_batch.rb

@@ -1,12 +1,14 @@
 require 'chef/chef_fs/parallelizer'
 require 'chef/provider/lwrp_base'
 require 'chef/provider/machine'
-require 'chef_metal/provider_action_handler'
+require 'chef_metal/chef_provider_action_handler'
 require 'chef_metal/add_prefix_action_handler'
 
 class Chef::Provider::MachineBatch < Chef::Provider::LWRPBase
 
-  include ChefMetal::ProviderActionHandler
+  def action_handler
+    @action_handler ||= ChefMetal::ChefProviderActionHandler.new(self)
+  end
 
   use_inline_resources
 
@@ -18,64 +20,59 @@ class Chef::Provider::MachineBatch < Chef::Provider::LWRPBase
     @parallelizer ||= Chef::ChefFS::Parallelizer.new(new_resource.max_simultaneous || 100)
   end
 
-  action :boot do
-    with_booted_machines
+  action :allocate do
+    by_new_driver.each do |driver, specs_and_options|
+      driver.allocate_machines(action_handler, specs_and_options, parallelizer) do |machine_spec|
+        machine_spec.save(action_handler)
+      end
+    end
+  end
+
+  action :ready do
+    with_ready_machines
   end
 
   action :setup do
-    with_booted_machines do |machine|
-      prefixed_handler = ChefMetal::AddPrefixActionHandler.new(self, "[#{machine[:resource].name}] ")
-      machine[:machine].setup_convergence(prefixed_handler, machine[:resource])
-      Chef::Provider::Machine.upload_files(prefixed_handler, machine[:machine], machine[:resource].files)
+    with_ready_machines do |m|
+      prefixed_handler = ChefMetal::AddPrefixActionHandler.new(action_handler, "[#{m[:resource].name}] ")
+      machine[:machine].setup_convergence(prefixed_handler)
+      Chef::Provider::Machine.upload_files(prefixed_handler, m[:machine], m[:resource].files)
     end
   end
 
   action :converge do
-    with_booted_machines do |machine|
-      prefixed_handler = ChefMetal::AddPrefixActionHandler.new(self, "[#{machine[:resource].name}] ")
-      machine[:machine].setup_convergence(prefixed_handler, machine[:resource])
-      Chef::Provider::Machine.upload_files(prefixed_handler, machine[:machine], machine[:resource].files)
-      machine[:machine].converge(prefixed_handler, machine[:resource].chef_server)
+    with_ready_machines do |m|
+      prefixed_handler = ChefMetal::AddPrefixActionHandler.new(action_handler, "[#{m[:resource].name}] ")
+      m[:machine].setup_convergence(prefixed_handler)
+      Chef::Provider::Machine.upload_files(prefixed_handler, m[:machine], m[:resource].files)
+      m[:machine].converge(prefixed_handler)
     end
   end
 
   action :stop do
-    parallel_do(@by_provisioner) do |provisioner, node_urls|
-      provisioner.stop_machines(self, node_urls.map { |n| @by_node[n][:provider].node_provider.current_json }, parallelizer)
+    parallel_do(by_current_driver) do |driver, specs_and_options|
+      driver.stop_machines(action_handler, specs_and_options, parallelizer)
     end
   end
 
-  action :delete do
-    parallel_do(@by_provisioner) do |provisioner, node_urls|
-      provisioner.delete_machines(self, node_urls.map { |n| @by_node[n][:provider].node_provider.current_json }, parallelizer)
+  action :destroy do
+    parallel_do(by_current_driver) do |driver, specs_and_options|
+      driver.destroy_machines(action_handler, specs_and_options, parallelizer)
     end
   end
 
-  def with_booted_machines
-    parallel_do(@by_provisioner) do |provisioner, node_urls|
-      machines = node_urls.map do |node_url|
-        # Fill in the provisioner options and output in case they got overwritten
-        machine = @by_node[node_url]
-        machine[:node] = machine[:provider].node_provider.new_json
-        machine[:node]['normal']['provisioner_options'] = machine[:resource].provisioner_options
-        machine[:node]['normal']['provisioner_output'] = machine[:provider].node_provider.current_json['normal']['provisioner_output']
-        machine
-      end
-
-      # TODO I don't understand why the object_id hack was necessary.  Using the
-      # node as a key didn't work. If we could pass node_urls through acquire_machines,
-      # that would solve the problem in a bestest way (nodes themselves are not
-      # necessarily unique without knowing the chef_server with which they are
-      # associated)
-      by_node_json = machines.inject({}) { |result, machine| result[machine[:node].object_id] = machine; result }
-      provisioner.acquire_machines(self, by_node_json.values.map { |m| m[:node] }, parallelizer) do |node_json, machine_obj|
-        machine = by_node_json[node_json.object_id]
+  def with_ready_machines
+    action_allocate
+    by_id = @machines.inject({}) { |hash,m| hash[m[:spec].id] = m; hash }
+    parallel_do(by_new_driver) do |driver, specs_and_options|
+      driver.ready_machines(action_handler, specs_and_options, parallelizer) do |machine|
+        m = by_id[machine.machine_spec.id]
 
-        machine[:machine] = machine_obj
+        m[:machine] = machine
         begin
-          yield machine if block_given?
+          yield m if block_given?
         ensure
-          machine_obj.disconnect
+          machine.disconnect
         end
       end
     end
@@ -88,29 +85,41 @@ class Chef::Provider::MachineBatch < Chef::Provider::LWRPBase
     parallelizer.parallelize(enum, options, &block).to_a
   end
 
-  def node_url(machine_resource)
-    "#{machine_resource.chef_server[:chef_server_url]}/nodes/#{machine_resource.name}"
+  def by_new_driver
+    result = {}
+    @machines.each do |m|
+      if m[:resource].driver
+        driver = run_context.chef_metal.driver_for(m[:resource].driver)
+        result[driver] ||= {}
+        result[driver][m[:spec]] = m[:options]
+      end
+    end
+    result
+  end
+
+  def by_current_driver
+    result = {}
+    @machines.each do |m|
+      if m[:spec].driver_url
+        driver = run_context.chef_metal.driver_for_url(m[:spec].driver_url)
+        result[driver] ||= {}
+        result[driver][m[:spec]] = m[:options]
+      end
+    end
+    result
   end
 
   def load_current_resource
-    # Figure out which machines are in the batch, remove duplicates, and retrieve
-    # the nodes from the Chef server if they exist.
-    @by_provisioner = {}
-    @by_node = {}
-    new_resource.machines.each do |machine_resource|
-      next if @by_node.has_key?(node_url(machine_resource))
-      next unless Array(machine_resource.action).include?(:create)
-      @by_node[node_url(machine_resource)] = {
+    # Load nodes in parallel
+    @machines = parallel_do(new_resource.machines) do |machine_resource|
+      provider = Chef::Provider::Machine.new(machine_resource, machine_resource.run_context)
+      provider.load_current_resource
+      {
         :resource => machine_resource,
-        :provider => Chef::Provider::Machine.new(machine_resource, run_context)
+        :spec => provider.machine_spec,
+        :options => provider.machine_options
       }
-      @by_provisioner[machine_resource.provisioner] ||= []
-      @by_provisioner[machine_resource.provisioner] << node_url(machine_resource)
-    end
-    # Load nodes in parallel
-    parallel_do(@by_node.values) do |machine|
-      machine[:provider].load_current_resource
-    end
+    end.to_a
   end
 
 end

data/lib/chef/provider/machine_execute.rb

@@ -1,10 +1,12 @@
 require 'chef/provider/lwrp_base'
-require 'cheffish/cheffish_server_api'
-require 'chef_metal/provider_action_handler'
+require 'chef_metal/chef_provider_action_handler'
+require 'chef_metal/machine'
 
 class Chef::Provider::MachineExecute < Chef::Provider::LWRPBase
 
-  include ChefMetal::ProviderActionHandler
+  def action_handler
+    @action_handler ||= ChefMetal::ChefProviderActionHandler.new(self)
+  end
 
   use_inline_resources
 
@@ -17,14 +19,12 @@ class Chef::Provider::MachineExecute < Chef::Provider::LWRPBase
       if new_resource.machine.kind_of?(ChefMetal::Machine)
         new_resource.machine
       else
-        # TODO this is inefficient, can we cache or something?
-        node = Cheffish::CheffishServerAPI.new(new_resource.chef_server).get("/nodes/#{new_resource.machine}")
-        new_resource.provisioner.connect_to_machine(node)
+        run_context.chef_metal.connect_to_machine(new_resource.machine, new_resource.chef_server)
       end
     end
   end
 
   action :run do
-    machine.execute(self, new_resource.command)
+    machine.execute(action_handler, new_resource.command)
   end
 end