clc-fork-chef-metal 0.11.beta.5

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler'
+require 'bundler/gem_tasks'
+
+task :spec do
+  require File.expand_path('spec/run')
+end

data/bin/metal

@@ -0,0 +1,276 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require 'chef_metal'
+require 'chef/rest'
+require 'chef/application'
+require 'chef/knife'
+require 'chef/run_context'
+require 'chef/server_api'
+require 'chef_metal/action_handler'
+require 'chef_metal/version'
+require 'chef_metal/chef_machine_spec'
+
+class ChefMetal::Application < Chef::Application
+
+  # Mimic self_pipe sleep from Unicorn to capture signals safely
+  SELF_PIPE = []
+
+  option :config_file,
+    :short => "-c CONFIG",
+    :long  => "--config CONFIG",
+    :description => "The configuration file to use"
+
+  option :log_level,
+    :short        => "-l LEVEL",
+    :long         => "--log_level LEVEL",
+    :description  => "Set the log level (debug, info, warn, error, fatal)",
+    :proc         => lambda { |l| l.to_sym }
+
+  option :log_location,
+    :short        => "-L LOGLOCATION",
+    :long         => "--logfile LOGLOCATION",
+    :description  => "Set the log file location, defaults to STDOUT - recommended for daemonizing",
+    :proc         => nil
+
+  option :node_name,
+    :short => "-N NODE_NAME",
+    :long => "--node-name NODE_NAME",
+    :description => "The node name for this client",
+    :proc => nil
+
+  option :chef_server_url,
+    :short => "-S CHEFSERVERURL",
+    :long => "--server CHEFSERVERURL",
+    :description => "The chef server URL",
+    :proc => nil
+
+  option :client_key,
+    :short        => "-k KEY_FILE",
+    :long         => "--client_key KEY_FILE",
+    :description  => "Set the client key file location",
+    :proc         => nil
+
+  option :local_mode,
+    :short        => "-z",
+    :long         => "--local-mode",
+    :description  => "Point chef-client at local repository",
+    :boolean      => true
+
+  option :chef_repo_path,
+    :long         => '--chef-repo-path=PATH',
+    :description  => "Path to Chef repository"
+
+  option :chef_zero_port,
+    :long         => "--chef-zero-port PORT",
+    :description  => "Port to start chef-zero on"
+
+  option :read_only,
+    :long         => "--[no-]read-only",
+    :description  => "Promise that execution will not modify the machine (helps with Docker in particular)"
+
+  option :stream,
+    :long         => "--[no-]stream",
+    :default      => true,
+    :boolean      => true,
+    :description  => "Whether to stream output from the machine (default: true)"
+
+  option :timeout,
+    :long         => "--timeout=TIMEOUT",
+    :default      => 15*60,
+    :description  => "Time to wait for program to execute, or 0 for no timeout (default: 15 minutes)"
+
+  def reconfigure
+    super
+
+    Chef::Config.chef_server_url = config[:chef_server_url] if config.has_key? :chef_server_url
+
+    Chef::Config.chef_repo_path = config[:chef_repo_path] if config.has_key? :chef_repo_path
+
+    Chef::Config.local_mode = config[:local_mode] if config.has_key?(:local_mode)
+    if Chef::Config.local_mode && !Chef::Config.has_key?(:cookbook_path) && !Chef::Config.has_key?(:chef_repo_path)
+      Chef::Config.chef_repo_path = Chef::Config.find_chef_repo_path(Dir.pwd)
+    end
+    Chef::Config.chef_zero.port = config[:chef_zero_port] if config[:chef_zero_port]
+  end
+
+  def setup_application
+  end
+
+  def load_config_file
+    if !config.has_key?(:config_file)
+      require 'chef/knife'
+      config[:config_file] = Chef::Knife.locate_config_file
+    end
+    super
+  end
+
+  def run_application
+    exit_code = 0
+
+    Cheffish.honor_local_mode do
+      command = cli_arguments.shift
+      case command
+      when 'execute'
+        connect_to_machines(cli_arguments.shift) do |machine|
+          machine.execute(action_handler, cli_arguments.join(' '), :read_only => config[:read_only], :stream => config[:stream], :timeout => config[:timeout].to_f)
+          puts result.stdout if result.stdout != '' && !config[:stream] && Chef::Config.log_level != :debug
+          STDERR.puts result.stderr if result.stderr != '' && !config[:stream] && Chef::Config.log_level != :debug
+          exit_code = result.exitstatus if result.exitstatus != 0
+        end
+      when 'destroy'
+        each_current_machine(cli_arguments.shift) do |driver, specs_and_options|
+          driver.destroy_machines(action_handler, specs_and_options, parallelizer)
+        end
+      when 'allocate'
+        each_new_machine(cli_arguments.shift) do |driver, specs_and_options|
+          driver.allocate_machines(action_handler, specs_and_options, parallelizer)
+        end
+      when 'ready'
+        each_new_machine(cli_arguments.shift) do |driver, specs_and_options|
+          driver.allocate_machines(action_handler, specs_and_options, parallelizer)
+          driver.ready_machines(action_handler, specs_and_options, parallelizer)
+        end
+      when 'setup'
+        each_new_machine(cli_arguments.shift) do |driver, specs_and_options|
+          driver.allocate_machines(action_handler, specs_and_options, parallelizer)
+          driver.ready_machines(action_handler, specs_and_options, parallelizer) do |machine|
+            machine.setup_convergence(action_handler)
+          end
+        end
+      when 'converge'
+        each_new_machine(cli_arguments.shift) do |driver, specs_and_options|
+          driver.allocate_machines(action_handler, specs_and_options, parallelizer)
+          driver.ready_machines(action_handler, specs_and_options, parallelizer) do |machine|
+            # TODO upload files?  Maybe they should be in machine_options?
+            machine.setup_convergence(action_handler)
+            machine.converge(action_handler)
+          end
+        end
+      when 'reconverge'
+        connect_to_machines(cli_arguments.shift) do |machine|
+          machine.converge(action_handler)
+        end
+      when 'stop'
+        each_current_machine(cli_arguments.shift) do |driver, specs_and_options|
+          driver.stop_machines(action_handler, specs_and_options, parallelizer)
+        end
+      when 'cat'
+        connect_to_machines(cli_arguments.shift) do |machine|
+          cli_arguments.each do |remote_path|
+            puts machine.read_file(remote_path)
+          end
+        end
+      when 'cp'
+        machines = {}
+        to = cli_arguments.pop
+        if to =~ /^([^\/:]+):(.+)$/
+          to_server = $1
+          machines[to_server] ||= ChefMetal.connect_to_machine(to_server)
+          to_path = $2
+          to_is_directory = machines[to_server].is_directory?(to_path)
+        else
+          to_server = nil
+          to_path = File.absolute_path(to)
+        end
+
+        cli_arguments.each do |from|
+          if from =~ /^([^\/:]+):(.+)$/
+            from_server = $1
+            from_path = $2
+            if to_server
+              raise "Cannot copy from one server to another, or intraserver (from=#{from}, to=#{to})"
+            end
+
+            machines[from_server] ||= connect_to_machine(from_server)
+            if File.directory?(to_path)
+              machines[from_server].download_file(action_handler, from_path, "#{to_path}/#{File.basename(from_path)}")
+            else
+              machines[from_server].download_file(action_handler, from_path, to_path)
+            end
+          else
+            from_server = nil
+            from_path = File.absolute_path(from)
+            if !to_server
+              raise "Cannot copy two local files.  One of the arguments must be MACHINE:PATH.  (from=#{from}, to=#{to})"
+            end
+
+            if to_is_directory
+              machines[to_server].upload_file(action_handler, from_path, "#{to_path}/#{File.basename(from_path)}")
+            else
+              machines[to_server].upload_file(action_handler, from_path, to_path)
+            end
+          end
+        end
+      else
+        Chef::Log.error("Command '#{command}' unrecognized")
+      end
+    end
+
+    exit(exit_code) if exit_code != 0
+  end
+
+  private
+
+  def rest
+    @rest ||= Chef::ServerAPI.new()
+  end
+
+  def machine_specs(*specs)
+    names = specs.collect_concat { |spec| spec.split(',') }.uniq
+    parallelizer.parallelize(names) { |name| ChefMetal::ChefMachineSpec.get(name) }.to_a
+  end
+
+  def each_new_machine(spec)
+    driver = ChefMetal.default_driver
+    specs_and_options = {}
+    machine_specs(spec).each do |machine_spec|
+      specs_and_options[machine_spec] = driver.config[:machine_options]
+    end
+    [ [ driver,  specs_and_options ] ]
+  end
+
+  def each_current_machine(spec)
+    grouped = machine_specs(spec).group_by { |machine_spec| machine_spec.driver_url }
+    parallelizer.parallelize(grouped) do |driver_url, machine_specs|
+      if driver_url
+        driver = ChefMetal.driver_for_url(driver_url)
+        specs_and_options = {}
+        machine_specs(spec).each do |machine_spec|
+          specs_and_options[machine_spec] = driver.config[:machine_options]
+        end
+        yield driver, specs_and_options
+      end
+    end.to_a
+  end
+
+  def connect_to_machines(spec)
+    grouped = machine_specs(spec).group_by { |machine_spec| machine_spec.driver_url }
+    grouped.collect_concat do |driver_url, machine_specs|
+      machine_specs.map { |machine_spec| ChefMetal.connect_to_machine(machine_spec) }
+    end
+  end
+
+  def parallelizer
+    Chef::ChefFS::Parallelizer
+  end
+
+  def action_handler
+    @action_handler ||= ActionHandler.new
+  end
+
+  class ActionHandler < ChefMetal::ActionHandler
+    def recipe_context
+      # TODO: somehow remove this code; should context really always be needed?
+      node = Chef::Node.new
+      node.name 'nothing'
+      node.automatic[:platform] = 'metal'
+      node.automatic[:platform_version] = ChefMetal::VERSION
+      Chef::RunContext.new(node, {},
+        Chef::EventDispatch::Dispatcher.new(Chef::Formatters::Doc.new(STDOUT,STDERR)))
+    end
+  end
+end
+
+ChefMetal::Application.new.run

data/lib/chef/provider/machine.rb

@@ -0,0 +1,147 @@
+require 'chef/provider/lwrp_base'
+require 'chef/provider/chef_node'
+require 'openssl'
+require 'chef_metal/chef_provider_action_handler'
+require 'chef_metal/chef_machine_spec'
+
+class Chef::Provider::Machine < Chef::Provider::LWRPBase
+
+  def action_handler
+    @action_handler ||= ChefMetal::ChefProviderActionHandler.new(self)
+  end
+
+  use_inline_resources
+
+  def whyrun_supported?
+    true
+  end
+
+  action :allocate do
+    new_driver.allocate_machine(action_handler, machine_spec, machine_options)
+    machine_spec.save(action_handler)
+  end
+
+  action :ready do
+    action_allocate
+    machine = current_driver.ready_machine(action_handler, machine_spec, machine_options)
+    machine_spec.save(action_handler)
+    machine
+  end
+
+  action :setup do
+    machine = action_ready
+    begin
+      machine.setup_convergence(action_handler)
+      machine_spec.save(action_handler)
+      upload_files(machine)
+    ensure
+      machine.disconnect
+    end
+  end
+
+  action :converge do
+    machine = action_ready
+    begin
+      machine.setup_convergence(action_handler)
+      machine_spec.save(action_handler)
+      upload_files(machine)
+      # If we were asked to converge, or anything changed, or if a converge has never succeeded, converge.
+      if new_resource.converge || (new_resource.converge.nil? && resource_updated?) ||
+         !machine_spec.node['automatic'] || machine_spec.node['automatic'].size == 0
+        machine.converge(action_handler)
+      end
+    ensure
+      machine.disconnect
+    end
+  end
+
+  action :converge_only do
+    machine = run_context.chef_metal.connect_to_machine(machine_spec, machine_options)
+    begin
+      machine.converge(action_handler)
+    ensure
+      machine.disconnect
+    end
+  end
+
+  action :stop do
+    if current_driver
+      current_driver.stop_machine(action_handler, machine_spec, machine_options)
+    end
+  end
+
+  action :destroy do
+    if current_driver
+      current_driver.destroy_machine(action_handler, machine_spec, machine_options)
+    end
+  end
+
+  def new_driver
+    run_context.chef_metal.driver_for(new_resource.driver)
+  end
+
+  def new_driver_config
+    run_context.chef_metal.driver_config_for(new_resource.driver)
+  end
+
+  def current_driver
+    run_context.chef_metal.current_driver || (
+      if machine_spec.driver_url
+        run_context.chef_metal.driver_for_url(machine_spec.driver_url)
+      end
+    )
+  end
+
+  attr_reader :machine_spec
+
+  def machine_options
+    configs = []
+    configs << {
+      :convergence_options =>
+        [ :chef_server,
+          :allow_overwrite_keys,
+          :source_key, :source_key_path, :source_key_pass_phrase,
+          :private_key_options,
+          :ohai_hints,
+          :public_key_path, :public_key_format,
+          :admin, :validator
+        ].inject({}) do |result, key|
+          result[key] = new_resource.send(key)
+          result
+        end
+    }
+    configs << new_resource.machine_options if new_resource.machine_options
+    configs << new_driver_config[:machine_options] if new_driver_config[:machine_options]
+    Cheffish::MergedConfig.new(*configs)
+  end
+
+  def load_current_resource
+    node_driver = Chef::Provider::ChefNode.new(new_resource, run_context)
+    node_driver.load_current_resource
+    json = node_driver.new_json
+    json['normal']['metal'] = node_driver.current_json['normal']['metal']
+    @machine_spec = ChefMetal::ChefMachineSpec.new(json, new_resource.chef_server)
+  end
+
+  def self.upload_files(action_handler, machine, files)
+    if files
+      files.each_pair do |remote_file, local|
+        if local.is_a?(Hash)
+          if local[:local_path]
+            machine.upload_file(action_handler, local[:local_path], remote_file)
+          else
+            machine.write_file(action_handler, remote_file, local[:content])
+          end
+        else
+          machine.upload_file(action_handler, local, remote_file)
+        end
+      end
+    end
+  end
+
+  private
+
+  def upload_files(machine)
+    Machine.upload_files(action_handler, machine, new_resource.files)
+  end
+end

data/lib/chef/provider/machine_batch.rb

@@ -0,0 +1,130 @@
+require 'chef/chef_fs/parallelizer'
+require 'chef/provider/lwrp_base'
+require 'chef/provider/machine'
+require 'chef_metal/chef_provider_action_handler'
+require 'chef_metal/add_prefix_action_handler'
+
+class Chef::Provider::MachineBatch < Chef::Provider::LWRPBase
+
+  def action_handler
+    @action_handler ||= ChefMetal::ChefProviderActionHandler.new(self)
+  end
+
+  use_inline_resources
+
+  def whyrun_supported?
+    true
+  end
+
+  def parallelizer
+    @parallelizer ||= Chef::ChefFS::Parallelizer.new(new_resource.max_simultaneous || 100)
+  end
+
+  action :allocate do
+    by_new_driver.each do |driver, specs_and_options|
+      driver.allocate_machines(action_handler, specs_and_options, parallelizer) do |machine_spec|
+        machine_spec.save(action_handler)
+      end
+    end
+  end
+
+  action :ready do
+    with_ready_machines
+  end
+
+  action :setup do
+    with_ready_machines do |m|
+      prefixed_handler = ChefMetal::AddPrefixActionHandler.new(action_handler, "[#{m[:resource].name}] ")
+      machine[:machine].setup_convergence(prefixed_handler)
+      m[:spec].save(action_handler)
+      Chef::Provider::Machine.upload_files(prefixed_handler, m[:machine], m[:resource].files)
+    end
+  end
+
+  action :converge do
+    with_ready_machines do |m|
+      prefixed_handler = ChefMetal::AddPrefixActionHandler.new(action_handler, "[#{m[:resource].name}] ")
+      m[:machine].setup_convergence(prefixed_handler)
+      m[:spec].save(action_handler)
+      Chef::Provider::Machine.upload_files(prefixed_handler, m[:machine], m[:resource].files)
+      m[:machine].converge(prefixed_handler)
+      m[:spec].save(action_handler)
+    end
+  end
+
+  action :stop do
+    parallel_do(by_current_driver) do |driver, specs_and_options|
+      driver.stop_machines(action_handler, specs_and_options, parallelizer)
+    end
+  end
+
+  action :destroy do
+    parallel_do(by_current_driver) do |driver, specs_and_options|
+      driver.destroy_machines(action_handler, specs_and_options, parallelizer)
+    end
+  end
+
+  def with_ready_machines
+    action_allocate
+    by_id = @machines.inject({}) { |hash,m| hash[m[:spec].id] = m; hash }
+    parallel_do(by_new_driver) do |driver, specs_and_options|
+      driver.ready_machines(action_handler, specs_and_options, parallelizer) do |machine|
+        machine.machine_spec.save(action_handler)
+
+        m = by_id[machine.machine_spec.id]
+
+        m[:machine] = machine
+        begin
+          yield m if block_given?
+        ensure
+          machine.disconnect
+        end
+      end
+    end
+  end
+
+  # TODO in many of these cases, the order of the results only matters because you
+  # want to match it up with the input.  Make a parallelize method that doesn't
+  # care about order and spits back results as quickly as possible.
+  def parallel_do(enum, options = {}, &block)
+    parallelizer.parallelize(enum, options, &block).to_a
+  end
+
+  def by_new_driver
+    result = {}
+    @machines.each do |m|
+      if m[:resource].driver
+        driver = run_context.chef_metal.driver_for(m[:resource].driver)
+        result[driver] ||= {}
+        result[driver][m[:spec]] = m[:options]
+      end
+    end
+    result
+  end
+
+  def by_current_driver
+    result = {}
+    @machines.each do |m|
+      if m[:spec].driver_url
+        driver = run_context.chef_metal.driver_for_url(m[:spec].driver_url)
+        result[driver] ||= {}
+        result[driver][m[:spec]] = m[:options]
+      end
+    end
+    result
+  end
+
+  def load_current_resource
+    # Load nodes in parallel
+    @machines = parallel_do(new_resource.machines) do |machine_resource|
+      provider = Chef::Provider::Machine.new(machine_resource, machine_resource.run_context)
+      provider.load_current_resource
+      {
+        :resource => machine_resource,
+        :spec => provider.machine_spec,
+        :options => provider.machine_options
+      }
+    end.to_a
+  end
+
+end