chef-metal 0.7 → 0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f78844f0b35265a0587a59130f2cc18f094c91ee
-  data.tar.gz: 4456458de1b2708269d784c6dd1c42559353b861
+  metadata.gz: 9811bbe55b66c09c1a01be9e87b80b918f2c5428
+  data.tar.gz: 1e5f30552fb823c7e11c9df9e1e08d5682604556
 SHA512:
-  metadata.gz: dbec23e421ace3000e3f09dca9142120683fbd5201d4187adf73f8381b36a1d4863a803a716e8bc77e69b0ce69b1d6c17980ea8fbdc355d37534d7a452a6de62
-  data.tar.gz: 3652afb4356159e48a3103c6fe1e9f6477219074453cd9fd11f164dce9c44f717f80c75e23d2d78f7202d7151c9f110e731d7e9aea109b816f64da67a0e38b0a
+  metadata.gz: e94c8a56ae93cdb19752de8ce82c33c5954def45c751b472d941c1c68242c03086fc50daa3f94bc4a9d87763049eea99e4b70b10312f738fa6e78fc768ec53f5
+  data.tar.gz: 91d84206a6fdcb0120f090255e4137e64b98a954c1af11e25bd8db127073744a9bdaad9ebde088d9ec3f96925ef07f87b6f7522ce7652b54f137b04c2b5ba779

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Chef Metal Changelog
 
+## 0.8 (4/8/2014)
+
+- New machine_execute resource! (irving@getchef.com)
+- Experimental "metal" command line: metal execute NODENAME COMMAND ARGS
+- Transport: Add ability to stream execute() for better nested chef-client debugging
+
 ## 0.7 (4/5/2014)
 
 - Change transport interface: add ability to rewrite URL instead of forwarding ports

data/bin/metal

@@ -0,0 +1,142 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require 'chef'
+require 'chef_metal'
+require 'chef/rest'
+require 'chef/application'
+require 'chef/knife'
+require 'chef/server_api'
+
+class ChefMetal::Application < Chef::Application
+
+  # Mimic self_pipe sleep from Unicorn to capture signals safely
+  SELF_PIPE = []
+
+  option :config_file,
+    :short => "-c CONFIG",
+    :long  => "--config CONFIG",
+    :description => "The configuration file to use"
+
+  option :log_level,
+    :short        => "-l LEVEL",
+    :long         => "--log_level LEVEL",
+    :description  => "Set the log level (debug, info, warn, error, fatal)",
+    :proc         => lambda { |l| l.to_sym }
+
+  option :log_location,
+    :short        => "-L LOGLOCATION",
+    :long         => "--logfile LOGLOCATION",
+    :description  => "Set the log file location, defaults to STDOUT - recommended for daemonizing",
+    :proc         => nil
+
+  option :node_name,
+    :short => "-N NODE_NAME",
+    :long => "--node-name NODE_NAME",
+    :description => "The node name for this client",
+    :proc => nil
+
+  option :chef_server_url,
+    :short => "-S CHEFSERVERURL",
+    :long => "--server CHEFSERVERURL",
+    :description => "The chef server URL",
+    :proc => nil
+
+  option :client_key,
+    :short        => "-k KEY_FILE",
+    :long         => "--client_key KEY_FILE",
+    :description  => "Set the client key file location",
+    :proc         => nil
+
+  option :local_mode,
+    :short        => "-z",
+    :long         => "--local-mode",
+    :description  => "Point chef-client at local repository",
+    :boolean      => true
+
+  option :chef_zero_port,
+    :long         => "--chef-zero-port PORT",
+    :description  => "Port to start chef-zero on"
+
+  def reconfigure
+    super
+
+    Chef::Config[:chef_server_url] = config[:chef_server_url] if config.has_key? :chef_server_url
+
+    Chef::Config.local_mode = config[:local_mode] if config.has_key?(:local_mode)
+    if Chef::Config.local_mode && !Chef::Config.has_key?(:cookbook_path) && !Chef::Config.has_key?(:chef_repo_path)
+      Chef::Config.chef_repo_path = Chef::Config.find_chef_repo_path(Dir.pwd)
+    end
+    Chef::Config.chef_zero.port = config[:chef_zero_port] if config[:chef_zero_port]
+
+    if Chef::Config[:daemonize]
+      Chef::Config[:interval] ||= 1800
+    end
+
+    if Chef::Config[:once]
+      Chef::Config[:interval] = nil
+      Chef::Config[:splay] = nil
+    end
+
+    if Chef::Config[:json_attribs]
+      config_fetcher = Chef::ConfigFetcher.new(Chef::Config[:json_attribs])
+      @chef_client_json = config_fetcher.fetch_json
+    end
+  end
+
+  def setup_application
+  end
+
+  def load_config_file
+    if !config.has_key?(:config_file)
+      require 'chef/knife'
+      config[:config_file] = Chef::Knife.locate_config_file
+    end
+    super
+  end
+
+  def run_application
+    Chef::Application.setup_server_connectivity
+
+    command = cli_arguments.shift
+    case command
+    when 'execute'
+      exit_code = 0
+      each_machine(cli_arguments.shift) do |machine|
+        puts "#{machine.node['name']}: running '#{cli_arguments.join(' ')}'"
+        result = machine.execute_always(cli_arguments.join(' '))
+        puts result.stdout if result.stdout != ''
+        STDERR.puts result.stderr if result.stderr != ''
+        exit_code = result.exitstatus if result.exitstatus != 0
+      end
+      exit(exit_code) if exit_code != 0
+    else
+      Chef::Log.error("Command '#{command}' unrecognized")
+    end
+
+    Chef::Application.destroy_server_connectivity
+  end
+
+  def rest
+    @rest ||= Chef::ServerAPI.new()
+  end
+
+  def each_machine(spec)
+    spec.split(',').each do |name|
+      node = rest.get("/nodes/#{name}")
+      provisioner_output = node['normal']['provisioner_output']
+      if !provisioner_output
+        Chef::Log.error("Node #{name} was not provisioned with Metal.")
+        next
+      end
+
+      provisioner = ChefMetal.provisioner_for_node(node)
+      machine = provisioner.connect_to_machine(node)
+
+      yield machine
+    end
+  end
+end
+
+ChefMetal::Application.new.run

data/lib/chef/provider/machine_execute.rb

@@ -0,0 +1,27 @@
+require 'chef/provider/lwrp_base'
+require 'cheffish/cheffish_server_api'
+
+class Chef::Provider::MachineExecute < Chef::Provider::LWRPBase
+
+  use_inline_resources
+
+  def whyrun_supported?
+    true
+  end
+
+  def machine
+    @machine ||= begin
+      if new_resource.machine.kind_of?(ChefMetal::Machine)
+        new_resource.machine
+      else
+        # TODO this is inefficient, can we cache or something?
+        node = Cheffish::CheffishServerAPI.new(new_resource.chef_server).get("/nodes/#{new_resource.machine}")
+        new_resource.provisioner.connect_to_machine(node)
+      end
+    end
+  end
+
+  action :run do
+    machine.execute(self, new_resource.command)
+  end
+end

data/lib/chef/resource/machine_execute.rb

@@ -0,0 +1,23 @@
+require 'chef/resource/lwrp_base'
+require 'chef_metal'
+require 'chef_metal/machine'
+require 'chef_metal/provisioner'
+
+class Chef::Resource::MachineExecute < Chef::Resource::LWRPBase
+  self.resource_name = 'machine_execute'
+
+  def initialize(*args)
+    super
+    @chef_server = Cheffish.enclosing_chef_server
+    @provisioner = ChefMetal.enclosing_provisioner
+  end
+
+  actions :run
+  default_action :run
+
+  attribute :command, :kind_of => String, :name_attribute => true
+  attribute :machine, :kind_of => [String, ChefMetal::Machine]
+
+  attribute :chef_server, :kind_of => Hash
+  attribute :provisioner, :kind_of => ChefMetal::Provisioner
+end

data/lib/chef_metal.rb

@@ -4,6 +4,8 @@ require 'chef/resource/machine'
 require 'chef/provider/machine'
 require 'chef/resource/machine_file'
 require 'chef/provider/machine_file'
+require 'chef/resource/machine_execute'
+require 'chef/provider/machine_execute'
 
 require 'chef_metal/inline_resource'
 

data/lib/chef_metal/convergence_strategy/install_cached.rb

@@ -33,7 +33,7 @@ module ChefMetal
       end
 
       def converge(action_handler, machine)
-        machine.execute(action_handler, 'chef-client')
+        machine.execute(action_handler, "chef-client -l #{Chef::Config.log_level.to_s}", :stream => true)
       end
 
       private

data/lib/chef_metal/convergence_strategy/install_sh.rb

@@ -29,7 +29,7 @@ module ChefMetal
       end
 
       def converge(action_handler, machine)
-        machine.execute(action_handler, 'chef-client')
+        machine.execute(action_handler, "chef-client -l #{Chef::Config.log_level.to_s}", :stream => true)
       end
     end
   end

data/lib/chef_metal/driver.rb

@@ -0,0 +1,86 @@
+module ChefMetal
+  class Provisioner
+    # Inflate a provisioner from node information; we don't want to force the
+    # driver to figure out what the provisioner really needs, since it varies
+    # from provisioner to provisioner.
+    #
+    # ## Parameters
+    # node - node to inflate the provisioner for
+    #
+    # returns a should return a Privisoner from the information provided
+    def self.inflate(node)
+      raise "#{self.class} does not override self.inflate"
+    end
+
+    # Acquire a machine, generally by provisioning it.  Returns a Machine
+    # object pointing at the machine, allowing useful actions like setup,
+    # converge, execute, file and directory.  The Machine object will have a
+    # "node" property which must be saved to the server (if it is any
+    # different from the original node object).
+    #
+    # ## Parameters
+    # action_handler - the action_handler object that is calling this method; this
+    #        is generally a provider, but could be anything that can support the
+    #        interface (i.e., in the case of the test kitchen metal driver for
+    #        acquiring and destroying VMs).
+    # node - node object (deserialized json) representing this machine.  If
+    #        the node has a provisioner_options hash in it, these will be used
+    #        instead of options provided by the provisioner.  TODO compare and
+    #        fail if different?
+    #        node will have node['normal']['provisioner_options'] in it with any
+    #        options. It is a hash with at least these options:
+    #
+    #           -- provisioner_url: <provisioner url>
+    #
+    #        node['normal']['provisioner_output'] will be populated with
+    #        information about the created machine.  For vagrant, it is a hash
+    #        with at least these options:
+    #
+    #           -- provisioner_url: <provisioner url>
+    #
+    def acquire_machine(action_handler, node)
+      raise "#{self.class} does not override acquire_machine"
+    end
+
+    # Connect to a machine without acquiring it.  This method will NOT make any
+    # changes to anything.
+    #
+    # ## Parameters
+    # node - node object (deserialized json) representing this machine.  The
+    #        node may have normal attributes "provisioner_options" and
+    #        "provisioner_output" in it, representing the input and output of
+    #        any prior "acquire_machine" process (if any).
+    #
+    def connect_to_machine(node)
+      raise "#{self.class} does not override connect_to_machine"
+    end
+
+    # Delete the given machine (idempotent).  Should destroy the machine,
+    # returning things to the state before acquire_machine was called.
+    def delete_machine(action_handler, node)
+      raise "#{self.class} does not override delete_machine"
+    end
+
+    # Stop the given machine.
+    def stop_machine(action_handler, node)
+      raise "#{self.class} does not override stop_machine"
+    end
+
+    # Provider notification that happens at the point a resource is declared
+    # (after all properties have been set on it)
+    def resource_created(machine)
+    end
+
+    protected
+
+    def save_node(provider, node, chef_server)
+      # Save the node and create the client.  TODO strip automatic attributes first so we don't race with "current state"
+      ChefMetal.inline_resource(provider) do
+        chef_node node['name'] do
+          chef_server chef_server
+          raw_json node
+        end
+      end
+    end
+  end
+end

data/lib/chef_metal/instance.rb

@@ -0,0 +1,54 @@
+module ChefMetal
+  class Provisioner
+    def create_machine(instance)
+    end
+    def delete_machine(instance)
+    end
+    def stop_machine(instance)
+    end
+  end
+end
+
+module ChefMetal
+  class Instance
+    def create
+    end
+    def delete
+    end
+    def stop
+    end
+    def converge
+    end
+    def
+
+  end
+end
+
+
+- MachineList (physical)
+  - Driver (lists, creates, deletes associated machines)
+    EC2
+    LXC
+    Docker
+    BareMetal
+  - Machine (physical)
+    - Transport
+      SSH (+Rsync)
+      WinRM
+      LXC IPC
+      Docker API
+    - Provisioner
+      Chef Client
+      Puppet
+
+- Cluster (logical)
+  Chef server
+  Puppet master
+  - Instance (logical)
+    create
+    delete
+    exists?
+    provision
+    verify
+    execute
+    upload/download

data/lib/chef_metal/machine.rb

@@ -17,10 +17,14 @@ module ChefMetal
       raise "converge not overridden on #{self.class}"
     end
 
-    def execute(action_handler, command)
+    def execute(action_handler, command, options = {})
       raise "execute not overridden on #{self.class}"
     end
 
+    def execute_always(command, options = {})
+      raise "execute_always not overridden on #{self.class}"
+    end
+
     def read_file(path)
       raise "read_file not overridden on #{self.class}"
     end

data/lib/chef_metal/machine/basic_machine.rb

@@ -23,14 +23,14 @@ module ChefMetal
         convergence_strategy.converge(action_handler, self)
       end
 
-      def execute(action_handler, command)
+      def execute(action_handler, command, options = {})
         action_handler.converge_by "run '#{command}' on #{node['name']}" do
-          transport.execute(command).error!
+          transport.execute(command, options).error!
         end
       end
 
-      def execute_always(command)
-        transport.execute(command)
+      def execute_always(command, options = {})
+        transport.execute(command, options)
       end
 
       def read_file(path)

data/lib/chef_metal/transport.rb

@@ -1,6 +1,6 @@
 module ChefMetal
   class Transport
-    def execute(command)
+    def execute(command, options = {})
       raise "execute not overridden on #{self.class}"
     end
 
@@ -31,5 +31,29 @@ module ChefMetal
     def available?
       raise "available? not overridden on #{self.class}"
     end
+
+    protected
+
+    # Helper to implement stdout/stderr streaming in execute
+    def stream_chunk(options, stdout_chunk, stderr_chunk)
+      if options[:stream].is_a?(Proc)
+        options[:stream].call(stdout_chunk, stderr_chunk)
+      else
+        if stdout_chunk
+          if options[:stream_stdout]
+            options[:stream_stdout].print stdout_chunk
+          elsif options[:stream]
+            STDOUT.print stdout_chunk
+          end
+        end
+        if stderr_chunk
+          if options[:stream_stderr]
+            options[:stream_stderr].print stderr_chunk
+          elsif options[:stream]
+            STDERR.print stderr_chunk
+          end
+        end
+      end
+    end
   end
 end

data/lib/chef_metal/transport/ssh.rb

@@ -4,7 +4,7 @@ require 'socket'
 
 module ChefMetal
   class Transport
-    class SSH < Transport
+    class SSH < ChefMetal::Transport
       def initialize(host, username, ssh_options, options)
         require 'net/ssh'
         require 'net/scp'
@@ -19,8 +19,8 @@ module ChefMetal
       attr_reader :ssh_options
       attr_reader :options
 
-      def execute(command)
-        Chef::Log.info("Executing #{command} on #{username}@#{host}")
+      def execute(command, execute_options = {})
+        Chef::Log.info("Executing #{options[:prefix]}#{command} on #{username}@#{host}")
         stdout = ''
         stderr = ''
         exitstatus = nil
@@ -38,10 +38,12 @@ module ChefMetal
 
             channel.on_data do |ch2, data|
               stdout << data
+              stream_chunk(execute_options, data, nil)
             end
 
             channel.on_extended_data do |ch2, type, data|
               stderr << data
+              stream_chunk(execute_options, nil, data)
             end
 
             channel.on_request "exit-status" do |ch, data|
@@ -53,9 +55,9 @@ module ChefMetal
         channel.wait
 
         Chef::Log.info("Completed #{command} on #{username}@#{host}: exit status #{exitstatus}")
-        Chef::Log.debug("Stdout was:\n#{stdout}") if stdout != ''
-        Chef::Log.info("Stderr was:\n#{stderr}") if stderr != ''
-        SSHResult.new(stdout, stderr, exitstatus)
+        Chef::Log.debug("Stdout was:\n#{stdout}") if stdout != '' && !options[:stream] && !options[:stream_stdout]
+        Chef::Log.info("Stderr was:\n#{stderr}") if stderr != '' && !options[:stream] && !options[:stream_stderr]
+        SSHResult.new(command, execute_options, stdout, stderr, exitstatus)
       end
 
       def read_file(path)
@@ -153,18 +155,25 @@ module ChefMetal
       end
 
       class SSHResult
-        def initialize(stdout, stderr, exitstatus)
+        def initialize(command, options, stdout, stderr, exitstatus)
+          @command = command
+          @options = options
           @stdout = stdout
           @stderr = stderr
           @exitstatus = exitstatus
         end
 
+        attr_reader :command
+        attr_reader :options
         attr_reader :stdout
         attr_reader :stderr
         attr_reader :exitstatus
 
         def error!
-          raise "Error: code #{exitstatus}.\nSTDOUT:#{stdout}\nSTDERR:#{stderr}" if exitstatus != 0
+          if exitstatus != 0
+            # TODO stdout/stderr is already printed at info/debug level.  Let's not print it twice, it's a lot.
+            msg = "Error: command '#{command}' exited with code #{exitstatus}.\n"
+          end
         end
       end
     end

data/lib/chef_metal/transport/winrm.rb

@@ -1,8 +1,9 @@
+require 'chef_metal/transport'
 require 'base64'
 
 module ChefMetal
   class Transport
-    class WinRM
+    class WinRM < ChefMetal::Transport
       def initialize(endpoint, type, options = {})
         @endpoint = endpoint
         @type = type
@@ -13,9 +14,11 @@ module ChefMetal
       attr_reader :type
       attr_reader :options
 
-      def execute(command)
-        output = session.run_powershell_script(command)
-        WinRMResult.new(output)
+      def execute(command, execute_options = {})
+        output = session.run_powershell_script(command) do |stdout, stderr|
+          stream_chunk(execute_options, stdout, stderr)
+        end
+        WinRMResult.new(command, execute_options, output)
       end
 
       def read_file(path)
@@ -50,7 +53,7 @@ $file.Close
       end
 
       def disconnect
-        # 
+        #
       end
 
       def escape(string)
@@ -68,7 +71,9 @@ $file.Close
       end
 
       class WinRMResult
-        def initialize(output)
+        def initialize(command, options, output)
+          @command = command
+          @options = options
           @exitstatus = output[:exitcode]
           @stdout = ''
           @stderr = ''
@@ -81,9 +86,15 @@ $file.Close
         attr_reader :stdout
         attr_reader :stderr
         attr_reader :exitstatus
+        attr_reader :command
+        attr_reader :options
 
         def error!
-          raise "Error: code #{exitstatus}.\nSTDOUT:#{stdout}\nSTDERR:#{stderr}" if exitstatus != 0
+          if exitstatus != 0
+            msg = "Error: command '#{command}' exited with code #{exitstatus}.\n"
+            msg << "STDOUT: #{stdout}" if !options[:stream] && !options[:stream_stdout]
+            msg << "STDERR: #{stderr}" if !options[:stream] && !options[:stream_stderr]
+          end
         end
       end
     end

data/lib/chef_metal/version.rb

@@ -1,3 +1,3 @@
 module ChefMetal
-  VERSION = '0.7'
+  VERSION = '0.8'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chef-metal
 version: !ruby/object:Gem::Version
-  version: '0.7'
+  version: '0.8'
 platform: ruby
 authors:
 - John Keiser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-06 00:00:00.000000000 Z
+date: 2014-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -138,7 +138,8 @@ dependencies:
         version: '0'
 description: A library for creating machines and infrastructures idempotently in Chef.
 email: jkeiser@opscode.com
-executables: []
+executables:
+- metal
 extensions: []
 extra_rdoc_files:
 - README.md
@@ -150,8 +151,10 @@ files:
 - README.md
 - CHANGELOG.md
 - lib/chef/provider/machine.rb
+- lib/chef/provider/machine_execute.rb
 - lib/chef/provider/machine_file.rb
 - lib/chef/resource/machine.rb
+- lib/chef/resource/machine_execute.rb
 - lib/chef/resource/machine_file.rb
 - lib/chef_metal/action_handler.rb
 - lib/chef_metal/aws_credentials.rb
@@ -161,7 +164,9 @@ files:
 - lib/chef_metal/convergence_strategy/no_converge.rb
 - lib/chef_metal/convergence_strategy/precreate_chef_objects.rb
 - lib/chef_metal/convergence_strategy.rb
+- lib/chef_metal/driver.rb
 - lib/chef_metal/inline_resource.rb
+- lib/chef_metal/instance.rb
 - lib/chef_metal/machine/basic_machine.rb
 - lib/chef_metal/machine/unix_machine.rb
 - lib/chef_metal/machine/windows_machine.rb
@@ -175,6 +180,7 @@ files:
 - lib/chef_metal/transport.rb
 - lib/chef_metal/version.rb
 - lib/chef_metal.rb
+- bin/metal
 homepage: http://wiki.opscode.com/display/chef
 licenses: []
 metadata: {}