clc-fork-chef-metal 0.11.beta.5

data/lib/chef_metal/machine/windows_machine.rb

@@ -0,0 +1,102 @@
+require 'chef_metal/machine/basic_machine'
+
+module ChefMetal
+  class Machine
+    class WindowsMachine < BasicMachine
+      def initialize(machine_spec, transport, convergence_strategy)
+        super
+      end
+
+      # Options include:
+      #
+      # command_prefix - prefix to put in front of any command, e.g. sudo
+      attr_reader :options
+
+      # Delete file
+      def delete_file(action_handler, path)
+        if file_exists?(path)
+          action_handler.perform_action "delete file #{escape(path)} on #{machine_spec.name}" do
+            transport.execute("Remove-Item #{escape(path)}").error!
+          end
+        end
+      end
+
+      def is_directory?(path)
+        parse_boolean(transport.execute("Test-Path #{escape(path)} -pathtype container", :read_only => true).stdout)
+      end
+
+      # Return true or false depending on whether file exists
+      def file_exists?(path)
+        parse_boolean(transport.execute("Test-Path #{escape(path)}", :read_only => true).stdout)
+      end
+
+      def files_different?(path, local_path, content=nil)
+        if !file_exists?(path) || (local_path && !File.exists?(local_path))
+          return true
+        end
+
+        # Get remote checksum of file (from http://stackoverflow.com/a/13926809)
+        result = transport.execute(<<-EOM, :read_only => true)
+$md5 = [System.Security.Cryptography.MD5]::Create("MD5")
+$fd = [System.IO.File]::OpenRead(#{path.inspect})
+$buf = new-object byte[] (1024*1024*8) # 8mb buffer
+while (($read_len = $fd.Read($buf,0,$buf.length)) -eq $buf.length){
+    $total += $buf.length
+    $md5.TransformBlock($buf,$offset,$buf.length,$buf,$offset)
+}
+# finalize the last read
+$md5.TransformFinalBlock($buf,0,$read_len)
+$hash = $md5.Hash
+# convert hash bytes to hex formatted string
+$hash | foreach { $hash_txt += $_.ToString("x2") }
+$hash_txt
+EOM
+        result.error!
+        remote_sum = result.stdout.split(' ')[0]
+        digest = Digest::SHA256.new
+        if content
+          digest.update(content)
+        else
+          File.open(local_path, 'rb') do |io|
+            while (buf = io.read(4096)) && buf.length > 0
+              digest.update(buf)
+            end
+          end
+        end
+        remote_sum != digest.hexdigest
+      end
+
+      def create_dir(action_handler, path)
+        if !file_exists?(path)
+          action_handler.perform_action "create directory #{path} on #{machine_spec.name}" do
+            transport.execute("New-Item #{escape(path)} -Type directory")
+          end
+        end
+      end
+
+      # Set file attributes { :owner, :group, :rights }
+#      def set_attributes(action_handler, path, attributes)
+#      end
+
+      # Get file attributes { :owner, :group, :rights }
+#      def get_attributes(path)
+#      end
+
+      def dirname_on_machine(path)
+        path.split(/[\\\/]/)[0..-2].join('\\')
+      end
+
+      def escape(string)
+        transport.escape(string)
+      end
+
+      def parse_boolean(string)
+        if string =~ /^\s*true\s*$/mi
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/chef_metal/machine_spec.rb

@@ -0,0 +1,78 @@
+module ChefMetal
+  #
+  # Specification for a machine. Sufficient information to find and contact it
+  # after it has been set up.
+  #
+  class MachineSpec
+    def initialize(node)
+      @node = node
+    end
+
+    attr_reader :node
+
+    #
+    # Globally unique identifier for this machine. Does not depend on the machine's
+    # location or existence.
+    #
+    def id
+      raise "id unimplemented"
+    end
+
+    #
+    # Name of the machine. Corresponds to the name in "machine 'name' do" ...
+    #
+    def name
+      node['name']
+    end
+
+    #
+    # Location of this machine. This should be a freeform hash, with enough
+    # information for the driver to look it up and create a Machine object to
+    # access it.
+    #
+    # This MUST include a 'driver_url' attribute with the driver's URL in it.
+    #
+    # chef-metal will do its darnedest to not lose this information.
+    #
+    def location
+      metal_attr('location')
+    end
+
+    #
+    # Set the location for this machine.
+    #
+    def location=(value)
+      set_metal_attr('location', value)
+    end
+
+    # URL to the driver.  Convenience for location['driver_url']
+    def driver_url
+      location ? location['driver_url'] : nil
+    end
+
+    #
+    # Save this node to the server.  If you have significant information that
+    # could be lost, you should do this as quickly as possible.  Data will be
+    # saved automatically for you after allocate_machine and ready_machine.
+    #
+    def save(action_handler)
+      raise "save unimplemented"
+    end
+
+    protected
+
+    def metal_attr(attr)
+      if node['normal'] && node['normal']['metal']
+        node['normal']['metal'][attr]
+      else
+        nil
+      end
+    end
+
+    def set_metal_attr(attr, value)
+      node['normal'] ||= {}
+      node['normal']['metal'] ||= {}
+      node['normal']['metal'][attr] = value
+    end
+  end
+end

data/lib/chef_metal/recipe_dsl.rb

@@ -0,0 +1,84 @@
+require 'chef_metal/chef_run_data'
+require 'chef/resource_collection'
+
+require 'chef/resource/machine'
+require 'chef/provider/machine'
+require 'chef/resource/machine_batch'
+require 'chef/provider/machine_batch'
+require 'chef/resource/machine_file'
+require 'chef/provider/machine_file'
+require 'chef/resource/machine_execute'
+require 'chef/provider/machine_execute'
+
+class Chef
+  module DSL
+    module Recipe
+      def with_driver(driver, &block)
+        if driver.is_a?(String)
+          run_context.chef_metal.with_driver(driver, &block)
+        elsif driver.is_a?(ChefMetal::Driver)
+          run_context.chef_metal.with_driver(run_context.chef_metal.driver_for_url(driver), &block)
+        else
+          raise "with_driver accepts either a driver URL string or a ChefMetal::Driver instance. You tried passing a #{driver.class}."
+        end
+      end
+
+      def with_machine_options(machine_options, &block)
+        run_context.chef_metal.with_machine_options(machine_options, &block)
+      end
+
+      def with_machine_batch(the_machine_batch, options = {}, &block)
+        if the_machine_batch.is_a?(String)
+          the_machine_batch = machine_batch the_machine_batch do
+            if options[:action]
+              action options[:action]
+            end
+            if options[:max_simultaneous]
+              max_simultaneous options[:max_simultaneous]
+            end
+          end
+        end
+        run_context.chef_metal.with_machine_batch(the_machine_batch, &block)
+      end
+
+      def current_machine_options
+        run_context.chef_metal.current_machine_options
+      end
+
+      def add_machine_options(options, &block)
+        run_context.chef_metal.add_machine_options(options, &block)
+      end
+
+      # When the machine resource is first declared, create a machine_batch (if there
+      # isn't one already)
+      def machine(name, &block)
+        if !run_context.chef_metal.current_machine_batch
+          run_context.chef_metal.with_machine_batch declare_resource(:machine_batch, 'default', caller[0])
+        end
+        declare_resource(:machine, name, caller[0], &block)
+      end
+    end
+  end
+
+  class Config
+    default(:driver) { ENV['CHEF_DRIVER'] }
+  #   config_context :drivers do
+  #     # each key is a driver_url, and each value can have driver, driver_options and machine_options
+  #     config_strict_mode false
+  #   end
+  #   config_context :driver_options do
+  #     # open ended for whatever the driver wants
+  #     config_strict_mode false
+  #   end
+  #   config_context :machine_options do
+  #     # open ended for whatever the driver wants
+  #     config_strict_mode false
+  #   end
+  end
+
+  class RunContext
+    def chef_metal
+      @chef_metal ||= ChefMetal::ChefRunData.new(config)
+    end
+  end
+end

data/lib/chef_metal/transport.rb

@@ -0,0 +1,87 @@
+require 'timeout'
+
+module ChefMetal
+  class Transport
+    DEFAULT_TIMEOUT = 15*60
+
+    # Execute a program on the remote host.
+    #
+    # == Arguments
+    # command: command to run.  May be a shell-escaped string or a pre-split array containing [PROGRAM, ARG1, ARG2, ...].
+    # options: hash of options, including but not limited to:
+    #          :timeout => NUM_SECONDS - time to wait before program finishes (throws an exception otherwise).  Set to nil or 0 to run with no timeout.  Defaults to 15 minutes.
+    #          :stream => BOOLEAN - true to stream stdout and stderr to the console.
+    #          :stream => BLOCK - block to stream stdout and stderr to (block.call(stdout_chunk, stderr_chunk))
+    #          :stream_stdout => FD - FD to stream stdout to (defaults to IO.stdout)
+    #          :stream_stderr => FD - FD to stream stderr to (defaults to IO.stderr)
+    #          :read_only => BOOLEAN - true if command is guaranteed not to change system state (useful for Docker)
+    def execute(command, options = {})
+      raise "execute not overridden on #{self.class}"
+    end
+
+    def read_file(path)
+      raise "read_file not overridden on #{self.class}"
+    end
+
+    def write_file(path, content)
+      raise "write_file not overridden on #{self.class}"
+    end
+
+    def download_file(path, local_path)
+      IO.write(local_path, read_file(path))
+    end
+
+    def upload_file(local_path, path)
+      write_file(path, IO.read(local_path))
+    end
+
+    def make_url_available_to_remote(local_url)
+      raise "make_url_available_to_remote not overridden on #{self.class}"
+    end
+
+    def disconnect
+      raise "disconnect not overridden on #{self.class}"
+    end
+
+    def available?
+      raise "available? not overridden on #{self.class}"
+    end
+
+    # Config hash, including :log_level and :logger as keys
+    def config
+      raise "config not overridden on #{self.class}"
+    end
+
+    protected
+
+    # Helper to implement stdout/stderr streaming in execute
+    def stream_chunk(options, stdout_chunk, stderr_chunk)
+      if options[:stream].is_a?(Proc)
+        options[:stream].call(stdout_chunk, stderr_chunk)
+      else
+        if stdout_chunk
+          if options[:stream_stdout]
+            options[:stream_stdout].print stdout_chunk
+          elsif options[:stream] || config[:log_level] == :debug
+            STDOUT.print stdout_chunk
+          end
+        end
+        if stderr_chunk
+          if options[:stream_stderr]
+            options[:stream_stderr].print stderr_chunk
+          elsif options[:stream] || config[:log_level] == :debug
+            STDERR.print stderr_chunk
+          end
+        end
+      end
+    end
+
+    def with_execute_timeout(options, &block)
+      Timeout::timeout(execute_timeout(options), &block)
+    end
+
+    def execute_timeout(options)
+      options.has_key?(:timeout) ? options[:timeout] : DEFAULT_TIMEOUT
+    end
+  end
+end

data/lib/chef_metal/transport/ssh.rb

@@ -0,0 +1,235 @@
+require 'chef_metal/transport'
+require 'uri'
+require 'socket'
+require 'timeout'
+require 'net/ssh'
+require 'net/scp'
+require 'net/ssh/gateway'
+
+module ChefMetal
+  class Transport
+    class SSH < ChefMetal::Transport
+      def initialize(host, username, ssh_options, options, global_config)
+        @host = host
+        @username = username
+        @ssh_options = ssh_options
+        @options = options
+        @config = global_config
+      end
+
+      attr_reader :host
+      attr_reader :username
+      attr_reader :ssh_options
+      attr_reader :options
+      attr_reader :config
+
+      def execute(command, execute_options = {})
+        Chef::Log.info("Executing #{options[:prefix]}#{command} on #{username}@#{host}")
+        stdout = ''
+        stderr = ''
+        exitstatus = nil
+        session # grab session outside timeout, it has its own timeout
+        with_execute_timeout(execute_options) do
+          channel = session.open_channel do |channel|
+            # Enable PTY unless otherwise specified, some instances require this
+            unless options[:ssh_pty_enable] == false
+              channel.request_pty do |chan, success|
+                 raise "could not get pty" if !success && options[:ssh_pty_enable]
+              end
+            end
+
+            channel.exec("#{options[:prefix]}#{command}") do |ch, success|
+              raise "could not execute command: #{command.inspect}" unless success
+
+              channel.on_data do |ch2, data|
+                stdout << data
+                stream_chunk(execute_options, data, nil)
+              end
+
+              channel.on_extended_data do |ch2, type, data|
+                stderr << data
+                stream_chunk(execute_options, nil, data)
+              end
+
+              channel.on_request "exit-status" do |ch, data|
+                exitstatus = data.read_long
+              end
+            end
+          end
+
+          channel.wait
+        end
+
+        Chef::Log.info("Completed #{command} on #{username}@#{host}: exit status #{exitstatus}")
+        Chef::Log.debug("Stdout was:\n#{stdout}") if stdout != '' && !options[:stream] && !options[:stream_stdout] && config[:log_level] != :debug
+        Chef::Log.info("Stderr was:\n#{stderr}") if stderr != '' && !options[:stream] && !options[:stream_stderr] && config[:log_level] != :debug
+        SSHResult.new(command, execute_options, stdout, stderr, exitstatus)
+      end
+
+      def read_file(path)
+        Chef::Log.debug("Reading file #{path} from #{username}@#{host}")
+        result = StringIO.new
+        download(path, result)
+        result.string
+      end
+
+      def download_file(path, local_path)
+        Chef::Log.debug("Downloading file #{path} from #{username}@#{host} to local #{local_path}")
+        download(path, local_path)
+      end
+
+      def write_file(path, content)
+        execute("mkdir -p #{File.dirname(path)}").error!
+        if options[:prefix]
+          # Make a tempfile on the other side, upload to that, and sudo mv / chown / etc.
+          remote_tempfile = "/tmp/#{File.basename(path)}.#{Random.rand(2**32)}"
+          Chef::Log.debug("Writing #{content.length} bytes to #{remote_tempfile} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(StringIO.new(content), remote_tempfile)
+          execute("mv #{remote_tempfile} #{path}").error!
+        else
+          Chef::Log.debug("Writing #{content.length} bytes to #{path} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(StringIO.new(content), path)
+        end
+      end
+
+      def upload_file(local_path, path)
+        execute("mkdir -p #{File.dirname(path)}").error!
+        if options[:prefix]
+          # Make a tempfile on the other side, upload to that, and sudo mv / chown / etc.
+          remote_tempfile = "/tmp/#{File.basename(path)}.#{Random.rand(2**32)}"
+          Chef::Log.debug("Uploading #{local_path} to #{remote_tempfile} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(local_path, remote_tempfile)
+          execute("mv #{remote_tempfile} #{path}").error!
+        else
+          Chef::Log.debug("Uploading #{local_path} to #{path} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(local_path, path)
+        end
+      end
+
+      def make_url_available_to_remote(local_url)
+        uri = URI(local_url)
+        host = Socket.getaddrinfo(uri.host, uri.scheme, nil, :STREAM)[0][3]
+        if host == '127.0.0.1' || host == '[::1]'
+          unless session.forward.active_remotes.any? { |port, bind| port == uri.port && bind == '127.0.0.1' }
+            # TODO IPv6
+            Chef::Log.debug("Forwarding local server 127.0.0.1:#{uri.port} to port #{uri.port} on #{username}@#{host}")
+            session.forward.remote(uri.port, '127.0.0.1', uri.port)
+          end
+        end
+        local_url
+      end
+
+      def disconnect
+        if @session
+          begin
+            Chef::Log.debug("Closing SSH session on #{username}@#{host}")
+            @session.close
+          rescue
+          end
+          @session = nil
+        end
+      end
+
+      def available?
+        # If you can't pwd within 10 seconds, you can't pwd
+        execute('pwd', :timeout => 10)
+        true
+      rescue Timeout::Error, Errno::EHOSTUNREACH, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ECONNRESET, Net::SSH::Disconnect
+        Chef::Log.debug("#{username}@#{host} unavailable: network connection failed or broke: #{$!.inspect}")
+        false
+      rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch
+        Chef::Log.debug("#{username}@#{host} unavailable: SSH authentication error: #{$!.inspect} ")
+        false
+      end
+
+      protected
+
+      def gateway?
+        options.key?(:ssh_gateway) and ! options[:ssh_gateway].nil?
+      end
+
+      def gateway
+        @gateway ||= begin
+          gw_host, gw_user = options[:ssh_gateway].split('@').reverse
+          gw_host, gw_port = gw_host.split(':')
+          gw_user = ssh_options[:ssh_username] unless gw_user
+
+          ssh_start_opts = { timeout:10 }.merge(ssh_options)
+          ssh_start_opts[:port] = gw_port || 22
+
+          Chef::Log.debug("Opening SSH gateway to #{gw_user}@#{gw_host} with options #{ssh_start_opts.inspect}")
+          begin
+            Net::SSH::Gateway.new(gw_host, gw_user, ssh_start_opts)
+          rescue Errno::ETIMEDOUT
+            Chef::Log.debug("Timed out connecting to gateway: #{$!}")
+            raise InitialConnectTimeout.new($!)
+          end
+        end
+      end
+
+      def session
+        @session ||= begin
+          ssh_start_opts = { timeout:10 }.merge(ssh_options)
+          Chef::Log.debug("Opening SSH connection to #{username}@#{host} with options #{ssh_start_opts.inspect}")
+          # Small initial connection timeout (10s) to help us fail faster when server is just dead
+          begin
+            if gateway? then gateway.ssh(host, username, ssh_start_opts)
+            else Net::SSH.start(host, username, ssh_start_opts)
+            end
+          rescue Timeout::Error
+            Chef::Log.debug("Timed out connecting to SSH: #{$!}")
+            raise InitialConnectTimeout.new($!)
+          end
+        end
+      end
+
+      def download(path, local_path)
+        channel = Net::SCP.new(session).download(path, local_path)
+        begin
+          channel.wait
+        rescue Net::SCP::Error => e
+          # TODO we need a way to distinguish between "directory of file does not exist" and "SCP did not finish successfully"
+          nil
+        # ensure the channel is closed when a rescue happens above
+        ensure
+          channel.close
+          channel.wait
+        end
+        nil
+      end
+
+      class SSHResult
+        def initialize(command, options, stdout, stderr, exitstatus)
+          @command = command
+          @options = options
+          @stdout = stdout
+          @stderr = stderr
+          @exitstatus = exitstatus
+        end
+
+        attr_reader :command
+        attr_reader :options
+        attr_reader :stdout
+        attr_reader :stderr
+        attr_reader :exitstatus
+
+        def error!
+          if exitstatus != 0
+            # TODO stdout/stderr is already printed at info/debug level.  Let's not print it twice, it's a lot.
+            msg = "Error: command '#{command}' exited with code #{exitstatus}.\n"
+            raise msg
+          end
+        end
+      end
+
+      class InitialConnectTimeout < Timeout::Error
+        def initialize(original_error)
+          super(original_error.message)
+          @original_error = original_error
+        end
+
+        attr_reader :original_error
+      end
+    end
+  end
+end