chef-provisioning 0.15

data/lib/chef/provisioning/machine/windows_machine.rb

@@ -0,0 +1,104 @@
+require 'chef/provisioning/machine/basic_machine'
+
+class Chef
+module Provisioning
+  class Machine
+    class WindowsMachine < BasicMachine
+      def initialize(machine_spec, transport, convergence_strategy)
+        super
+      end
+
+      # Options include:
+      #
+      # command_prefix - prefix to put in front of any command, e.g. sudo
+      attr_reader :options
+
+      # Delete file
+      def delete_file(action_handler, path)
+        if file_exists?(path)
+          action_handler.perform_action "delete file #{escape(path)} on #{machine_spec.name}" do
+            transport.execute("Remove-Item #{escape(path)}").error!
+          end
+        end
+      end
+
+      def is_directory?(path)
+        parse_boolean(transport.execute("Test-Path #{escape(path)} -pathtype container", :read_only => true).stdout)
+      end
+
+      # Return true or false depending on whether file exists
+      def file_exists?(path)
+        parse_boolean(transport.execute("Test-Path #{escape(path)}", :read_only => true).stdout)
+      end
+
+      def files_different?(path, local_path, content=nil)
+        if !file_exists?(path) || (local_path && !File.exists?(local_path))
+          return true
+        end
+
+        # Get remote checksum of file (from http://stackoverflow.com/a/13926809)
+        result = transport.execute(<<-EOM, :read_only => true)
+$md5 = [System.Security.Cryptography.MD5]::Create("MD5")
+$fd = [System.IO.File]::OpenRead(#{path.inspect})
+$buf = new-object byte[] (1024*1024*8) # 8mb buffer
+while (($read_len = $fd.Read($buf,0,$buf.length)) -eq $buf.length){
+    $total += $buf.length
+    $md5.TransformBlock($buf,$offset,$buf.length,$buf,$offset)
+}
+# finalize the last read
+$md5.TransformFinalBlock($buf,0,$read_len)
+$hash = $md5.Hash
+# convert hash bytes to hex formatted string
+$hash | foreach { $hash_txt += $_.ToString("x2") }
+$hash_txt
+EOM
+        result.error!
+        remote_sum = result.stdout.split(' ')[0]
+        digest = Digest::SHA256.new
+        if content
+          digest.update(content)
+        else
+          File.open(local_path, 'rb') do |io|
+            while (buf = io.read(4096)) && buf.length > 0
+              digest.update(buf)
+            end
+          end
+        end
+        remote_sum != digest.hexdigest
+      end
+
+      def create_dir(action_handler, path)
+        if !file_exists?(path)
+          action_handler.perform_action "create directory #{path} on #{machine_spec.name}" do
+            transport.execute("New-Item #{escape(path)} -Type directory")
+          end
+        end
+      end
+
+      # Set file attributes { :owner, :group, :rights }
+#      def set_attributes(action_handler, path, attributes)
+#      end
+
+      # Get file attributes { :owner, :group, :rights }
+#      def get_attributes(path)
+#      end
+
+      def dirname_on_machine(path)
+        path.split(/[\\\/]/)[0..-2].join('\\')
+      end
+
+      def escape(string)
+        transport.escape(string)
+      end
+
+      def parse_boolean(string)
+        if string =~ /^\s*true\s*$/mi
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end
+end

data/lib/chef/provisioning/machine_spec.rb

@@ -0,0 +1,82 @@
+class Chef
+module Provisioning
+  #
+  # Specification for a machine. Sufficient information to find and contact it
+  # after it has been set up.
+  #
+  class MachineSpec
+    def initialize(node)
+      @node = node
+      # Upgrade from metal to chef_provisioning ASAP.
+      if node['normal'] && !node['normal']['chef_provisioning'] && node['normal']['metal']
+        node['normal']['chef_provisioning'] = node['normal'].delete('metal')
+      end
+    end
+
+    attr_reader :node
+
+    #
+    # Globally unique identifier for this machine. Does not depend on the machine's
+    # location or existence.
+    #
+    def id
+      raise "id unimplemented"
+    end
+
+    #
+    # Name of the machine. Corresponds to the name in "machine 'name' do" ...
+    #
+    def name
+      node['name']
+    end
+
+    #
+    # Location of this machine. This should be a freeform hash, with enough
+    # information for the driver to look it up and create a Machine object to
+    # access it.
+    #
+    # This MUST include a 'driver_url' attribute with the driver's URL in it.
+    #
+    # chef-provisioning will do its darnedest to not lose this information.
+    #
+    def location
+      chef_provisioning_attr('location')
+    end
+
+    #
+    # Set the location for this machine.
+    #
+    def location=(value)
+      set_chef_provisioning_attr('location', value)
+    end
+
+    # URL to the driver.  Convenience for location['driver_url']
+    def driver_url
+      location ? location['driver_url'] : nil
+    end
+
+    #
+    # Save this node to the server.  If you have significant information that
+    # could be lost, you should do this as quickly as possible.  Data will be
+    # saved automatically for you after allocate_machine and ready_machine.
+    #
+    def save(action_handler)
+      raise "save unimplemented"
+    end
+
+    protected
+
+    def chef_provisioning_attr(attr)
+      if node['normal'] && node['normal']['chef_provisioning']
+        node['normal']['chef_provisioning'][attr]
+      end
+    end
+
+    def set_chef_provisioning_attr(attr, value)
+      node['normal'] ||= {}
+      node['normal']['chef_provisioning'] ||= {}
+      node['normal']['chef_provisioning'][attr] = value
+    end
+  end
+end
+end

data/lib/chef/provisioning/recipe_dsl.rb

@@ -0,0 +1,103 @@
+require 'chef/provisioning/chef_run_data'
+require 'chef/resource_collection'
+require 'chef/resource/chef_data_bag_resource'
+
+require 'chef/resource/machine'
+require 'chef/provider/machine'
+require 'chef/resource/machine_batch'
+require 'chef/provider/machine_batch'
+require 'chef/resource/machine_file'
+require 'chef/provider/machine_file'
+require 'chef/resource/machine_execute'
+require 'chef/provider/machine_execute'
+require 'chef/resource/machine_image'
+require 'chef/provider/machine_image'
+require 'chef/resource/load_balancer'
+require 'chef/provider/load_balancer'
+
+class Chef
+  module DSL
+    module Recipe
+
+      def with_data_center(data_center, &block)
+        run_context.chef_metal.with_data_center(data_center, &block)
+      end
+
+      def with_driver(driver, options = nil, &block)
+        run_context.chef_provisioning.with_driver(driver, options, &block)
+      end
+
+      def with_machine_options(machine_options, &block)
+        run_context.chef_provisioning.with_machine_options(machine_options, &block)
+      end
+
+      def current_machine_options
+        run_context.chef_provisioning.current_machine_options
+      end
+
+      def add_machine_options(options, &block)
+        run_context.chef_provisioning.add_machine_options(options, &block)
+      end
+
+      def with_image_options(image_options, &block)
+        run_context.chef_provisioning.with_image_options(image_options, &block)
+      end
+
+      def current_image_options
+        run_context.chef_provisioning.current_image_options
+      end
+
+      NOT_PASSED = Object.new
+
+      @@next_machine_batch_index = 0
+
+      def machine_batch_default_name
+        @@next_machine_batch_index += 1
+        if @@next_machine_batch_index > 1
+          "default#{@@next_machine_batch_index}"
+        else
+          "default"
+        end
+      end
+
+      def machine_batch(name = nil, &block)
+        name ||= machine_batch_default_name
+        recipe = self
+        declare_resource(:machine_batch, name, caller[0]) do
+          from_recipe recipe
+          instance_eval(&block)
+        end
+      end
+
+    end
+  end
+
+  class Config
+    default(:driver) { ENV['CHEF_DRIVER'] }
+  #   config_context :drivers do
+  #     # each key is a driver_url, and each value can have driver, driver_options and machine_options
+  #     config_strict_mode false
+  #   end
+  #   config_context :driver_options do
+  #     # open ended for whatever the driver wants
+  #     config_strict_mode false
+  #   end
+  #   config_context :machine_options do
+  #     # open ended for whatever the driver wants
+  #     config_strict_mode false
+  #   end
+  end
+
+  class RunContext
+    def chef_provisioning
+      @chef_provisioning ||= Chef::Provisioning::ChefRunData.new(config)
+    end
+    alias :chef_metal :chef_provisioning
+  end
+
+  class ResourceCollection
+    def previous_index
+      @insert_after_idx ? @insert_after_idx : @resources.length - 1
+    end
+  end
+end

data/lib/chef/provisioning/transport.rb

@@ -0,0 +1,95 @@
+require 'timeout'
+
+class Chef
+module Provisioning
+  class Transport
+    DEFAULT_TIMEOUT = 15*60
+
+    # Execute a program on the remote host.
+    #
+    # == Arguments
+    # command: command to run.  May be a shell-escaped string or a pre-split
+    #          array containing [PROGRAM, ARG1, ARG2, ...].
+    # options: hash of options, including but not limited to:
+    #          :timeout => NUM_SECONDS - time to wait before program finishes
+    #                      (throws an exception otherwise).  Set to nil or 0 to
+    #                      run with no timeout.  Defaults to 15 minutes.
+    #          :stream => BOOLEAN - true to stream stdout and stderr to the console.
+    #          :stream => BLOCK - block to stream stdout and stderr to
+    #                     (block.call(stdout_chunk, stderr_chunk))
+    #          :stream_stdout => FD - FD to stream stdout to (defaults to IO.stdout)
+    #          :stream_stderr => FD - FD to stream stderr to (defaults to IO.stderr)
+    #          :read_only => BOOLEAN - true if command is guaranteed not to
+    #                        change system state (useful for Docker)
+    def execute(command, options = {})
+      raise "execute not overridden on #{self.class}"
+    end
+
+    # TODO: make exceptions for these instead of just returning nil / silently failing
+    def read_file(path)
+      raise "read_file not overridden on #{self.class}"
+    end
+
+    def write_file(path, content)
+      raise "write_file not overridden on #{self.class}"
+    end
+
+    def download_file(path, local_path)
+      IO.write(local_path, read_file(path))
+    end
+
+    def upload_file(local_path, path)
+      write_file(path, IO.read(local_path))
+    end
+
+    def make_url_available_to_remote(local_url)
+      raise "make_url_available_to_remote not overridden on #{self.class}"
+    end
+
+    def disconnect
+      raise "disconnect not overridden on #{self.class}"
+    end
+
+    def available?
+      raise "available? not overridden on #{self.class}"
+    end
+
+    # Config hash, including :log_level and :logger as keys
+    def config
+      raise "config not overridden on #{self.class}"
+    end
+
+    protected
+
+    # Helper to implement stdout/stderr streaming in execute
+    def stream_chunk(options, stdout_chunk, stderr_chunk)
+      if options[:stream].is_a?(Proc)
+        options[:stream].call(stdout_chunk, stderr_chunk)
+      else
+        if stdout_chunk
+          if options[:stream_stdout]
+            options[:stream_stdout].print stdout_chunk
+          elsif options[:stream] || config[:log_level] == :debug
+            STDOUT.print stdout_chunk
+          end
+        end
+        if stderr_chunk
+          if options[:stream_stderr]
+            options[:stream_stderr].print stderr_chunk
+          elsif options[:stream] || config[:log_level] == :debug
+            STDERR.print stderr_chunk
+          end
+        end
+      end
+    end
+
+    def with_execute_timeout(options, &block)
+      Timeout::timeout(execute_timeout(options), &block)
+    end
+
+    def execute_timeout(options)
+      options.has_key?(:timeout) ? options[:timeout] : DEFAULT_TIMEOUT
+    end
+  end
+end
+end

data/lib/chef/provisioning/transport/ssh.rb

@@ -0,0 +1,343 @@
+require 'chef/provisioning/transport'
+require 'chef/log'
+require 'uri'
+require 'socket'
+require 'timeout'
+require 'net/ssh'
+require 'net/scp'
+require 'net/ssh/gateway'
+
+class Chef
+module Provisioning
+  class Transport
+    class SSH < Chef::Provisioning::Transport
+      #
+      # Create a new SSH transport.
+      #
+      # == Arguments
+      #
+      # - host: the host to connect to, e.g. '145.14.51.45'
+      # - username: the username to connect with
+      # - ssh_options: a list of options to Net::SSH.start
+      # - options: a hash of options for the transport itself, including:
+      #   - :prefix: a prefix to send before each command (e.g. "sudo ")
+      #   - :ssh_pty_enable: set to false to disable pty (some instances don't
+      #     support this, most do)
+      #   - :ssh_gateway: the gateway to use, e.g. "jkeiser@145.14.51.45:222".
+      #     nil (the default) means no gateway.
+      # - global_config: an options hash that looks suspiciously similar to
+      #   Chef::Config, containing at least the key :log_level.
+      #
+      # The options are used in
+      #   Net::SSH.start(host, username, ssh_options)
+
+      def initialize(host, username, ssh_options, options, global_config)
+        @host = host
+        @username = username
+        @ssh_options = ssh_options
+        @options = options
+        @config = global_config
+      end
+
+      attr_reader :host
+      attr_reader :username
+      attr_reader :ssh_options
+      attr_reader :options
+      attr_reader :config
+
+      def execute(command, execute_options = {})
+        Chef::Log.info("Executing #{options[:prefix]}#{command} on #{username}@#{host}")
+        stdout = ''
+        stderr = ''
+        exitstatus = nil
+        session # grab session outside timeout, it has its own timeout
+        with_execute_timeout(execute_options) do
+          channel = session.open_channel do |channel|
+            # Enable PTY unless otherwise specified, some instances require this
+            unless options[:ssh_pty_enable] == false
+              channel.request_pty do |chan, success|
+                 raise "could not get pty" if !success && options[:ssh_pty_enable]
+              end
+            end
+
+            channel.exec("#{options[:prefix]}#{command}") do |ch, success|
+              raise "could not execute command: #{command.inspect}" unless success
+
+              channel.on_data do |ch2, data|
+                stdout << data
+                stream_chunk(execute_options, data, nil)
+              end
+
+              channel.on_extended_data do |ch2, type, data|
+                stderr << data
+                stream_chunk(execute_options, nil, data)
+              end
+
+              channel.on_request "exit-status" do |ch, data|
+                exitstatus = data.read_long
+              end
+            end
+          end
+
+          channel.wait
+        end
+
+        Chef::Log.info("Completed #{command} on #{username}@#{host}: exit status #{exitstatus}")
+        Chef::Log.debug("Stdout was:\n#{stdout}") if stdout != '' && !options[:stream] && !options[:stream_stdout] && config[:log_level] != :debug
+        Chef::Log.info("Stderr was:\n#{stderr}") if stderr != '' && !options[:stream] && !options[:stream_stderr] && config[:log_level] != :debug
+        SSHResult.new(command, execute_options, stdout, stderr, exitstatus)
+      end
+
+      def read_file(path)
+        Chef::Log.debug("Reading file #{path} from #{username}@#{host}")
+        result = StringIO.new
+        download(path, result)
+        result.string
+      end
+
+      def download_file(path, local_path)
+        Chef::Log.debug("Downloading file #{path} from #{username}@#{host} to local #{local_path}")
+        download(path, local_path)
+      end
+
+      def write_file(path, content)
+        execute("mkdir -p #{File.dirname(path)}").error!
+        if options[:prefix]
+          # Make a tempfile on the other side, upload to that, and sudo mv / chown / etc.
+          remote_tempfile = "/tmp/#{File.basename(path)}.#{Random.rand(2**32)}"
+          Chef::Log.debug("Writing #{content.length} bytes to #{remote_tempfile} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(StringIO.new(content), remote_tempfile)
+          execute("mv #{remote_tempfile} #{path}").error!
+        else
+          Chef::Log.debug("Writing #{content.length} bytes to #{path} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(StringIO.new(content), path)
+        end
+      end
+
+      def upload_file(local_path, path)
+        execute("mkdir -p #{File.dirname(path)}").error!
+        if options[:prefix]
+          # Make a tempfile on the other side, upload to that, and sudo mv / chown / etc.
+          remote_tempfile = "/tmp/#{File.basename(path)}.#{Random.rand(2**32)}"
+          Chef::Log.debug("Uploading #{local_path} to #{remote_tempfile} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(local_path, remote_tempfile)
+          begin
+            execute("mv #{remote_tempfile} #{path}").error!
+          rescue
+            # Clean up if we were unable to move
+            execute("rm #{remote_tempfile}").error!
+          end
+        else
+          Chef::Log.debug("Uploading #{local_path} to #{path} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(local_path, path)
+        end
+      end
+
+      def make_url_available_to_remote(local_url)
+        uri = URI(local_url)
+        if is_local_machine(uri.host)
+          port, host = forward_port(uri.port, uri.host, uri.port, 'localhost')
+          if !port
+            # Try harder if the port is already taken
+            port, host = forward_port(uri.port, uri.host, 0, 'localhost')
+            if !port
+              raise "Error forwarding port: could not forward #{uri.port} or 0"
+            end
+          end
+          uri.host = host
+          uri.port = port
+        end
+        Chef::Log.info("Port forwarded: local URL #{local_url} is available to #{self.host} as #{uri.to_s} for the duration of this SSH connection.")
+        uri.to_s
+      end
+
+      def disconnect
+        if @session
+          begin
+            Chef::Log.debug("Closing SSH session on #{username}@#{host}")
+            @session.close
+          rescue
+          ensure
+            @session = nil
+          end
+        end
+      end
+
+      def available?
+        # If you can't pwd within 10 seconds, you can't pwd
+        execute('pwd', :timeout => 10)
+        true
+      rescue Timeout::Error, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::EHOSTDOWN, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ECONNRESET, Net::SSH::Disconnect
+        Chef::Log.debug("#{username}@#{host} unavailable: network connection failed or broke: #{$!.inspect}")
+        disconnect
+        false
+      rescue Net::SSH::AuthenticationFailed, Net::SSH::HostKeyMismatch
+        Chef::Log.debug("#{username}@#{host} unavailable: SSH authentication error: #{$!.inspect} ")
+        disconnect
+        false
+      end
+
+      protected
+
+      def session
+        @session ||= begin
+          ssh_start_opts = { timeout:10 }.merge(ssh_options)
+          Chef::Log.debug("Opening SSH connection to #{username}@#{host} with options #{ssh_start_opts.inspect}")
+          # Small initial connection timeout (10s) to help us fail faster when server is just dead
+          begin
+            if gateway? then gateway.ssh(host, username, ssh_start_opts)
+            else Net::SSH.start(host, username, ssh_start_opts)
+            end
+          rescue Timeout::Error
+            Chef::Log.debug("Timed out connecting to SSH: #{$!}")
+            raise InitialConnectTimeout.new($!)
+          end
+        end
+      end
+
+      def download(path, local_path)
+        if options[:prefix]
+          # Make a tempfile on the other side, upload to that, and sudo mv / chown / etc.
+          remote_tempfile = "/tmp/#{File.basename(path)}.#{Random.rand(2**32)}"
+          Chef::Log.debug("Downloading #{path} from #{remote_tempfile} to #{local_path} on #{username}@#{host}")
+          begin
+            execute("cp #{path} #{remote_tempfile}").error!
+            execute("chown #{username} #{remote_tempfile}").error!
+            do_download remote_tempfile, local_path
+          rescue => e
+              Chef::Log.error "Unable to download #{path} to #{local_path} on #{username}@#{host} -- #{e}"
+              nil
+          ensure
+            # Clean up afterwards
+            begin
+              execute("rm #{remote_tempfile}").error!
+            rescue => e
+              Chef::Log.warn "Unable to clean up #{remote_tempfile} on #{username}@#{host} -- #{e}"
+            end
+          end
+        else
+          do_download path, local_path
+        end
+      end
+
+      def do_download(path, local_path)
+        channel = Net::SCP.new(session).download(path, local_path)
+        begin
+          channel.wait
+          Chef::Log.debug "SCP completed for: #{path} to #{local_path}"
+        rescue Net::SCP::Error => e
+          Chef::Log.error "Error with SCP: #{e}"
+          # TODO we need a way to distinguish between "directory or file does not exist" and "SCP did not finish successfully"
+          nil
+        ensure
+          # ensure the channel is closed
+          channel.close
+          channel.wait
+        end
+
+        nil
+      end
+
+      class SSHResult
+        def initialize(command, options, stdout, stderr, exitstatus)
+          @command = command
+          @options = options
+          @stdout = stdout
+          @stderr = stderr
+          @exitstatus = exitstatus
+        end
+
+        attr_reader :command
+        attr_reader :options
+        attr_reader :stdout
+        attr_reader :stderr
+        attr_reader :exitstatus
+
+        def error!
+          if exitstatus != 0
+            # TODO stdout/stderr is already printed at info/debug level.  Let's not print it twice, it's a lot.
+            msg = "Error: command '#{command}' exited with code #{exitstatus}.\n"
+            raise msg
+          end
+        end
+      end
+
+      class InitialConnectTimeout < Timeout::Error
+        def initialize(original_error)
+          super(original_error.message)
+          @original_error = original_error
+        end
+
+        attr_reader :original_error
+      end
+
+      private
+
+      def gateway?
+        options.key?(:ssh_gateway) and ! options[:ssh_gateway].nil?
+      end
+
+      def gateway
+        gw_user, gw_host = options[:ssh_gateway].split('@')
+        gw_host, gw_port = gw_host.split(':')
+        gw_user = ssh_options[:ssh_username] unless gw_user
+
+        ssh_start_opts = { timeout:10 }.merge(ssh_options)
+        ssh_start_opts[:port] = gw_port || 22
+
+        Chef::Log.debug("Opening SSH gateway to #{gw_user}@#{gw_host} with options #{ssh_start_opts.inspect}")
+        begin
+          Net::SSH::Gateway.new(gw_host, gw_user, ssh_start_opts)
+        rescue Errno::ETIMEDOUT
+          Chef::Log.debug("Timed out connecting to gateway: #{$!}")
+          raise InitialConnectTimeout.new($!)
+        end
+      end
+
+      def is_local_machine(host)
+        local_addrs = Socket.ip_address_list
+        host_addrs = Addrinfo.getaddrinfo(host, nil)
+        local_addrs.any? do |local_addr|
+          host_addrs.any? do |host_addr|
+            local_addr.ip_address == host_addr.ip_address
+          end
+        end
+      end
+
+      # Forwards a port over the connection, and returns the
+      def forward_port(local_port, local_host, remote_port, remote_host)
+        # This bit is from the documentation.
+        if session.forward.respond_to?(:active_remote_destinations)
+          got_remote_port, remote_host = session.forward.active_remote_destinations[[local_port, local_host]]
+          if !got_remote_port
+            Chef::Log.debug("Forwarding local server #{local_host}:#{local_port} to #{username}@#{self.host}")
+
+            session.forward.remote(local_port, local_host, remote_port, remote_host) do |actual_remote_port|
+              got_remote_port = actual_remote_port || :error
+              :no_exception # I'll take care of it myself, thanks
+            end
+            # Kick SSH until we get a response
+            session.loop { !got_remote_port }
+            if got_remote_port == :error
+              return nil
+            end
+          end
+          [ got_remote_port, remote_host ]
+        else
+          @forwarded_ports ||= {}
+          remote_port, remote_host = @forwarded_ports[[local_port, local_host]]
+          if !remote_port
+            Chef::Log.debug("Forwarding local server #{local_host}:#{local_port} to #{username}@#{self.host}")
+            old_active_remotes = session.forward.active_remotes
+            session.forward.remote(local_port, local_host, local_port)
+            session.loop { !(session.forward.active_remotes.length > old_active_remotes.length) }
+            remote_port, remote_host = (session.forward.active_remotes - old_active_remotes).first
+            @forwarded_ports[[local_port, local_host]] = [ remote_port, remote_host ]
+          end
+          [ remote_port, remote_host ]
+        end
+      end
+    end
+  end
+end
+end