chef-metal 0.1

data/lib/chef_metal/provisioner/vagrant_provisioner.rb

@@ -0,0 +1,327 @@
+require 'chef/mixin/shell_out'
+require 'chef_metal/provisioner'
+
+module ChefMetal
+  class Provisioner
+
+    # Provisions machines in vagrant.
+    class VagrantProvisioner < Provisioner
+
+      include Chef::Mixin::ShellOut
+
+      # Create a new vagrant provisioner.
+      #
+      # ## Parameters
+      # cluster_path - path to the directory containing the vagrant files, which
+      #                should have been created with the vagrant_cluster resource.
+      def initialize(cluster_path)
+        @cluster_path = cluster_path
+      end
+
+      attr_reader :cluster_path
+
+      # Acquire a machine, generally by provisioning it.  Returns a Machine
+      # object pointing at the machine, allowing useful actions like setup,
+      # converge, execute, file and directory.  The Machine object will have a
+      # "node" property which must be saved to the server (if it is any
+      # different from the original node object).
+      #
+      # ## Parameters
+      # provider - the provider object that is calling this method.
+      # node - node object (deserialized json) representing this machine.  If
+      #        the node has a provisioner_options hash in it, these will be used
+      #        instead of options provided by the provisioner.  TODO compare and
+      #        fail if different?
+      #        node will have node['normal']['provisioner_options'] in it with any options.
+      #        It is a hash with this format:
+      #
+      #           -- provisioner_url: vagrant:<cluster_path>
+      #           -- vagrant_options: hash of properties of the "config"
+      #              object, i.e. "vm.box" => "ubuntu12" and "vm.box_url"
+      #           -- vagrant_config: string containing other vagrant config.
+      #              Should assume the variable "config" represents machine config.
+      #              Will be written verbatim into the vm's Vagrantfile.
+      #           -- transport_options: hash of options specifying the transport.
+      #                :type => :ssh
+      #                :type => :winrm
+      #                If not specified, ssh is used unless vm.guest is :windows.  If that is
+      #                the case, the windows options are used and the port forward for 5985
+      #                is detected.
+      #           -- up_timeout: maximum time, in seconds, to wait for vagrant
+      #              to bring up the machine.  Defaults to 10 minutes.
+      #
+      #        node['normal']['provisioner_output'] will be populated with information
+      #        about the created machine.  For vagrant, it is a hash with this
+      #        format:
+      #
+      #           -- provisioner_url: vagrant_cluster://<current_node>/<cluster_path>
+      #           -- vm_name: name of vagrant vm created
+      #           -- vm_file_path: path to machine-specific vagrant config file
+      #              on disk
+      #           -- forwarded_ports: hash with key as guest_port => host_port
+      #
+      def acquire_machine(provider, node)
+        # Set up the modified node data
+        provisioner_options = node['normal']['provisioner_options']
+        vm_name = node['name']
+        old_provisioner_output = node['normal']['provisioner_output']
+        node['normal']['provisioner_output'] = provisioner_output = {
+          'provisioner_url' => provisioner_url(provider),
+          'vm_name' => vm_name,
+          'vm_file_path' => File.join(cluster_path, "#{vm_name}.vm")
+        }
+        # Preserve existing forwarded ports
+        provisioner_output['forwarded_ports'] = old_provisioner_output['forwarded_ports'] if old_provisioner_output
+
+        # TODO compare new options to existing and fail if we cannot change it
+        # over (perhaps introduce a boolean that will force a delete and recreate
+        # in such a case)
+
+        # Determine contents of vm file
+        vm_file_content = "Vagrant.configure('2') do |outer_config|\n"
+        vm_file_content << "  outer_config.vm.define #{vm_name.inspect} do |config|\n"
+        merged_vagrant_options = { 'vm.hostname' => node['name'] }
+        merged_vagrant_options.merge!(provisioner_options['vagrant_options']) if provisioner_options['vagrant_options']
+        merged_vagrant_options.each_pair do |key, value|
+          vm_file_content << "    config.#{key} = #{value.inspect}\n"
+        end
+        vm_file_content << provisioner_options['vagrant_config'] if provisioner_options['vagrant_config']
+        vm_file_content << "  end\nend\n"
+
+        # Set up vagrant file
+        vm_file = ChefMetal.inline_resource(provider) do
+          file provisioner_output['vm_file_path'] do
+            content vm_file_content
+            action :create
+          end
+        end
+
+        # Check current status of vm
+        current_status = vagrant_status(vm_name)
+        up_timeout = provisioner_options['up_timeout'] || 10*60
+
+        if current_status != 'running'
+          # Run vagrant up if vm is not running
+          provider.converge_by "run vagrant up #{vm_name} (status was '#{current_status}')" do
+            result = shell_out("vagrant up #{vm_name}", :cwd => cluster_path, :timeout => up_timeout)
+            if result.exitstatus != 0
+              raise "vagrant up #{vm_name} failed!\nSTDOUT:#{result.stdout}\nSTDERR:#{result.stderr}"
+            end
+            parse_vagrant_up(result.stdout, node)
+          end
+        elsif vm_file.updated_by_last_action?
+          # Run vagrant reload if vm is running and vm file changed
+          provider.converge_by "run vagrant reload #{vm_name}" do
+            result = shell_out("vagrant reload #{vm_name}", :cwd => cluster_path, :timeout => up_timeout)
+            if result.exitstatus != 0
+              raise "vagrant reload #{vm_name} failed!\nSTDOUT:#{result.stdout}\nSTDERR:#{result.stderr}"
+            end
+            parse_vagrant_up(result.stdout, node)
+          end
+        end
+
+        # Create machine object for callers to use
+        machine_for(node)
+      end
+
+      # Connect to machine without acquiring it
+      def connect_to_machine(node)
+        machine_for(node)
+      end
+
+      def delete_machine(provider, node)
+        if node['normal'] && node['normal']['provisioner_output']
+          provisioner_output = node['normal']['provisioner_output']
+        else
+          provisioner_output = {}
+        end
+        vm_name = provisioner_output['vm_name'] || node['name']
+        current_status = vagrant_status(vm_name)
+        if current_status != 'not created'
+          provider.converge_by "run vagrant destroy -f #{vm_name} (status was '#{current_status}')" do
+            result = shell_out("vagrant destroy -f #{vm_name}", :cwd => cluster_path)
+            if result.exitstatus != 0
+              raise "vagrant destroy failed!\nSTDOUT:#{result.stdout}\nSTDERR:#{result.stderr}"
+            end
+          end
+        end
+
+        convergence_strategy_for(node).delete_chef_objects(provider, node)
+
+        vm_file_path = provisioner_output['vm_file_path'] || File.join(cluster_path, "#{vm_name}.vm")
+        ChefMetal.inline_resource(provider) do
+          file vm_file_path do
+            action :delete
+          end
+        end
+      end
+
+      def stop_machine(provider, node)
+        if node['normal'] && node['normal']['provisioner_output']
+          provisioner_output = node['normal']['provisioner_output']
+        else
+          provisioner_output = {}
+        end
+        vm_name = provisioner_output['vm_name'] || node['name']
+        current_status = vagrant_status(vm_name)
+        if current_status == 'running'
+          provider.converge_by "run vagrant halt #{vm_name} (status was '#{current_status}')" do
+            result = shell_out("vagrant halt #{vm_name}", :cwd => cluster_path)
+            if result.exitstatus != 0
+              raise "vagrant halt failed!\nSTDOUT:#{result.stdout}\nSTDERR:#{result.stderr}"
+            end
+          end
+        end
+      end
+
+
+      # Used by vagrant_cluster and machine to get the string used to configure vagrant
+      def self.vagrant_config_string(vagrant_config, variable, line_prefix)
+        hostname = name.gsub(/[^A-Za-z0-9\-]/, '-')
+
+        result = ''
+        vagrant_config.each_pair do |key, value|
+          result += "#{line_prefix}#{variable}.#{key} = #{value.inspect}\n"
+        end
+        result
+      end
+
+      protected
+
+      def provisioner_url(provider)
+        "vagrant_cluster://#{provider.node['name']}#{cluster_path}"
+      end
+
+      def parse_vagrant_up(output, node)
+        # Grab forwarded port info
+        in_forwarding_ports = false
+        output.lines.each do |line|
+          if in_forwarding_ports
+            if line =~ /-- (\d+) => (\d+)/
+              node['normal']['provisioner_output']['forwarded_ports'][$1] = $2
+            else
+              in_forwarding_ports = false
+            end
+          elsif line =~ /Forwarding ports...$/
+            node['normal']['provisioner_output']['forwarded_ports'] = {}
+            in_forwarding_ports = true
+          end
+        end
+      end
+
+      def machine_for(node)
+        if vagrant_option(node, 'vm.guest').to_s == 'windows'
+          require 'chef_metal/machine/windows_machine'
+          ChefMetal::Machine::WindowsMachine.new(node, transport_for(node), convergence_strategy_for(node))
+        else
+          require 'chef_metal/machine/unix_machine'
+          ChefMetal::Machine::UnixMachine.new(node, transport_for(node), convergence_strategy_for(node))
+        end
+      end
+
+      def convergence_strategy_for(node)
+        if vagrant_option(node, 'vm.guest').to_s == 'windows'
+          require 'chef_metal/convergence_strategy/install_msi'
+          ChefMetal::ConvergenceStrategy::InstallMsi.new
+        else
+          require 'chef_metal/convergence_strategy/install_sh'
+          ChefMetal::ConvergenceStrategy::InstallSh.new
+        end
+      end
+
+      def transport_for(node)
+        if vagrant_option(node, 'vm.guest').to_s == 'windows'
+          create_winrm_transport(node)
+        else
+          create_ssh_transport(node)
+        end
+      end
+
+      def vagrant_option(node, option)
+        if node['normal']['provisioner_options'] &&
+           node['normal']['provisioner_options']['vagrant_options']
+          node['normal']['provisioner_options']['vagrant_options'][option]
+        else
+          nil
+        end
+      end
+
+      def vagrant_status(name)
+        status_output = shell_out("vagrant status #{name}", :cwd => cluster_path).stdout
+        if status_output =~ /^#{name}\s+([^\n]+)\s+\(([^\n]+)\)$/m
+          $1
+        else
+          'not created'
+        end
+      end
+
+      def create_winrm_transport(node)
+        require 'chef_metal/transport/winrm'
+
+        provisioner_output = node['default']['provisioner_output'] || {}
+        forwarded_ports = provisioner_output['forwarded_ports'] || {}
+
+        # TODO IPv6 loopback?  What do we do for that?
+        hostname = vagrant_option(node, 'winrm.host') || '127.0.0.1'
+        port = vagrant_option(node, 'winrm.port') || forwarded_ports[5985] || 5985
+        endpoint = "http://#{hostname}:#{port}/wsman"
+        type = :plaintext
+        options = {
+          :user => vagrant_option(node, 'winrm.username') || 'vagrant',
+          :pass => vagrant_option(node, 'winrm.password') || 'vagrant',
+          :disable_sspi => true
+        }
+
+        ChefMetal::Transport::WinRM.new(endpoint, type, options)
+      end
+
+      def create_ssh_transport(node)
+        require 'chef_metal/transport/ssh'
+
+        vagrant_ssh_config = vagrant_ssh_config_for(node)
+        hostname = vagrant_ssh_config['HostName']
+        username = vagrant_ssh_config['User']
+        ssh_options = {
+          :port => vagrant_ssh_config['Port'],
+          :auth_methods => ['publickey'],
+          :user_known_hosts_file => vagrant_ssh_config['UserKnownHostsFile'],
+          :paranoid => yes_or_no(vagrant_ssh_config['StrictHostKeyChecking']),
+          :keys => [ strip_quotes(vagrant_ssh_config['IdentityFile']) ],
+          :keys_only => yes_or_no(vagrant_ssh_config['IdentitiesOnly'])
+        }
+        ssh_options[:auth_methods] = %w(password) if yes_or_no(vagrant_ssh_config['PasswordAuthentication'])
+        options = {
+          :prefix => 'sudo '
+        }
+        ChefMetal::Transport::SSH.new(hostname, username, ssh_options, options)
+      end
+
+      def vagrant_ssh_config_for(node)
+        vagrant_ssh_config = {}
+        result = shell_out("vagrant ssh-config #{node['normal']['provisioner_output']['vm_name']}", :cwd => cluster_path)
+        result.stdout.lines.inject({}) do |result, line|
+          line =~ /^\s*(\S+)\s+(.+)/
+          vagrant_ssh_config[$1] = $2
+        end
+        vagrant_ssh_config
+      end
+
+      def yes_or_no(str)
+        case str
+        when 'yes'
+          true
+        else
+          false
+        end
+      end
+
+      def strip_quotes(str)
+        if str[0] == '"' && str[-1] == '"' && str.size >= 2
+          str[1..-2]
+        else
+          str
+        end
+      end
+    end
+  end
+end

data/lib/chef_metal/recipe_dsl.rb

@@ -0,0 +1,26 @@
+require 'chef_metal'
+require 'chef_metal/provisioner/fog_provisioner'
+
+class Chef
+  class Recipe
+    def with_provisioner(provisioner, &block)
+      ChefMetal.with_provisioner(provisioner, &block)
+    end
+
+    def with_provisioner_options(provisioner_options, &block)
+      ChefMetal.with_provisioner_options(provisioner_options, &block)
+    end
+
+    def with_vagrant_cluster(cluster_path, &block)
+      ChefMetal.with_vagrant_cluster(cluster_path, &block)
+    end
+
+    def with_vagrant_box(box_name, vagrant_options = {}, &block)
+      ChefMetal.with_vagrant_box(box_name, vagrant_options, &block)
+    end
+
+    def with_fog_ec2_provisioner(options = {}, &block)
+      ChefMetal.with_provisioner(ChefMetal::Provisioner::FogProvisioner.new({ :provider => 'AWS' }.merge(options)), &block)
+    end
+  end
+end

data/lib/chef_metal/transport.rb

@@ -0,0 +1,36 @@
+module ChefMetal
+  class Transport
+    def execute(command)
+      raise "execute not overridden on #{self.class}"
+    end
+
+    def read_file(path)
+      raise "read_file not overridden on #{self.class}"
+    end
+
+    def write_file(path, content)
+      raise "write_file not overridden on #{self.class}"
+    end
+
+    def download_file(path, local_path)
+      IO.write(local_path, read_file(path))
+    end
+
+    def upload_file(local_path, path)
+      write_file(path, IO.read(local_path))
+    end
+
+    # Forward requests to a port on the guest to a server on the host
+    def forward_remote_port_to_local(remote_port, local_port)
+      raise "forward_remote_port_to_local not overridden on #{self.class}"
+    end
+
+    def disconnect
+      raise "disconnect not overridden on #{self.class}"
+    end
+
+    def available?
+      raise "available? not overridden on #{self.class}"
+    end
+  end
+end

data/lib/chef_metal/transport/ssh.rb

@@ -0,0 +1,157 @@
+require 'chef_metal/transport'
+
+module ChefMetal
+  class Transport
+    class SSH < Transport
+      def initialize(host, username, ssh_options, options)
+        require 'net/ssh'
+        require 'net/scp'
+        @host = host
+        @username = username
+        @ssh_options = ssh_options
+        @options = options
+      end
+
+      attr_reader :host
+      attr_reader :username
+      attr_reader :ssh_options
+      attr_reader :options
+
+      def execute(command)
+        Chef::Log.info("Executing #{command} on #{username}@#{host}")
+        stdout = ''
+        stderr = ''
+        exitstatus = nil
+        channel = session.open_channel do |channel|
+          channel.exec("#{options[:prefix]}#{command}") do |ch, success|
+            raise "could not execute command: #{command.inspect}" unless success
+
+            channel.on_data do |ch2, data|
+              stdout << data
+            end
+
+            channel.on_extended_data do |ch2, type, data|
+              stderr << data
+            end
+
+            channel.on_request "exit-status" do |ch, data|
+              exitstatus = data.read_long
+            end
+          end
+        end
+
+        channel.wait
+
+        Chef::Log.info("Completed #{command} on #{username}@#{host}: exit status #{exitstatus}")
+        Chef::Log.debug("Stdout was:\n#{stdout}") if stdout != ''
+        Chef::Log.info("Stderr was:\n#{stderr}") if stderr != ''
+        SSHResult.new(stdout, stderr, exitstatus)
+      end
+
+      def read_file(path)
+        Chef::Log.debug("Reading file #{path} from #{username}@#{host}")
+        result = StringIO.new
+        download(path, result)
+        result.string
+      end
+
+      def download_file(path, local_path)
+        Chef::Log.debug("Downloading file #{path} from #{username}@#{host} to local #{local_path}")
+        download(path, local_path)
+      end
+
+      def write_file(path, content)
+        if options[:prefix]
+          # Make a tempfile on the other side, upload to that, and sudo mv / chown / etc.
+          remote_tempfile = "/tmp/#{File.basename(path)}.#{Random.rand(2**32)}"
+          Chef::Log.debug("Writing #{content.length} bytes to #{remote_tempfile} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(StringIO.new(content), remote_tempfile)
+          execute("mv #{remote_tempfile} #{path}")
+        else
+          Chef::Log.debug("Writing #{content.length} bytes to #{path} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(StringIO.new(content), path)
+        end
+      end
+
+      def upload_file(local_path, path)
+        if options[:prefix]
+          # Make a tempfile on the other side, upload to that, and sudo mv / chown / etc.
+          remote_tempfile = "/tmp/#{File.basename(path)}.#{Random.rand(2**32)}"
+          Chef::Log.debug("Uploading #{local_path} to #{remote_tempfile} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(local_path, remote_tempfile)
+          execute("mv #{remote_tempfile} #{path}")
+        else
+          Chef::Log.debug("Uploading #{local_path} to #{path} on #{username}@#{host}")
+          Net::SCP.new(session).upload!(local_path, path)
+        end
+      end
+
+      def forward_remote_port_to_local(remote_port, local_port)
+        # TODO IPv6
+        Chef::Log.debug("Forwarding local server 127.0.0.1:#{local_port} to port #{remote_port} on #{username}@#{host}")
+        session.forward.remote(local_port, "127.0.0.1", remote_port)
+      end
+
+      def disconnect
+        if @session
+          begin
+            Chef::Log.debug("Closing SSH session on #{username}@#{host}")
+            @session.close
+          rescue
+          end
+          @session = nil
+        end
+      end
+
+      def available?
+        execute('pwd')
+        true
+      rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ECONNRESET, Net::SSH::AuthenticationFailed, Net::SSH::Disconnect, Net::SSH::HostKeyMismatch
+        Chef::Log.debug("#{username}@#{host} unavailable: could not execute 'pwd' on #{host}: #{$!.inspect}")
+        false
+      end
+
+      protected
+
+      def session
+        @session ||= begin
+          Chef::Log.debug("Opening SSH connection to #{username}@#{host} with options #{ssh_options.inspect}")
+          Net::SSH.start(host, username, ssh_options)
+        end
+      end
+
+      def download(path, local_path)
+        channel = Net::SCP.new(session).download(path, local_path)
+        begin
+          channel.wait
+        rescue Net::SCP::Error
+          nil
+        rescue
+          # This works around https://github.com/net-ssh/net-scp/pull/10 until a new net-scp is merged.
+          begin
+            channel.close
+            channel.wait
+          rescue Net::SCP::Error
+            nil
+          end
+        end
+      end
+
+      class SSHResult
+        def initialize(stdout, stderr, exitstatus)
+          @stdout = stdout
+          @stderr = stderr
+          @exitstatus = exitstatus
+        end
+
+        attr_reader :stdout
+        attr_reader :stderr
+        attr_reader :exitstatus
+
+        def error!
+          raise "Error: code #{exitstatus}.\nSTDOUT:#{stdout}\nSTDERR:#{stderr}" if exitstatus != 0
+        end
+      end
+    end
+  end
+end