chef-metal 0.1

data/lib/chef_metal/machine/unix_machine.rb

@@ -0,0 +1,108 @@
+require 'chef_metal/machine/basic_machine'
+require 'digest'
+
+module ChefMetal
+  class Machine
+    class UnixMachine < BasicMachine
+      def initialize(node, transport, convergence_strategy)
+        super
+      end
+
+      # Options include:
+      #
+      # command_prefix - prefix to put in front of any command, e.g. sudo
+      attr_reader :options
+
+      # Delete file
+      def delete_file(provider, path)
+        if file_exists?(path)
+          provider.converge_by "delete file #{path} on #{node['name']}" do
+            transport.execute("rm -f #{path}").error!
+          end
+        end
+      end
+
+      # Return true or false depending on whether file exists
+      def file_exists?(path)
+        result = transport.execute("ls -d #{path}")
+        result.exitstatus == 0 && result.stdout != ''
+      end
+
+      def files_different?(path, local_path, content=nil)
+        if !file_exists?(path)
+          return true
+        end
+
+        # Get remote checksum of file
+        result = transport.execute("md5sum -b #{path}")
+        result.error!
+        remote_sum = result.stdout.split(' ')[0]
+
+        digest = Digest::MD5.new
+        if content
+          digest.update(content)
+        else
+          File.open(local_path, 'rb') do |io|
+            while (buf = io.read(4096)) && buf.length > 0
+              digest.update(buf)
+            end
+          end
+        end
+        remote_sum != digest.hexdigest
+      end
+
+      def create_dir(provider, path)
+        if !file_exists?(path)
+          provider.converge_by "create directory #{path} on #{node['name']}" do
+            transport.execute("mkdir #{path}").error!
+          end
+        end
+      end
+
+      # Set file attributes { mode, :owner, :group }
+      def set_attributes(provider, path, attributes)
+        if attributes[:mode] || attributes[:owner] || attributes[:group]
+          current_attributes = get_file_attributes(path)
+          if attributes[:mode] && current_attributes[:mode] != attributes[:mode]
+            provider.converge_by "change mode of #{path} on #{node['name']} from #{current_attributes[:mode].to_i(8)} to #{attributes[:mode].to_i(8)}" do
+              transport.execute("chmod #{attributes[:mode].to_i(8)} #{path}").error!
+            end
+          end
+          if attributes[:owner] && current_attributes[:owner] != attributes[:owner]
+            provider.converge_by "change group of #{path} on #{node['name']} from #{current_attributes[:owner]} to #{attributes[:owner]}" do
+              transport.execute("chown #{attributes[:owner]} #{path}").error!
+            end
+          end
+          if attributes[:group] && current_attributes[:group] != attributes[:group]
+            provider.converge_by "change group of #{path} on #{node['name']} from #{current_attributes[:group]} to #{attributes[:group]}" do
+              transport.execute("chgrp #{attributes[:group]} #{path}").error!
+            end
+          end
+        end
+      end
+
+      # Get file attributes { :mode, :owner, :group }
+      def get_attributes(path)
+        file_info = transport.execute("ls -ld #{path}").stdout.split(/\s+/)
+        if file_info.size <= 1
+          raise "#{path} does not exist in set_attributes()"
+        end
+        result = {
+          :mode => 0,
+          :owner => file_info[2],
+          :group => file_info[3]
+        }
+        attribute_string = file_info[0]
+        0.upto(attribute_string.length-1).each do |i|
+          result[:mode] <<= 1
+          result[:mode] += (attribute_string[i] == '-' ? 0 : 1)
+        end
+        result
+      end
+
+      def dirname_on_machine(path)
+        path.split('/')[0..-2].join('/')
+      end
+    end
+  end
+end

data/lib/chef_metal/machine/windows_machine.rb

@@ -0,0 +1,94 @@
+require 'chef_metal/machine/basic_machine'
+
+module ChefMetal
+  class Machine
+    class WindowsMachine < BasicMachine
+      def initialize(node, transport, convergence_strategy)
+        super
+      end
+
+      # Options include:
+      #
+      # command_prefix - prefix to put in front of any command, e.g. sudo
+      attr_reader :options
+
+      # Delete file
+      def delete_file(provider, path)
+        if file_exists?(path)
+          provider.converge_by "delete file #{escape(path)} on #{node['name']}" do
+            transport.execute("Remove-Item #{escape(path)}").error!
+          end
+        end
+      end
+
+      # Return true or false depending on whether file exists
+      def file_exists?(path)
+        parse_boolean(transport.execute("Test-Path #{escape(path)}").stdout)
+      end
+
+      def files_different?(path, local_path, content=nil)
+        # Get remote checksum of file (from http://stackoverflow.com/a/13926809)
+        result = transport.execute <<-EOM
+$md5 = [System.Security.Cryptography.MD5]::Create("MD5")
+$fd = [System.IO.File]::OpenRead(#{path.inspect})
+$buf = new-object byte[] (1024*1024*8) # 8mb buffer
+while (($read_len = $fd.Read($buf,0,$buf.length)) -eq $buf.length){
+    $total += $buf.length
+    $md5.TransformBlock($buf,$offset,$buf.length,$buf,$offset)
+}
+# finalize the last read
+$md5.TransformFinalBlock($buf,0,$read_len)
+$hash = $md5.Hash
+# convert hash bytes to hex formatted string
+$hash | foreach { $hash_txt += $_.ToString("x2") }
+$hash_txt
+EOM
+        result.error!
+        remote_sum = result.stdout.split(' ')[0]
+        digest = Digest::SHA256.new
+        if content
+          digest.update(content)
+        else
+          File.open(local_path, 'rb') do |io|
+            while (buf = io.read(4096)) && buf.length > 0
+              digest.update(buf)
+            end
+          end
+        end
+        remote_sum != digest.hexdigest
+      end
+
+      def create_dir(provider, path)
+        if !file_exists?(path)
+          provider.converge_by "create directory #{path} on #{node['name']}" do
+            transport.execute("New-Item #{escape(path)} -Type directory")
+          end
+        end
+      end
+
+      # Set file attributes { :owner, :group, :rights }
+#      def set_attributes(provider, path, attributes)
+#      end
+
+      # Get file attributes { :owner, :group, :rights }
+#      def get_attributes(path)
+#      end
+
+      def dirname_on_machine(path)
+        path.split(/[\\\/]/)[0..-2].join('\\')
+      end
+
+      def escape(string)
+        transport.escape(string)
+      end
+
+      def parse_boolean(string)
+        if string =~ /^\s*true\s*$/mi
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/chef_metal/provisioner.rb

@@ -0,0 +1,71 @@
+module ChefMetal
+  class Provisioner
+    # Acquire a machine, generally by provisioning it.  Returns a Machine
+    # object pointing at the machine, allowing useful actions like setup,
+    # converge, execute, file and directory.  The Machine object will have a
+    # "node" property which must be saved to the server (if it is any
+    # different from the original node object).
+    #
+    # ## Parameters
+    # provider - the provider object that is calling this method.
+    # node - node object (deserialized json) representing this machine.  If
+    #        the node has a provisioner_options hash in it, these will be used
+    #        instead of options provided by the provisioner.  TODO compare and
+    #        fail if different?
+    #        node will have node['normal']['provisioner_options'] in it with any
+    #        options. It is a hash with at least these options:
+    #
+    #           -- provisioner_url: <provisioner url>
+    #
+    #        node['normal']['provisioner_output'] will be populated with
+    #        information about the created machine.  For vagrant, it is a hash
+    #        with at least these options:
+    #
+    #           -- provisioner_url: <provisioner url>
+    #
+    def acquire_machine(provider, node)
+      raise "#{self.class} does not override acquire_machine"
+    end
+
+    # Connect to a machine without acquiring it.  This method will NOT make any
+    # changes to anything.
+    #
+    # ## Parameters
+    # node - node object (deserialized json) representing this machine.  The
+    #        node may have normal attributes "provisioner_options" and
+    #        "provisioner_output" in it, representing the input and output of
+    #        any prior "acquire_machine" process (if any).
+    #
+    def connect_to_machine(node)
+      raise "#{self.class} does not override connect_to_machine"
+    end
+
+    # Delete the given machine (idempotent).  Should destroy the machine,
+    # returning things to the state before acquire_machine was called.
+    def delete_machine(provider, node)
+      raise "#{self.class} does not override delete_machine"
+    end
+
+    # Stop the given machine.
+    def stop_machine(provider, node)
+      raise "#{self.class} does not override stop_machine"
+    end
+
+    # Provider notification that happens at the point a resource is declared
+    # (after all properties have been set on it)
+    def resource_created(machine)
+    end
+
+    protected
+
+    def save_node(provider, node, chef_server)
+      # Save the node and create the client.  TODO strip automatic attributes first so we don't race with "current state"
+      ChefMetal.inline_resource(provider) do
+        chef_node node['name'] do
+          chef_server chef_server
+          raw_json node
+        end
+      end
+    end
+  end
+end

data/lib/chef_metal/provisioner/fog_provisioner.rb

@@ -0,0 +1,378 @@
+require 'chef_metal/provisioner'
+require 'chef_metal/aws_credentials'
+
+module ChefMetal
+  class Provisioner
+
+    # Provisions machines in vagrant.
+    class FogProvisioner < Provisioner
+
+      include Chef::Mixin::ShellOut
+
+      DEFAULT_OPTIONS = {
+        :create_timeout => 600,
+        :start_timeout => 600,
+        :ssh_timeout => 20
+      }
+
+      # Create a new vagrant provisioner.
+      #
+      # ## Parameters
+      # compute_options - hash of options to be passed to Fog::Compute.new
+      # Special options:
+      #   - :base_bootstrap_options is merged with bootstrap_options in acquire_machine
+      #     to present the full set of bootstrap options.  Write down any bootstrap_options
+      #     you intend to apply universally here.
+      #   - :aws_credentials is an AWS CSV file (created with Download Credentials)
+      #     containing your aws key information.  If you do not specify aws_access_key_id
+      #     and aws_secret_access_key explicitly, the first line from this file
+      #     will be used.  You may pass a Cheffish::AWSCredentials object.
+      #   - :create_timeout - the time to wait for the instance to boot to ssh (defaults to 600)
+      #   - :start_timeout - the time to wait for the instance to start (defaults to 600)
+      #   - :ssh_timeout - the time to wait for ssh to be available if the instance is detected as up (defaults to 20)
+      def initialize(compute_options)
+        @base_bootstrap_options = compute_options.delete(:base_bootstrap_options) || {}
+        if compute_options[:provider] == 'AWS'
+          aws_credentials = compute_options.delete(:aws_credentials)
+          if aws_credentials
+            @aws_credentials = aws_credentials
+          else
+            @aws_credentials = ChefMetal::AWSCredentials.new
+            @aws_credentials.load_default
+          end
+          compute_options[:aws_access_key_id] ||= @aws_credentials.default[:access_key_id]
+          compute_options[:aws_secret_access_key] ||= @aws_credentials.default[:secret_access_key]
+        end
+        @key_pairs = {}
+        @compute_options = compute_options
+        @base_bootstrap_options_for = {}
+      end
+
+      attr_reader :compute_options
+      attr_reader :aws_credentials
+      attr_reader :key_pairs
+
+      def current_base_bootstrap_options
+        result = @base_bootstrap_options.dup
+        if compute_options[:provider] == 'AWS'
+          if key_pairs.size > 0
+            last_pair_name = key_pairs.keys.last
+            last_pair = key_pairs[last_pair_name]
+            result[:key_name] ||= last_pair_name
+            result[:private_key_path] ||= last_pair.private_key_path
+            result[:public_key_path] ||= last_pair.public_key_path
+          end
+        end
+        result
+      end
+
+      # Acquire a machine, generally by provisioning it.  Returns a Machine
+      # object pointing at the machine, allowing useful actions like setup,
+      # converge, execute, file and directory.  The Machine object will have a
+      # "node" property which must be saved to the server (if it is any
+      # different from the original node object).
+      #
+      # ## Parameters
+      # provider - the provider object that is calling this method.
+      # node - node object (deserialized json) representing this machine.  If
+      #        the node has a provisioner_options hash in it, these will be used
+      #        instead of options provided by the provisioner.  TODO compare and
+      #        fail if different?
+      #        node will have node['normal']['provisioner_options'] in it with any options.
+      #        It is a hash with this format:
+      #
+      #           -- provisioner_url: fog:<relevant_fog_options>
+      #           -- bootstrap_options: hash of options to pass to compute.servers.create
+      #           -- is_windows: true if windows.  TODO detect this from ami?
+      #           -- create_timeout - the time to wait for the instance to boot to ssh (defaults to 600)
+      #           -- start_timeout - the time to wait for the instance to start (defaults to 600)
+      #           -- ssh_timeout - the time to wait for ssh to be available if the instance is detected as up (defaults to 20)
+      #
+      #        Example bootstrap_options for ec2:
+      #           :image_id =>'ami-311f2b45',
+      #           :flavor_id =>'t1.micro',
+      #           :key_name => 'pey-pair-name'
+      #
+      #        node['normal']['provisioner_output'] will be populated with information
+      #        about the created machine.  For vagrant, it is a hash with this
+      #        format:
+      #
+      #           -- provisioner_url: fog:<relevant_fog_options>
+      #           -- server_id: the ID of the server so it can be found again
+      #
+      def acquire_machine(provider, node)
+        # Set up the modified node data
+        provisioner_options = node['normal']['provisioner_options'] || {}
+        provisioner_output = node['normal']['provisioner_output'] || {
+          'provisioner_url' => provisioner_url
+        }
+
+        if provisioner_output['provisioner_url'] != provisioner_url
+          raise "Switching providers for a machine is not currently supported!  Use machine :destroy and then re-create the machine on the new provider."
+        end
+
+        node['normal']['provisioner_output'] = provisioner_output
+
+        if provisioner_output['server_id']
+
+          # If the server already exists, make sure it is up
+
+          # TODO verify that the server info matches the specification (ami, etc.)\
+          server = server_for(node)
+          if !server
+            Chef::Log.warn "Machine #{node['name']} (#{provisioner_output['server_id']} on #{provisioner_url}) is not associated with the ec2 account.  Recreating ..."
+            need_to_create = true
+          elsif server.state == 'terminated' # Can't come back from that
+            Chef::Log.warn "Machine #{node['name']} (#{server.id} on #{provisioner_url}) is terminated.  Recreating ..."
+            need_to_create = true
+          else
+            need_to_create = false
+            if !server.ready?
+              provider.converge_by "start machine #{node['name']} (#{server.id} on #{provisioner_url})" do
+                server.start
+              end
+              provider.converge_by "wait for machine #{node['name']} (#{server.id} on #{provisioner_url}) to be ready" do
+                wait_until_ready(server, option_for(node, :start_timeout))
+              end
+            else
+              wait_until_ready(server, option_for(node, :ssh_timeout))
+            end
+          end
+        else
+          need_to_create = true
+        end
+
+        if need_to_create
+          # If the server does not exist, create it
+          bootstrap_options = bootstrap_options_for(provider.new_resource, node)
+
+          start_time = Time.now
+          timeout = option_for(node, :create_timeout)
+
+          description = [ "create machine #{node['name']} on #{provisioner_url}" ]
+          bootstrap_options.each_pair { |key,value| description << "    #{key}: #{value.inspect}" }
+          server = nil
+          provider.converge_by description do
+            server = compute.servers.create(bootstrap_options)
+            provisioner_output['server_id'] = server.id
+            # Save quickly in case something goes wrong
+            save_node(provider, node, provider.new_resource.chef_server)
+          end
+
+          if server
+            # Re-retrieve the server in a more malleable form and wait for it to be ready
+            server = compute.servers.get(server.id)
+            provider.converge_by "machine #{node['name']} created as #{server.id} on #{provisioner_url}" do
+            end
+            # Wait for the machine to come up and for ssh to start listening
+            transport = nil
+            _self = self
+            provider.converge_by "wait for machine #{node['name']} to boot" do
+              server.wait_for(timeout - (Time.now - start_time)) do
+                if ready?
+                  transport ||= _self.transport_for(server)
+                  begin
+                    transport.execute('pwd')
+                    true
+                  rescue Errno::ECONNREFUSED, Net::SSH::Disconnect
+                    false
+                  rescue
+                    true
+                  end
+                else
+                  false
+                end
+              end
+            end
+
+            # If there is some other error, we just wait patiently for SSH
+            begin
+              server.wait_for(option_for(node, :ssh_timeout)) { transport.available? }
+            rescue Fog::Errors::TimeoutError
+              # Sometimes (on EC2) the machine comes up but gets stuck or has
+              # some other problem.  If this is the case, we restart the server
+              # to unstick it.  Reboot covers a multitude of sins.
+              Chef::Log.warn "Machine #{node['name']} (#{server.id} on #{provisioner_url}) was started but SSH did not come up.  Rebooting machine in an attempt to unstick it ..."
+              provider.converge_by "reboot machine #{node['name']} to try to unstick it" do
+                server.reboot
+              end
+              provider.converge_by "wait for machine #{node['name']} to be ready after reboot" do
+                wait_until_ready(server, option_for(node, :start_timeout))
+              end
+            end
+          end
+        end
+
+        # Create machine object for callers to use
+        machine_for(node, server)
+      end
+
+      # Connect to machine without acquiring it
+      def connect_to_machine(node)
+        machine_for(node)
+      end
+
+      def delete_machine(provider, node)
+        if node['normal']['provisioner_output'] && node['normal']['provisioner_output']['server_id']
+          server = compute.servers.get(node['normal']['provisioner_output']['server_id'])
+          provider.converge_by "destroy machine #{node['name']} (#{server.id} at #{provisioner_url}" do
+            server.destroy
+          end
+          convergence_strategy_for(node).delete_chef_objects(provider, node)
+        end
+      end
+
+      def stop_machine(provider, node)
+        # If the machine doesn't exist, we silently do nothing
+        if node['normal']['provisioner_output'] && node['normal']['provisioner_output']['server_id']
+          server = compute.servers.get(node['normal']['provisioner_output']['server_id'])
+          provider.converge_by "stop machine #{node['name']} (#{server.id} at #{provisioner_url}" do
+            server.stop
+          end
+        end
+      end
+
+      def resource_created(machine)
+        @base_bootstrap_options_for[machine] = current_base_bootstrap_options
+      end
+
+
+      def compute
+        @compute ||= begin
+          require 'fog/compute'
+          require 'fog'
+          Fog::Compute.new(compute_options)
+        end
+      end
+
+      def provisioner_url
+        provider_identifier = case compute_options[:provider]
+          when 'AWS'
+            compute_options[:aws_access_key_id]
+          else
+            '???'
+        end
+        "fog:#{compute_options['provider']}:#{provider_identifier}"
+      end
+
+      def transport_for(server)
+        # TODO winrm
+        create_ssh_transport(server)
+      end
+
+      protected
+
+      def option_for(node, key)
+        if node['normal']['provisioner_options'] && node['normal']['provisioner_options'][key.to_s]
+          node['normal']['provisioner_options'][key.to_s]
+        elsif compute_options[key]
+          compute_options[key]
+        else
+          DEFAULT_OPTIONS[key]
+        end
+      end
+
+      def symbolize_keys(options)
+        options.inject({}) { |result,key,value| result[key.to_sym] = value; result }
+      end
+
+      def server_for(node)
+        if node['normal']['provisioner_output'] && node['normal']['provisioner_output']['server_id']
+          compute.servers.get(node['normal']['provisioner_output']['server_id'])
+        else
+          nil
+        end
+      end
+
+      def bootstrap_options_for(machine, node)
+        provisioner_options = node['normal']['provisioner_options'] || {}
+        bootstrap_options = @base_bootstrap_options_for[machine] || current_base_bootstrap_options
+        bootstrap_options = bootstrap_options.merge(symbolize_keys(provisioner_options['bootstrap_options'] || {}))
+        require 'socket'
+        require 'etc'
+        tags = {
+            'Name' => node['name'],
+            'BootstrapChefServer' => machine.chef_server[:chef_server_url],
+            'BootstrapHost' => Socket.gethostname,
+            'BootstrapUser' => Etc.getlogin,
+            'BootstrapNodeName' => node['name']
+        }
+        if machine.chef_server[:options] && machine.chef_server[:options][:data_store]
+          tags['ChefLocalRepository'] = machine.chef_server[:options][:data_store].chef_fs.fs_description
+        end
+        # User-defined tags override the ones we set
+        tags.merge!(bootstrap_options[:tags]) if bootstrap_options[:tags]
+        bootstrap_options.merge!({ :tags => tags })
+      end
+
+      def machine_for(node, server = nil)
+        server ||= server_for(node)
+        if !server
+          raise "Server for node #{node['name']} has not been created!"
+        end
+
+        if node['normal']['provisioner_options'] && node['normal']['provisioner_options']['is_windows']
+          require 'chef_metal/machine/windows_machine'
+          ChefMetal::Machine::WindowsMachine.new(node, transport_for(server), convergence_strategy_for(node))
+        else
+          require 'chef_metal/machine/unix_machine'
+          ChefMetal::Machine::UnixMachine.new(node, transport_for(server), convergence_strategy_for(node))
+        end
+      end
+
+      def convergence_strategy_for(node)
+        if node['normal']['provisioner_options'] && node['normal']['provisioner_options']['is_windows']
+          require 'chef_metal/convergence_strategy/install_msi'
+          ChefMetal::ConvergenceStrategy::InstallMsi.new
+        else
+          require 'chef_metal/convergence_strategy/install_sh'
+          ChefMetal::ConvergenceStrategy::InstallSh.new
+        end
+      end
+
+      def ssh_options_for(server)
+        result = {
+#          :user_known_hosts_file => vagrant_ssh_config['UserKnownHostsFile'],
+#          :paranoid => yes_or_no(vagrant_ssh_config['StrictHostKeyChecking']),
+          :auth_methods => [ 'publickey' ],
+          :keys => [ server.private_key ],
+          :keys_only => true
+        }
+        if compute_options[:provider] == 'AWS'
+          # TODO generalize for others?
+          result[:keys] = [ server.private_key || key_pairs[server.key_name].private_key_path ]
+          result[:host_key_alias] = "#{server.id}.ec2"
+        else
+          private_key_path = nil
+        end
+        result
+      end
+
+      def create_ssh_transport(server)
+        require 'chef_metal/transport/ssh'
+
+        ssh_options = ssh_options_for(server)
+        options = {
+          :prefix => 'sudo '
+        }
+        ChefMetal::Transport::SSH.new(server.public_ip_address, 'ubuntu', ssh_options, options)
+      end
+
+      def wait_until_ready(server, timeout)
+        transport = nil
+        _self = self
+        server.wait_for(timeout) do
+          if transport
+            transport.available?
+          elsif ready?
+            # Don't create the transport until the machine is ready (we won't have the host till then)
+            transport = _self.transport_for(server)
+            transport.available?
+          else
+            false
+          end
+        end
+      end
+    end
+  end
+end