chef-metal-fogsphere 0.1.0.alpha.1 → 0.1.0.alpha.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f498e6888cc6338d35fc9e418440d9f06ed49765
+  data.tar.gz: 7121c432739e9f8caa9b5a76922f90b0999285a6
+SHA512:
+  metadata.gz: e2c422291302a8b16d079714a889c9b446b1b5d474934d02c183fac483aec0e69404ae14d2f6eff7e593b9074285c03304816f7ee14d9941f453baafe40e771d
+  data.tar.gz: 5a434cadf0d45da3eea5fcb985323363d87d94bd59abd94cf29766d2689c389817792e4c3c5c65761a1e1aaa729fd997bb60559cfee67c145a2ae7627de4124d

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/LICENSE

@@ -0,0 +1,13 @@
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/chef-metal-fogsphere.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "chef-metal-fogsphere"
+  gem.version       = '0.1.0.alpha.2'
+  gem.license       = 'Apache 2.0'
+  gem.authors       = ["Matt Wrock"]
+  gem.email         = ["matt@centurylinkcloud.com"]
+  gem.description   = "A fork of Chef-Metal-fog for vsphere"
+  gem.summary       = gem.description
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = []
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'rbvmomi', '~> 1.5.1'
+  gem.add_dependency 'chef-metal'
+end

data/lib/chef_metal_clc.rb

@@ -0,0 +1,11 @@
+require 'chef_metal'
+
+class Chef
+  module DSL
+    module Recipe
+      def with_vsphere_provisioner(options = {}, &block)
+        run_context.chef_metal.with_provisioner(ChefMetalVsphere::VsphereProvisioner.new({ :provider => 'vsphere' }.merge(options)), &block)
+      end
+    end
+  end
+end

data/lib/clc_provisioner.rb

@@ -0,0 +1,419 @@
+require 'chef_metal/provisioner'
+require 'chef_metal/aws_credentials'
+require 'chef_metal/openstack_credentials'
+require 'chef_metal/machine/windows_machine'
+require 'chef_metal/machine/unix_machine'
+require 'chef_metal/convergence_strategy/install_msi'
+require 'chef_metal/convergence_strategy/install_cached'
+require 'chef_metal/transport/ssh'
+require 'fog'
+require 'fog/core'
+require 'fog/compute'
+require 'fog/aws'
+
+module ChefMetalVsphere
+  # Provisions machines in vagrant.
+  class VsphereProvisioner < ChefMetal::Provisioner
+    include Chef::Mixin::ShellOut
+
+    DEFAULT_OPTIONS = {
+        :create_timeout => 600,
+        :start_timeout => 600,
+        :ssh_timeout => 20
+    } unless defined? DEFAULT_OPTIONS
+
+    def self.inflate(node_json)
+      url = node_json['normal']['provisioner_output']['provisioner_url']
+      scheme, provider, id = url.split(':', 3)
+      VsphereProvisioner.new({ :provider => provider }, id)
+    end
+
+    # Create a new fog provisioner.
+    #
+    # ## Parameters
+    # compute_options - hash of options to be passed to Fog::Compute.new
+    # Special options:
+    #   - :base_bootstrap_options is merged with bootstrap_options in acquire_machine
+    #     to present the full set of bootstrap options.  Write down any bootstrap_options
+    #     you intend to apply universally here.
+    #   - :aws_credentials is an AWS CSV file (created with Download Credentials)
+    #     containing your aws key information.  If you do not specify aws_access_key_id
+    #     and aws_secret_access_key explicitly, the first line from this file
+    #     will be used.  You may pass a Cheffish::AWSCredentials object.
+    #   - :create_timeout - the time to wait for the instance to boot to ssh (defaults to 600)
+    #   - :start_timeout - the time to wait for the instance to start (defaults to 600)
+    #   - :ssh_timeout - the time to wait for ssh to be available if the instance is detected as up (defaults to 20)
+    # id - the ID in the provisioner_url (fog:PROVIDER:ID)
+    def initialize(compute_options, id=nil)
+      @compute_options = compute_options
+      @base_bootstrap_options = compute_options.delete(:base_bootstrap_options) || {}
+      @base_bootstrap_options_for = {}
+    end
+
+    attr_reader :compute_options
+
+    def current_base_bootstrap_options
+      result = @base_bootstrap_options.dup
+      result
+    end
+
+    # Inflate a provisioner from node_json information; we don't want to force the
+    # driver to figure out what the provisioner really needs, since it varies
+    # from provisioner to provisioner.
+    #
+    # ## Parameters
+    # node_json - node_json to inflate the provisioner for
+    #
+    # returns a FogProvisioner
+    # TODO: def self.inflate(node_json)
+    # right now, not implemented, will raise error from base class until overridden
+
+    # Acquire a machine, generally by provisioning it.  Returns a Machine
+    # object pointing at the machine, allowing useful actions like setup,
+    # converge, execute, file and directory.  The Machine object will have a
+    # "node_json" property which must be saved to the server (if it is any
+    # different from the original node_json object).
+    #
+    # ## Parameters
+    # action_handler - the action_handler object that is calling this method; this
+    #        is generally a action_handler, but could be anything that can support the
+    #        ChefMetal::ActionHandler interface (i.e., in the case of the test
+    #        kitchen metal driver for acquiring and destroying VMs; see the base
+    #        class for what needs providing).
+    # node_json - node_json object (deserialized json) representing this machine.  If
+    #        the node_json has a provisioner_options hash in it, these will be used
+    #        instead of options provided by the provisioner.  TODO compare and
+    #        fail if different?
+    #        node_json will have node_json['normal']['provisioner_options'] in it with any options.
+    #        It is a hash with this format:
+    #
+    #           -- provisioner_url: fog:<relevant_fog_options>
+    #           -- bootstrap_options: hash of options to pass to compute.servers.create
+    #           -- is_windows: true if windows.  TODO detect this from ami?
+    #           -- create_timeout - the time to wait for the instance to boot to ssh (defaults to 600)
+    #           -- start_timeout - the time to wait for the instance to start (defaults to 600)
+    #           -- ssh_timeout - the time to wait for ssh to be available if the instance is detected as up (defaults to 20)
+    #
+    #        Example bootstrap_options for ec2:
+    #          'bootstrap_options' => {
+    #            'image_id' =>'ami-311f2b45',
+    #            'flavor_id' =>'t1.micro',
+    #            'key_name' => 'key-pair-name'
+    #          }
+    #
+    #        node_json['normal']['provisioner_output'] will be populated with information
+    #        about the created machine.  For vagrant, it is a hash with this
+    #        format:
+    #
+    #           -- provisioner_url: fog:<relevant_fog_options>
+    #           -- server_id: the ID of the server so it can be found again
+    #
+    def acquire_machine(action_handler, node_json)
+      # Set up the modified node_json data
+      provisioner_output = node_json['normal']['provisioner_output'] || {
+          'provisioner_url' => provisioner_url,
+          'provisioner_version' => '0.0.1'
+      }
+
+      if provisioner_output['provisioner_url'] != provisioner_url
+        raise "Switching a machine's provider from #{provisioner_output['provisioner_url']} to #{provisioner_url} for is not currently supported!  Use machine :destroy and then re-create the machine on the new provisioner."
+      end
+
+      node_json['normal']['provisioner_output'] = provisioner_output
+
+      if provisioner_output['server_id']
+
+        # If the server already exists, make sure it is up
+
+        # TODO verify that the server info matches the specification (ami, etc.)\
+        server = server_for(node_json)
+        if !server
+          Chef::Log.warn "Machine #{node_json['name']} (#{provisioner_output['server_id']} on #{provisioner_url}) is not associated with the ec2 account.  Recreating ..."
+          need_to_create = true
+        elsif server.respond_to?(:state) && %w(terminated archive).include?(server.state) # Can't come back from that
+          Chef::Log.warn "Machine #{node_json['name']} (#{server.id} on #{provisioner_url}) is terminated.  Recreating ..."
+          need_to_create = true
+        else
+          need_to_create = false
+          if !server.ready?
+            action_handler.perform_action "start machine #{node_json['name']} (#{server.id} on #{provisioner_url})" do
+              server.start
+            end
+            action_handler.perform_action "wait for machine #{node_json['name']} (#{server.id} on #{provisioner_url}) to be ready" do
+              wait_until_ready(node_json, server, option_for(node_json, :start_timeout))
+            end
+          else
+            wait_until_ready(node_json, server, option_for(node_json, :ssh_timeout))
+          end
+        end
+      else
+        need_to_create = true
+      end
+
+      if need_to_create
+        # If the server does not exist, create it
+        bootstrap_options = bootstrap_options_for(action_handler.new_resource, node_json)
+        bootstrap_options = bootstrap_options.merge(:name => node_json['name'])
+
+        start_time = Time.now
+        timeout = option_for(node_json, :create_timeout)
+
+        description = [ "create machine #{node_json['name']} on #{provisioner_url}" ]
+        bootstrap_options.each_pair { |key,value| description << "    #{key}: #{value.inspect}" }
+        server = nil
+        action_handler.perform_action description do
+          clone_results = compute.vm_clone(convert_to_strings(bootstrap_options))
+          server = compute.servers.get(clone_results['new_vm']['id'])
+          if(bootstrap_options[:additional_disk_size_gb] && bootstrap_options[:additional_disk_size_gb].is_a?(Integer) && bootstrap_options[:additional_disk_size_gb] > 0)
+            volume_options = {
+                'server_id' => server.id,
+                'datastore' => bootstrap_options[:datastore],
+                'size_gb' => bootstrap_options[:additional_disk_size_gb]
+            }
+            volume = compute.volumes.create(volume_options)
+          end
+          provisioner_output['server_id'] = server.id
+          # Save quickly in case something goes wrong
+          save_node(action_handler, node_json, action_handler.new_resource.chef_server)
+        end
+
+        if server
+          @@ip_pool_lock = Mutex.new
+          # Re-retrieve the server in a more malleable form and wait for it to be ready
+          server = compute.servers.get(server.id)
+          action_handler.perform_action "machine #{node_json['name']} created as #{server.id} on #{provisioner_url}" do
+          end
+          # Wait for the machine to come up
+          Chef::Log.info "waiting for machine to come up"
+          server.wait_for(timeout - (Time.now - start_time)) { tools_state != 'toolsNotRunning' }
+
+          if has_static_ip(node_json)
+            Chef::Log.info "waiting for customizations to complete"
+            sleep(30)
+            Chef::Log.info "rebooting..."
+            server.reboot
+          end
+
+          # wait for ssh to start listening
+          transport = nil
+          _self = self
+          action_handler.perform_action "wait for machine #{node_json['name']} to boot" do
+            server.wait_for(timeout - (Time.now - start_time)) do
+              if tools_state != 'toolsNotRunning'
+                transport ||= _self.transport_for(server, node_json)
+                begin
+                  transport.execute('pwd')
+                  Chef::Log.info "ssh succeeded"
+                  true
+                rescue ChefMetal::Transport::SSH::InitialConnectTimeout, Errno::ECONNREFUSED, Net::SSH::Disconnect, Errno::EHOSTUNREACH => err
+                  Chef::Log.info "ssh failed1: #{err}"
+                  false
+                rescue Exception => e
+                  Chef::Log.info "ssh failed2: #{e}"
+                  true
+                end
+              else
+                false
+              end
+            end
+          end
+
+          # If there is some other error, we just wait patiently for SSH
+          Chef::Log.info "entering next block"
+          begin
+            server.wait_for(option_for(node_json, :ssh_timeout)) { transport.available? }
+          rescue Fog::Errors::TimeoutError
+            # Sometimes (on EC2) the machine comes up but gets stuck or has
+            # some other problem.  If this is the case, we restart the server
+            # to unstick it.  Reboot covers a multitude of sins.
+            Chef::Log.warn "Machine #{node_json['name']} (#{server.id} on #{provisioner_url}) was started but SSH did not come up.  Rebooting machine in an attempt to unstick it ..."
+            action_handler.perform_action "reboot machine #{node_json['name']} to try to unstick it" do
+              server.reboot
+            end
+            Chef::Log.info "rebooted"
+            action_handler.perform_action "wait for machine #{node_json['name']} to be ready after reboot" do
+              wait_until_ready(node_json, server, option_for(node_json, :start_timeout))
+            end
+            Chef::Log.info "reported ready"
+          end
+        end
+      end
+
+      # Create machine object for callers to use
+      machine_for(node_json, server)
+    end
+
+    def has_static_ip(node_json)
+      bootstrap_options = node_json['normal']['provisioner_options']['bootstrap_options']
+      if bootstrap_options.has_key?('customization_spec')
+        bootstrap_options = bootstrap_options['customization_spec']
+        if bootstrap_options.has_key?('ipsettings')
+          bootstrap_options = bootstrap_options['ipsettings']
+          if bootstrap_options.has_key?('ip')
+            return true
+          end
+        end
+      end
+      false
+    end
+
+    def convert_to_strings(objay)
+      if objay.kind_of?(Array)
+        objay.map { |v| convert_to_strings(v) }
+      elsif objay.kind_of?(Hash)
+        Hash[objay.map { |(k, v)| [k.to_s, convert_to_strings(v)] }]
+      else
+        objay
+      end
+    end
+
+    # Connect to machine without acquiring it
+    def connect_to_machine(node_json)
+      machine_for(node_json)
+    end
+
+    def delete_machine(action_handler, node_json)
+      if node_json['normal']['provisioner_output'] && node_json['normal']['provisioner_output']['server_id']
+        server = compute.servers.get(node_json['normal']['provisioner_output']['server_id'])
+        if server
+          action_handler.perform_action "destroy machine #{node_json['name']} (#{node_json['normal']['provisioner_output']['server_id']} at #{provisioner_url})" do
+            server.destroy
+          end
+        end
+        convergence_strategy_for(node_json).cleanup_convergence(action_handler, node_json)
+      end
+    end
+
+    def stop_machine(action_handler, node_json)
+      # If the machine doesn't exist, we silently do nothing
+      if node_json['normal']['provisioner_output'] && node_json['normal']['provisioner_output']['server_id']
+        server = compute.servers.get(node_json['normal']['provisioner_output']['server_id'])
+        action_handler.perform_action "stop machine #{node_json['name']} (#{server.id} at #{provisioner_url})" do
+          server.stop
+        end
+      end
+    end
+
+    def resource_created(machine)
+      @base_bootstrap_options_for[machine] = current_base_bootstrap_options
+    end
+
+    def compute
+      @compute ||= Fog::Compute.new(compute_options)
+    end
+
+    def provisioner_url
+      provider_identifier = compute_options[:vsphere_server]
+      "fog:#{compute_options[:provider]}:#{provider_identifier}"
+    end
+
+    # Not meant to be part of public interface
+    def transport_for(server, node_json = nil)
+      # TODO winrm
+      create_ssh_transport(server, node_json)
+    end
+
+    protected
+
+    def option_for(node_json, key)
+      if node_json['normal']['provisioner_options'] && node_json['normal']['provisioner_options'][key.to_s]
+        node_json['normal']['provisioner_options'][key.to_s]
+      elsif compute_options[key]
+        compute_options[key]
+      else
+        DEFAULT_OPTIONS[key]
+      end
+    end
+
+    def symbolize_keys(options)
+      options.inject({}) { |result,(key,value)| result[key.to_sym] = value; result }
+    end
+
+    def server_for(node_json)
+      if node_json['normal']['provisioner_output'] && node_json['normal']['provisioner_output']['server_id']
+        compute.servers.get(node_json['normal']['provisioner_output']['server_id'])
+      else
+        nil
+      end
+    end
+
+    def bootstrap_options_for(machine, node_json)
+      provisioner_options = node_json['normal']['provisioner_options'] || {}
+      bootstrap_options = @base_bootstrap_options_for[machine] || current_base_bootstrap_options
+      bootstrap_options = bootstrap_options.merge(symbolize_keys(provisioner_options['bootstrap_options'] || {}))
+      require 'socket'
+      require 'etc'
+      tags = {
+          'Name' => node_json['name'],
+          'BootstrapChefServer' => machine.chef_server[:chef_server_url],
+          'BootstrapHost' => Socket.gethostname,
+          'BootstrapUser' => Etc.getlogin,
+          'Bootstrapnode_jsonName' => node_json['name']
+      }
+      if machine.chef_server[:options] && machine.chef_server[:options][:data_store]
+        tags['ChefLocalRepository'] = machine.chef_server[:options][:data_store].chef_fs.fs_description
+      end
+      # User-defined tags override the ones we set
+      tags.merge!(bootstrap_options[:tags]) if bootstrap_options[:tags]
+      bootstrap_options.merge!({ :tags => tags })
+
+      bootstrap_options
+    end
+
+    def machine_for(node_json, server = nil)
+      server ||= server_for(node_json)
+      if !server
+        raise "Server for node_json #{node_json['name']} has not been created!"
+      end
+
+      if node_json['normal']['provisioner_options'] && node_json['normal']['provisioner_options']['is_windows']
+        ChefMetal::Machine::WindowsMachine.new(node_json, transport_for(server, node_json), convergence_strategy_for(node_json))
+      else
+        ChefMetal::Machine::UnixMachine.new(node_json, transport_for(server, node_json), convergence_strategy_for(node_json))
+      end
+    end
+
+    def convergence_strategy_for(node_json)
+      if node_json['normal']['provisioner_options'] && node_json['normal']['provisioner_options']['is_windows']
+        @windows_convergence_strategy ||= begin
+          options = {}
+          provisioner_options = node_json['normal']['provisioner_options'] || {}
+          options[:chef_client_timeout] = provisioner_options['chef_client_timeout'] if provisioner_options.has_key?('chef_client_timeout')
+          ChefMetal::ConvergenceStrategy::InstallMsi.new(options)
+        end
+      else
+        @unix_convergence_strategy ||= begin
+          options = {}
+          provisioner_options = node_json['normal']['provisioner_options'] || {}
+          options[:chef_client_timeout] = provisioner_options['chef_client_timeout'] if provisioner_options.has_key?('chef_client_timeout')
+          ChefMetal::ConvergenceStrategy::InstallCached.new(options)
+        end
+      end
+    end
+
+    def create_ssh_transport(server, node_json)
+      username = compute_options[:ssh_username] || 'root'
+      
+      bootstrap_options = node_json['normal']['provisioner_options']['bootstrap_options']
+      remote_host = has_static_ip(node_json) ? bootstrap_options['customization_spec']['ipsettings']['ip'] : server.ipaddress
+
+      ChefMetal::Transport::SSH.new(remote_host, username, {:password => compute_options[:ssh_password], :paranoid => false}, {})
+    end
+
+    def wait_until_ready(node_json, server, timeout)
+      transport = nil
+      _self = self
+      server.wait_for(timeout) do
+        if transport
+          transport.available?
+        elsif tools_state != 'toolsNotRunning'
+          # Don't create the transport until the machine is ready (we won't have the host till then)
+          transport = _self.transport_for(server, node_json)
+          transport.available?
+        else
+          false
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chef-metal-fogsphere
 version: !ruby/object:Gem::Version
-  version: 0.1.0.alpha.1
-  prerelease: 6
+  version: 0.1.0.alpha.2
 platform: ruby
 authors:
 - Matt Wrock
@@ -14,33 +13,29 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rbvmomi
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.5.1
 - !ruby/object:Gem::Dependency
   name: chef-metal
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: A fork of Chef-Metal-fog for vsphere
@@ -49,31 +44,36 @@ email:
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- ".gitignore"
+- LICENSE
+- chef-metal-fogsphere-0.1.0.alpha.1.zip
+- chef-metal-fogsphere.gemspec
+- lib/chef_metal_clc.rb
+- lib/clc_provisioner.rb
 homepage: 
 licenses:
 - Apache 2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>'
+  - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.28
+rubygems_version: 2.2.1
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A fork of Chef-Metal-fog for vsphere
 test_files: []
 has_rdoc: