chef-metal 0.13 → 0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f7a30676d9791f178e7448693969b295a85d5f4
-  data.tar.gz: cac264ccd9b787d7ff1a799f3b0994973eb06a1a
+  metadata.gz: 3a913392bf4f33acc38f2c62be960b979affc698
+  data.tar.gz: 7980f8eed3169aed7d50eda6a5c0836825b324ae
 SHA512:
-  metadata.gz: 31858346827c13c25a8fb8a5a6e222a424db9c610a86e643971e5a4620fb9b863eb8418c3235830d0d519870e07a519359e548a6e94099c6cc17b50251cd2255
-  data.tar.gz: fec942a369b4d2e82a33090bcf47caa51cda90346c9f867ac00d5633d95f9c633e98a786ddb997d43466623ca184342845e1e1ad7756d1f3444b49ef713f6c31
+  metadata.gz: 50d27c172313e6c3d53bcc392f9d09cab8e9e4282ef77d46c65e854718e6f036415dd77570c1fc014c784768dd18b4962a0fd7f29a5045f7590b3e7d4f32d172
+  data.tar.gz: 147d41779a646160b1b93021ca116991d3d5740edd6284d248100b2d022e9366cec4453fd916bc6f4ed34daeeabe41dcef9ee4795e43a923d41b954d2fd805a6

data/CHANGELOG.md

@@ -1,9 +1,56 @@
 # Chef Metal Changelog
 
+## 0.14 (8/18/2013)
+
+
+- FEATURE: Add the machine_image resource (@jkeiser, @johnewart):
+  ```ruby
+  machine_image 'base' do
+    machine_options :bootstrap_options => { :image_id => 'ami-1234798123431', :ssh_username => 'root' }
+    recipe 'secure_base'
+    recipe 'corp_users'
+  end
+  # Build an image based on 'base' that has apache
+  machine_image 'apache' do
+    # All bootstrap options, like ssh_username, are carried forward
+    from_image 'base'
+    recipe 'apache2'
+  end
+  # Build an image with my web app based on the apache base image
+  machine_image 'myapp' do
+    from_image 'apache'
+    recipe 'mywebapp'
+  end
+  # Build an image with mysql and my schema based on the corporate base image
+  machine_image 'mydb' do
+    from_image 'base'
+    recipe 'mysql'
+    recipe 'myschema'
+  end
+  # Build a DB machine from mydb.  Does not reinstall stuff! :)
+  machine 'db' do
+    from_image 'mydb'
+  end
+  # Build a web app machine from myapp.  Does not reinstall stuff! :)
+  machine 'myapp1' do
+    from_image 'myapp'
+  end
+  ```
+  - Creates a node with the name of the machine_image, which contains metadata
+    like the username of the image.  This makes things like AWS image registries
+    possible.
+- Fix the no_converge convergence strategy (@johnewart)
+- SSH port forwarding improvements:
+  - Detects *any* IP on the localhost and forwards it--not just 127.0.0.1
+  - Binds to localhost on the remote side instead of 127.0.0.1, allowing for IPv6 communication
+  - Tries multiple ports--if the origin port is already taken, tries "0" (ephemeral).
+- Fix SSH race condition causing port forwarding to happen twice (and fail miserably)
+- Add ChefMetal.connect_to_machine('mario')
+
 ## 0.13 (6/17/2014)
 
 - make winrm work again (@mwrock)
-- add bootstrap_proxy as a convergence_option for when target machines require a proxy
+- add bootstrap_proxy as a convergence_option for when target machines require a proxy (@MrMMorris)
 
 ## 0.12.1 (6/18/2014)
 

data/README.md

@@ -234,9 +234,14 @@ knife edit /groups/admin.json -e <editor>
 ```
 Then add:
 ```
- "groups": [
-    "clients"
+{
+  "users": [
+    "pivotal" # This is an internal superuser for Hosted/Enterprise Chef
+  ],
+  "groups": [
+    "clients" # This is what you need to add
   ]
+}
 ```
 
 This can also be done through the Chef Server web UI (Administration tab > Groups > select admins Group > Add 'clients'

data/lib/chef/provider/machine.rb

@@ -94,18 +94,31 @@ class Chef::Provider::Machine < Chef::Provider::LWRPBase
     end
   end
 
+  def from_image_spec
+    @from_image_spec ||= begin
+      if new_resource.from_image
+        ChefMetal::ChefImageSpec.get(new_resource.from_image, new_resource.chef_server)
+      else
+        nil
+      end
+    end
+  end
+
   def new_machine_options
     machine_options(new_driver)
   end
 
   def current_machine_options
-    if current_driver
-      machine_options(current_driver)
-    end
+    machine_options(current_driver)
   end
 
   def machine_options(driver)
     configs = []
+
+    if from_image_spec && from_image_spec.machine_options
+      configs << from_image_spec.machine_options
+    end
+
     configs << {
       :convergence_options =>
         [ :chef_server,

data/lib/chef/provider/machine_image.rb

@@ -0,0 +1,54 @@
+require 'chef/provider/lwrp_base'
+require 'chef/provider/chef_node'
+require 'openssl'
+require 'chef_metal/chef_provider_action_handler'
+require 'chef_metal/chef_image_spec'
+
+class Chef::Provider::MachineImage < Chef::Provider::LWRPBase
+  def action_handler
+    @action_handler ||= ChefMetal::ChefProviderActionHandler.new(self)
+  end
+
+  def load_current_resource
+  end
+
+  # Get the driver specified in the resource
+  def new_driver
+    @new_driver ||= run_context.chef_metal.driver_for(new_resource.driver)
+  end
+
+  action :create do
+    # Get the image mapping on the server (from name to image-id)
+    image_spec = ChefMetal::ChefImageSpec.get(new_resource.name, new_resource.chef_server) ||
+                 ChefMetal::ChefImageSpec.empty(new_resource.name, new_resource.chef_server)
+    if image_spec.location
+      # TODO check for real existence and maybe update
+    else
+      #
+      # Create a new image
+      #
+      image_spec.machine_options = new_resource.machine_options
+      create_image(image_spec)
+    end
+  end
+
+  action :destroy do
+  end
+
+  def create_image(image_spec)
+    # 1. Create the exemplar machine
+    machine_provider = Chef::Provider::Machine.new(new_resource, run_context)
+    machine_provider.load_current_resource
+    machine_provider.action_converge
+
+    # 2. Create the image
+    new_driver.allocate_image(action_handler, image_spec, new_resource.image_options,
+                              machine_provider.machine_spec)
+
+    # 3. Save the linkage from name -> image id
+    image_spec.save(action_handler)
+
+    # 4. Wait for image to be ready
+    new_driver.ready_image(action_handler, image_spec, new_resource.image_options)
+  end
+end

data/lib/chef/resource/machine.rb

@@ -57,6 +57,12 @@ class Chef::Resource::Machine < Chef::Resource::LWRPBase
   # files '/remote/path.txt' => { :content => 'woo' }
   attribute :files, :kind_of => Hash
 
+  # The named machine_image to start from.  Specify the name of a machine_image
+  # object and the default machine_options will be set to use that image.
+  # == Examples
+  # from_image 'company_base_image'
+  attribute :from_image, :kind_of => String
+
   # A single file to upload, in the format REMOTE_PATH, LOCAL_PATH|HASH.
   # This directive may be passed multiple times, and multiple files will be uploaded.
   # == Examples

data/lib/chef/resource/machine_image.rb

@@ -0,0 +1,29 @@
+require 'chef/resource/lwrp_base'
+require 'cheffish'
+require 'chef_metal'
+require 'cheffish/merged_config'
+
+class Chef::Resource::MachineImage < Chef::Resource::Machine
+  self.resource_name = 'machine_image'
+
+  def initialize(*args)
+    super
+    @image_options = run_context.chef_metal.current_image_options
+    @machine_options = run_context.chef_metal.current_machine_options
+  end
+
+  # TODO check if we still have all the actions from machine
+  actions :create, :destroy, :archive
+  default_action :create
+
+  attribute :image_options, :kind_of => Hash
+
+  # TODO
+  # - policy for old version cleanup
+  # - when attributes or recipes of the machine_image changes, or when the actual
+  #   recipes therein change, we will regenerate the image
+  # - attribute to force or turn off image updates
+  # - ability to rebuild new image based on the previous image
+  # - ability to specify whether and how to keep the image-building machine around
+  # - ability to specify whether to leave the chef config around or not
+end

data/lib/chef_metal.rb

@@ -71,9 +71,13 @@ module ChefMetal
   end
 
   def self.connect_to_machine(machine_spec, config = Cheffish.profiled_config)
+    chef_server = Cheffish.default_chef_server(config)
+    if machine_spec.is_a?(String)
+      machine_spec = ChefMachineSpec.get(machine_spec, chef_server)
+    end
     driver = driver_for_url(machine_spec.driver_url, config)
     if driver
-      machine_options = { :convergence_options => { :chef_server => Cheffish.default_chef_server(config) } }
+      machine_options = { :convergence_options => { :chef_server => chef_server } }
       machine_options = Cheffish::MergedConfig.new(config[:machine_options], machine_options) if config[:machine_options]
       driver.connect_to_machine(machine_spec, machine_options)
     else

data/lib/chef_metal/chef_image_spec.rb

@@ -0,0 +1,106 @@
+require 'chef_metal'
+require 'cheffish'
+require 'chef_metal/image_spec'
+
+module ChefMetal
+  #
+  # Specification for a image. Sufficient information to find and contact it
+  # after it has been set up.
+  #
+  class ChefImageSpec < ImageSpec
+    def initialize(node, chef_server)
+      super(node)
+      @chef_server = chef_server
+    end
+
+    #
+    # Get a ImageSpec from the chef server.  If the node does not exist on the
+    # server, it returns nil.
+    #
+    def self.get(name, chef_server = Cheffish.default_chef_server)
+      chef_api = Cheffish.chef_server_api(chef_server)
+      begin
+        data = chef_api.get("/data/images/#{name}")
+        data['machine_options'] = strings_to_symbols(data['machine_options'])
+        ChefImageSpec.new(data, chef_server)
+      rescue Net::HTTPServerException => e
+        if e.response.code == '404'
+          nil
+        else
+          raise
+        end
+      end
+    end
+
+    # Creates a new empty ImageSpec with the given name.
+    def self.empty(id, chef_server = Cheffish.default_chef_server)
+      ChefImageSpec.new({ 'id' => id }, chef_server)
+    end
+
+    #
+    # Globally unique identifier for this image. Does not depend on the image's
+    # location or existence.
+    #
+    def id
+      ChefImageSpec.id_from(chef_server, name)
+    end
+
+    def self.id_from(chef_server, name)
+      "#{chef_server[:chef_server_url]}/data/images/#{name}"
+    end
+
+    #
+    # Save this node to the server.  If you have significant information that
+    # could be lost, you should do this as quickly as possible.  Data will be
+    # saved automatically for you after allocate_image and ready_image.
+    #
+    def save(action_handler)
+      # Save the node to the server.
+      _self = self
+      _chef_server = _self.chef_server
+      ChefMetal.inline_resource(action_handler) do
+        chef_data_bag_item _self.name do
+          data_bag 'images'
+          chef_server _chef_server
+          raw_data _self.image_data
+        end
+      end
+    end
+
+    def delete(action_handler)
+      # Save the node to the server.
+      _self = self
+      _chef_server = _self.chef_server
+      ChefMetal.inline_resource(action_handler) do
+        chef_data_bag_item _self.name do
+          data_bag 'images'
+          chef_server _chef_server
+          action :destroy
+        end
+      end
+    end
+
+    protected
+
+    attr_reader :chef_server
+
+    #
+    # Chef API object for the given Chef server
+    #
+    def chef_api
+      Cheffish.server_api_for(chef_server)
+    end
+
+    def self.strings_to_symbols(data)
+      if data.is_a?(Hash)
+        result = {}
+        data.each_pair do |key, value|
+          result[key.to_sym] = strings_to_symbols(value)
+        end
+        result
+      else
+        data
+      end
+    end
+  end
+end

data/lib/chef_metal/chef_machine_spec.rb

@@ -1,3 +1,4 @@
+require 'pry'
 require 'chef_metal'
 require 'cheffish'
 require 'chef_metal/machine_spec'

data/lib/chef_metal/chef_run_data.rb

@@ -16,6 +16,7 @@ module ChefMetal
     attr_reader :current_driver
 
     with :machine_options
+    with :image_options
 
     def with_driver(driver, options = nil, &block)
       if drivers[driver] && options
@@ -49,6 +50,14 @@ module ChefMetal
       end
     end
 
+    def current_image_options
+      if @current_image_options
+        @current_image_options
+      else
+        {}
+      end
+    end
+
     def add_machine_options(options, &block)
       with_machine_options(Chef::Mixin::DeepMerge.hash_only_merge(current_machine_options, options), &block)
     end

data/lib/chef_metal/convergence_strategy/no_converge.rb

@@ -14,7 +14,6 @@ module ChefMetal
       end
 
       def setup_convergence(action_handler, machine)
-        machine_spec.save(action_handler)
       end
 
       def converge(action_handler, machine)

data/lib/chef_metal/driver.rb

@@ -182,6 +182,27 @@ module ChefMetal
       raise "#{self.class} does not implement stop_machine"
     end
 
+    #
+    # Allocate an image. Returns quickly with an ID that tracks the image.
+    #
+    def allocate_image(action_handler, image_spec, image_options, machine_spec)
+      raise "#{self.class} does not implement create_image"
+    end
+
+    #
+    # Ready an image, waiting till the point where it is ready to be used.
+    #
+    def ready_image(action_handler, image_spec, image_options)
+      raise "#{self.class} does not implement ready_image"
+    end
+
+    #
+    # Destroy an image.
+    #
+    def destroy_image(action_handler, image_spec, image_options)
+      raise "#{self.class} does not implement destroy_image"
+    end
+
     #
     # Optional interface methods
     #

data/lib/chef_metal/image_spec.rb

@@ -0,0 +1,70 @@
+module ChefMetal
+  #
+  # Specification for a image. Sufficient information to find and contact it
+  # after it has been set up.
+  #
+  class ImageSpec
+    def initialize(image_data)
+      @image_data = image_data
+    end
+
+    attr_reader :image_data
+
+    #
+    # Globally unique identifier for this image. Does not depend on the image's
+    # location or existence.
+    #
+    def id
+      raise "id unimplemented"
+    end
+
+    #
+    # Name of the image. Corresponds to the name in "image 'name' do" ...
+    #
+    def name
+      image_data['id']
+    end
+
+    #
+    # Location of this image. This should be a freeform hash, with enough
+    # information for the driver to look it up and create a image object to
+    # access it.
+    #
+    # This MUST include a 'driver_url' attribute with the driver's URL in it.
+    #
+    # chef-metal will do its darnedest to not lose this information.
+    #
+    def location
+      image_data['location']
+    end
+
+    #
+    # Set the location for this image.
+    #
+    def location=(value)
+      image_data['location'] = value
+    end
+
+    def machine_options
+      image_data['machine_options']
+    end
+
+    def machine_options=(value)
+      image_data['machine_options'] = value
+    end
+
+    # URL to the driver.  Convenience for location['driver_url']
+    def driver_url
+      location ? location['driver_url'] : nil
+    end
+
+    #
+    # Save this image_data to the server.  If you have significant information that
+    # could be lost, you should do this as quickly as possible.  image_data will be
+    # saved automatically for you after allocate_image and ready_image.
+    #
+    def save(action_handler)
+      raise "save unimplemented"
+    end
+  end
+end

data/lib/chef_metal/recipe_dsl.rb

@@ -9,6 +9,8 @@ require 'chef/resource/machine_file'
 require 'chef/provider/machine_file'
 require 'chef/resource/machine_execute'
 require 'chef/provider/machine_execute'
+require 'chef/resource/machine_image'
+require 'chef/provider/machine_image'
 
 class Chef
   module DSL
@@ -29,6 +31,14 @@ class Chef
         run_context.chef_metal.add_machine_options(options, &block)
       end
 
+      def with_image_options(image_options, &block)
+        run_context.chef_metal.with_image_options(image_options, &block)
+      end
+
+      def current_image_options
+        run_context.chef_metal.current_image_options
+      end
+
       NOT_PASSED = Object.new
 
       def auto_batch_machines(value = NOT_PASSED)

data/lib/chef_metal/transport/ssh.rb

@@ -1,4 +1,5 @@
 require 'chef_metal/transport'
+require 'chef/log'
 require 'uri'
 require 'socket'
 require 'timeout'
@@ -108,14 +109,20 @@ module ChefMetal
 
       def make_url_available_to_remote(local_url)
         uri = URI(local_url)
-        host = Socket.getaddrinfo(uri.host, uri.scheme, nil, :STREAM)[0][3]
-        if host == '127.0.0.1' || host == '::1'
-          unless session.forward.active_remotes.any? { |port, bind| port == uri.port && bind == uri.host }
-            Chef::Log.debug("Forwarding local server #{uri.host}:#{uri.port} to port #{uri.port} on #{username}@#{self.host}")
-            session.forward.remote(uri.port, uri.host, uri.port)
+        if is_local_machine(uri.host)
+          port, host = forward_port(uri.port, uri.host, uri.port, 'localhost')
+          if !port
+            # Try harder if the port is already taken
+            port, host = forward_port(uri.port, uri.host, 0, 'localhost')
+            if !port
+              raise "Error forwarding port: could not forward #{uri.port} or 0"
+            end
           end
+          uri.host = host
+          uri.port = port
         end
-        local_url
+        Chef::Log.info("Port forwarded: local URL #{local_url} is available to #{self.host} as #{uri.to_s} for the duration of this SSH connection.")
+        uri.to_s
       end
 
       def disconnect
@@ -232,6 +239,50 @@ module ChefMetal
           raise InitialConnectTimeout.new($!)
         end
       end
+
+      def is_local_machine(host)
+        local_addrs = Socket.ip_address_list
+        host_addrs = Addrinfo.getaddrinfo(host, nil)
+        local_addrs.any? do |local_addr|
+          host_addrs.any? do |host_addr|
+            local_addr.ip_address == host_addr.ip_address
+          end
+        end
+      end
+
+      # Forwards a port over the connection, and returns the
+      def forward_port(local_port, local_host, remote_port, remote_host)
+        # This bit is from the documentation.
+        if session.forward.respond_to?(:active_remote_destinations)
+          got_remote_port, remote_host = session.forward.active_remote_destinations[[local_port, local_host]]
+          if !got_remote_port
+            Chef::Log.debug("Forwarding local server #{local_host}:#{local_port} to #{username}@#{self.host}")
+
+            session.forward.remote(local_port, local_host, remote_port, remote_host) do |actual_remote_port|
+              got_remote_port = actual_remote_port || :error
+              :no_exception # I'll take care of it myself, thanks
+            end
+            # Kick SSH until we get a response
+            session.loop { !got_remote_port }
+            if got_remote_port == :error
+              return nil
+            end
+          end
+          [ got_remote_port, remote_host ]
+        else
+          @forwarded_ports ||= {}
+          remote_port, remote_host = @forwarded_ports[[local_port, local_host]]
+          if !remote_port
+            Chef::Log.debug("Forwarding local server #{local_host}:#{local_port} to #{username}@#{self.host}")
+            old_active_remotes = session.forward.active_remotes
+            session.forward.remote(local_port, local_host, local_port)
+            session.loop { !(session.forward.active_remotes.length > old_active_remotes.length) }
+            remote_port, remote_host = (session.forward.active_remotes - old_active_remotes).first
+            @forwarded_ports[[local_port, local_host]] = [ remote_port, remote_host ]
+          end
+          [ remote_port, remote_host ]
+        end
+      end
     end
   end
 end

data/lib/chef_metal/version.rb

@@ -1,3 +1,3 @@
 module ChefMetal
-  VERSION = '0.13'
+  VERSION = '0.14'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chef-metal
 version: !ruby/object:Gem::Version
-  version: '0.13'
+  version: '0.14'
 platform: ruby
 authors:
 - John Keiser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-15 00:00:00.000000000 Z
+date: 2014-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -155,13 +155,16 @@ files:
 - lib/chef/provider/machine_batch.rb
 - lib/chef/provider/machine_execute.rb
 - lib/chef/provider/machine_file.rb
+- lib/chef/provider/machine_image.rb
 - lib/chef/resource/machine.rb
 - lib/chef/resource/machine_batch.rb
 - lib/chef/resource/machine_execute.rb
 - lib/chef/resource/machine_file.rb
+- lib/chef/resource/machine_image.rb
 - lib/chef_metal.rb
 - lib/chef_metal/action_handler.rb
 - lib/chef_metal/add_prefix_action_handler.rb
+- lib/chef_metal/chef_image_spec.rb
 - lib/chef_metal/chef_machine_spec.rb
 - lib/chef_metal/chef_provider_action_handler.rb
 - lib/chef_metal/chef_run_data.rb
@@ -172,6 +175,7 @@ files:
 - lib/chef_metal/convergence_strategy/no_converge.rb
 - lib/chef_metal/convergence_strategy/precreate_chef_objects.rb
 - lib/chef_metal/driver.rb
+- lib/chef_metal/image_spec.rb
 - lib/chef_metal/machine.rb
 - lib/chef_metal/machine/basic_machine.rb
 - lib/chef_metal/machine/unix_machine.rb