clc-fork-chef-metal 0.13.alpha.1 → 0.14.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 92fd9ce1269e598cb791750506afc09fe1ebd514
-  data.tar.gz: 51fbc3e51bb706e79995554ee1f9f433077f2b16
+  metadata.gz: d31f76e87ba58390c47ec3723b1365ac0e548d49
+  data.tar.gz: 8c8ee1fb4b04a64cec1a14284662f61cd189a4b2
 SHA512:
-  metadata.gz: 84fc7691daf6f62d12d8f8bed0e621a3c62b2f195efbb6afb9c9efa29d6615290c51894c588bc0005f1ac62e78ebb7aa5d5b10decd2013896401712450e191e1
-  data.tar.gz: f7dfa7d4df6eaecd8635118c9a2afc2e60ae25c5b98b94f1d53a4704d239ea323ff02e3dab61a44b8225fb7eabf99d772548b1d1a24fdc2e439c614cddd44fcc
+  metadata.gz: 183c855125905188c74526fe492b9122bfbe762541d2bdc3432d72561387910a519ddc2d1ebc0c06b7de2a49648c34062f50f8bf154b07e7ede5a9f9d91693ca
+  data.tar.gz: 727800f1ffd242ae4b04758235d4c4531aeefdc747ce7738750b700108561dbab7b3668b58791636fd51c624849f59d3aad5511f5b435c4584b01287cd7d4dd4

data/CHANGELOG.md

@@ -1,5 +1,55 @@
 # Chef Metal Changelog
 
+## 0.14.1 (8/18/2014)
+
+- Fix "metal execute mario ls" to work again
+
+## 0.14 (8/18/2014)
+
+- FEATURE: Add the machine_image resource (@jkeiser, @johnewart):
+  ```ruby
+  machine_image 'base' do
+    machine_options :bootstrap_options => { :image_id => 'ami-1234798123431', :ssh_username => 'root' }
+    recipe 'secure_base'
+    recipe 'corp_users'
+  end
+  # Build an image based on 'base' that has apache
+  machine_image 'apache' do
+    # All bootstrap options, like ssh_username, are carried forward
+    from_image 'base'
+    recipe 'apache2'
+  end
+  # Build an image with my web app based on the apache base image
+  machine_image 'myapp' do
+    from_image 'apache'
+    recipe 'mywebapp'
+  end
+  # Build an image with mysql and my schema based on the corporate base image
+  machine_image 'mydb' do
+    from_image 'base'
+    recipe 'mysql'
+    recipe 'myschema'
+  end
+  # Build a DB machine from mydb.  Does not reinstall stuff! :)
+  machine 'db' do
+    from_image 'mydb'
+  end
+  # Build a web app machine from myapp.  Does not reinstall stuff! :)
+  machine 'myapp1' do
+    from_image 'myapp'
+  end
+  ```
+  - Creates a node with the name of the machine_image, which contains metadata
+    like the username of the image.  This makes things like AWS image registries
+    possible.
+- Fix the no_converge convergence strategy (@johnewart)
+- SSH port forwarding improvements:
+  - Detects *any* IP on the localhost and forwards it--not just 127.0.0.1
+  - Binds to localhost on the remote side instead of 127.0.0.1, allowing for IPv6 communication
+  - Tries multiple ports--if the origin port is already taken, tries "0" (ephemeral).
+- Fix SSH race condition causing port forwarding to happen twice (and fail miserably)
+- Add ChefMetal.connect_to_machine('mario')
+
 ## 0.13 (6/17/2014)
 
 - make winrm work again (@mwrock)

data/README.md

@@ -25,8 +25,6 @@ Try It Out
 
 You can try out Metal in many different flavors.
 
-HINT: chef-metal looks prettiest with chef 11.14 alpha.  `gem install chef --pre` to get it.
-
 ### Vagrant
 
 To give it a spin, install Vagrant and VirtualBox and try this from the `chef-metal/docs/examples` directory:

data/bin/metal

@@ -114,7 +114,7 @@ class ChefMetal::Application < Chef::Application
       case command
       when 'execute'
         connect_to_machines(cli_arguments.shift) do |machine|
-          machine.execute(action_handler, cli_arguments.join(' '), :read_only => config[:read_only], :stream => config[:stream], :timeout => config[:timeout].to_f)
+          result = machine.execute(action_handler, cli_arguments.join(' '), :read_only => config[:read_only], :stream => config[:stream], :timeout => config[:timeout].to_f)
           puts result.stdout if result.stdout != '' && !config[:stream] && Chef::Config.log_level != :debug
           STDERR.puts result.stderr if result.stderr != '' && !config[:stream] && Chef::Config.log_level != :debug
           exit_code = result.exitstatus if result.exitstatus != 0
@@ -246,9 +246,8 @@ class ChefMetal::Application < Chef::Application
   end
 
   def connect_to_machines(spec)
-    grouped = machine_specs(spec).group_by { |machine_spec| machine_spec.driver_url }
-    grouped.collect_concat do |driver_url, machine_specs|
-      machine_specs.map { |machine_spec| ChefMetal.connect_to_machine(machine_spec) }
+    machine_specs(spec).each do |machine_spec|
+      yield ChefMetal.connect_to_machine(machine_spec)
     end
   end
 

data/lib/chef/provider/machine.rb

@@ -94,18 +94,31 @@ class Chef::Provider::Machine < Chef::Provider::LWRPBase
     end
   end
 
+  def from_image_spec
+    @from_image_spec ||= begin
+      if new_resource.from_image
+        ChefMetal::ChefImageSpec.get(new_resource.from_image, new_resource.chef_server)
+      else
+        nil
+      end
+    end
+  end
+
   def new_machine_options
     machine_options(new_driver)
   end
 
   def current_machine_options
-    if current_driver
-      machine_options(current_driver)
-    end
+    machine_options(current_driver)
   end
 
   def machine_options(driver)
     configs = []
+
+    if from_image_spec && from_image_spec.machine_options
+      configs << from_image_spec.machine_options
+    end
+
     configs << {
       :convergence_options =>
         [ :chef_server,

data/lib/chef/provider/machine_image.rb

@@ -0,0 +1,54 @@
+require 'chef/provider/lwrp_base'
+require 'chef/provider/chef_node'
+require 'openssl'
+require 'chef_metal/chef_provider_action_handler'
+require 'chef_metal/chef_image_spec'
+
+class Chef::Provider::MachineImage < Chef::Provider::LWRPBase
+  def action_handler
+    @action_handler ||= ChefMetal::ChefProviderActionHandler.new(self)
+  end
+
+  def load_current_resource
+  end
+
+  # Get the driver specified in the resource
+  def new_driver
+    @new_driver ||= run_context.chef_metal.driver_for(new_resource.driver)
+  end
+
+  action :create do
+    # Get the image mapping on the server (from name to image-id)
+    image_spec = ChefMetal::ChefImageSpec.get(new_resource.name, new_resource.chef_server) ||
+                 ChefMetal::ChefImageSpec.empty(new_resource.name, new_resource.chef_server)
+    if image_spec.location
+      # TODO check for real existence and maybe update
+    else
+      #
+      # Create a new image
+      #
+      image_spec.machine_options = new_resource.machine_options
+      create_image(image_spec)
+    end
+  end
+
+  action :destroy do
+  end
+
+  def create_image(image_spec)
+    # 1. Create the exemplar machine
+    machine_provider = Chef::Provider::Machine.new(new_resource, run_context)
+    machine_provider.load_current_resource
+    machine_provider.action_converge
+
+    # 2. Create the image
+    new_driver.allocate_image(action_handler, image_spec, new_resource.image_options,
+                              machine_provider.machine_spec)
+
+    # 3. Save the linkage from name -> image id
+    image_spec.save(action_handler)
+
+    # 4. Wait for image to be ready
+    new_driver.ready_image(action_handler, image_spec, new_resource.image_options)
+  end
+end

data/lib/chef/resource/machine.rb

@@ -57,6 +57,12 @@ class Chef::Resource::Machine < Chef::Resource::LWRPBase
   # files '/remote/path.txt' => { :content => 'woo' }
   attribute :files, :kind_of => Hash
 
+  # The named machine_image to start from.  Specify the name of a machine_image
+  # object and the default machine_options will be set to use that image.
+  # == Examples
+  # from_image 'company_base_image'
+  attribute :from_image, :kind_of => String
+
   # A single file to upload, in the format REMOTE_PATH, LOCAL_PATH|HASH.
   # This directive may be passed multiple times, and multiple files will be uploaded.
   # == Examples

data/lib/chef/resource/machine_image.rb

@@ -0,0 +1,29 @@
+require 'chef/resource/lwrp_base'
+require 'cheffish'
+require 'chef_metal'
+require 'cheffish/merged_config'
+
+class Chef::Resource::MachineImage < Chef::Resource::Machine
+  self.resource_name = 'machine_image'
+
+  def initialize(*args)
+    super
+    @image_options = run_context.chef_metal.current_image_options
+    @machine_options = run_context.chef_metal.current_machine_options
+  end
+
+  # TODO check if we still have all the actions from machine
+  actions :create, :destroy, :archive
+  default_action :create
+
+  attribute :image_options, :kind_of => Hash
+
+  # TODO
+  # - policy for old version cleanup
+  # - when attributes or recipes of the machine_image changes, or when the actual
+  #   recipes therein change, we will regenerate the image
+  # - attribute to force or turn off image updates
+  # - ability to rebuild new image based on the previous image
+  # - ability to specify whether and how to keep the image-building machine around
+  # - ability to specify whether to leave the chef config around or not
+end

data/lib/chef_metal/action_handler.rb

@@ -42,9 +42,12 @@ module ChefMetal
     # action that needs to be done.
     def perform_action(description)
       if should_perform_actions
-        yield
+        result = yield
+      else
+        result = nil
       end
       performed_action(description)
+      result
     end
 
     # Open a stream which can be printed to and closed

data/lib/chef_metal/chef_image_spec.rb

@@ -0,0 +1,106 @@
+require 'chef_metal'
+require 'cheffish'
+require 'chef_metal/image_spec'
+
+module ChefMetal
+  #
+  # Specification for a image. Sufficient information to find and contact it
+  # after it has been set up.
+  #
+  class ChefImageSpec < ImageSpec
+    def initialize(node, chef_server)
+      super(node)
+      @chef_server = chef_server
+    end
+
+    #
+    # Get a ImageSpec from the chef server.  If the node does not exist on the
+    # server, it returns nil.
+    #
+    def self.get(name, chef_server = Cheffish.default_chef_server)
+      chef_api = Cheffish.chef_server_api(chef_server)
+      begin
+        data = chef_api.get("/data/images/#{name}")
+        data['machine_options'] = strings_to_symbols(data['machine_options'])
+        ChefImageSpec.new(data, chef_server)
+      rescue Net::HTTPServerException => e
+        if e.response.code == '404'
+          nil
+        else
+          raise
+        end
+      end
+    end
+
+    # Creates a new empty ImageSpec with the given name.
+    def self.empty(id, chef_server = Cheffish.default_chef_server)
+      ChefImageSpec.new({ 'id' => id }, chef_server)
+    end
+
+    #
+    # Globally unique identifier for this image. Does not depend on the image's
+    # location or existence.
+    #
+    def id
+      ChefImageSpec.id_from(chef_server, name)
+    end
+
+    def self.id_from(chef_server, name)
+      "#{chef_server[:chef_server_url]}/data/images/#{name}"
+    end
+
+    #
+    # Save this node to the server.  If you have significant information that
+    # could be lost, you should do this as quickly as possible.  Data will be
+    # saved automatically for you after allocate_image and ready_image.
+    #
+    def save(action_handler)
+      # Save the node to the server.
+      _self = self
+      _chef_server = _self.chef_server
+      ChefMetal.inline_resource(action_handler) do
+        chef_data_bag_item _self.name do
+          data_bag 'images'
+          chef_server _chef_server
+          raw_data _self.image_data
+        end
+      end
+    end
+
+    def delete(action_handler)
+      # Save the node to the server.
+      _self = self
+      _chef_server = _self.chef_server
+      ChefMetal.inline_resource(action_handler) do
+        chef_data_bag_item _self.name do
+          data_bag 'images'
+          chef_server _chef_server
+          action :destroy
+        end
+      end
+    end
+
+    protected
+
+    attr_reader :chef_server
+
+    #
+    # Chef API object for the given Chef server
+    #
+    def chef_api
+      Cheffish.server_api_for(chef_server)
+    end
+
+    def self.strings_to_symbols(data)
+      if data.is_a?(Hash)
+        result = {}
+        data.each_pair do |key, value|
+          result[key.to_sym] = strings_to_symbols(value)
+        end
+        result
+      else
+        data
+      end
+    end
+  end
+end

data/lib/chef_metal/chef_machine_spec.rb

@@ -1,3 +1,4 @@
+require 'pry'
 require 'chef_metal'
 require 'cheffish'
 require 'chef_metal/machine_spec'
@@ -53,6 +54,9 @@ module ChefMetal
     # saved automatically for you after allocate_machine and ready_machine.
     #
     def save(action_handler)
+      if location && (!location.is_a?(Hash) || !location['driver_url'])
+        raise "Drivers must specify a canonical driver_url in machine_spec.location.  Contact your driver's author."
+      end
       # Save the node to the server.
       _self = self
       _chef_server = _self.chef_server

data/lib/chef_metal/chef_run_data.rb

@@ -16,6 +16,7 @@ module ChefMetal
     attr_reader :current_driver
 
     with :machine_options
+    with :image_options
 
     def with_driver(driver, options = nil, &block)
       if drivers[driver] && options
@@ -49,6 +50,14 @@ module ChefMetal
       end
     end
 
+    def current_image_options
+      if @current_image_options
+        @current_image_options
+      else
+        {}
+      end
+    end
+
     def add_machine_options(options, &block)
       with_machine_options(Chef::Mixin::DeepMerge.hash_only_merge(current_machine_options, options), &block)
     end

data/lib/chef_metal/convergence_strategy/no_converge.rb

@@ -14,7 +14,6 @@ module ChefMetal
       end
 
       def setup_convergence(action_handler, machine)
-        machine_spec.save(action_handler)
       end
 
       def converge(action_handler, machine)

data/lib/chef_metal/driver.rb

@@ -182,6 +182,27 @@ module ChefMetal
       raise "#{self.class} does not implement stop_machine"
     end
 
+    #
+    # Allocate an image. Returns quickly with an ID that tracks the image.
+    #
+    def allocate_image(action_handler, image_spec, image_options, machine_spec)
+      raise "#{self.class} does not implement create_image"
+    end
+
+    #
+    # Ready an image, waiting till the point where it is ready to be used.
+    #
+    def ready_image(action_handler, image_spec, image_options)
+      raise "#{self.class} does not implement ready_image"
+    end
+
+    #
+    # Destroy an image.
+    #
+    def destroy_image(action_handler, image_spec, image_options)
+      raise "#{self.class} does not implement destroy_image"
+    end
+
     #
     # Optional interface methods
     #

data/lib/chef_metal/image_spec.rb

@@ -0,0 +1,70 @@
+module ChefMetal
+  #
+  # Specification for a image. Sufficient information to find and contact it
+  # after it has been set up.
+  #
+  class ImageSpec
+    def initialize(image_data)
+      @image_data = image_data
+    end
+
+    attr_reader :image_data
+
+    #
+    # Globally unique identifier for this image. Does not depend on the image's
+    # location or existence.
+    #
+    def id
+      raise "id unimplemented"
+    end
+
+    #
+    # Name of the image. Corresponds to the name in "image 'name' do" ...
+    #
+    def name
+      image_data['id']
+    end
+
+    #
+    # Location of this image. This should be a freeform hash, with enough
+    # information for the driver to look it up and create a image object to
+    # access it.
+    #
+    # This MUST include a 'driver_url' attribute with the driver's URL in it.
+    #
+    # chef-metal will do its darnedest to not lose this information.
+    #
+    def location
+      image_data['location']
+    end
+
+    #
+    # Set the location for this image.
+    #
+    def location=(value)
+      image_data['location'] = value
+    end
+
+    def machine_options
+      image_data['machine_options']
+    end
+
+    def machine_options=(value)
+      image_data['machine_options'] = value
+    end
+
+    # URL to the driver.  Convenience for location['driver_url']
+    def driver_url
+      location ? location['driver_url'] : nil
+    end
+
+    #
+    # Save this image_data to the server.  If you have significant information that
+    # could be lost, you should do this as quickly as possible.  image_data will be
+    # saved automatically for you after allocate_image and ready_image.
+    #
+    def save(action_handler)
+      raise "save unimplemented"
+    end
+  end
+end

data/lib/chef_metal/recipe_dsl.rb

@@ -9,6 +9,8 @@ require 'chef/resource/machine_file'
 require 'chef/provider/machine_file'
 require 'chef/resource/machine_execute'
 require 'chef/provider/machine_execute'
+require 'chef/resource/machine_image'
+require 'chef/provider/machine_image'
 
 class Chef
   module DSL
@@ -29,6 +31,14 @@ class Chef
         run_context.chef_metal.add_machine_options(options, &block)
       end
 
+      def with_image_options(image_options, &block)
+        run_context.chef_metal.with_image_options(image_options, &block)
+      end
+
+      def current_image_options
+        run_context.chef_metal.current_image_options
+      end
+
       NOT_PASSED = Object.new
 
       def auto_batch_machines(value = NOT_PASSED)

data/lib/chef_metal/transport/ssh.rb

@@ -141,7 +141,7 @@ module ChefMetal
         # If you can't pwd within 10 seconds, you can't pwd
         execute('pwd', :timeout => 10)
         true
-      rescue Timeout::Error, Errno::EHOSTUNREACH, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ECONNRESET, Net::SSH::Disconnect
+      rescue Timeout::Error, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::EHOSTDOWN, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::ECONNRESET, Net::SSH::Disconnect
         Chef::Log.debug("#{username}@#{host} unavailable: network connection failed or broke: #{$!.inspect}")
         disconnect
         false

data/lib/chef_metal/transport/winrm.rb

@@ -19,6 +19,7 @@ module ChefMetal
 
       def execute(command, execute_options = {})
         output = with_execute_timeout(execute_options) do
+          session.set_timeout(execute_timeout(execute_options))
           session.run_powershell_script(command) do |stdout, stderr|
             stream_chunk(execute_options, stdout, stderr)
           end

data/lib/chef_metal/version.rb

@@ -1,3 +1,3 @@
 module ChefMetal
-  VERSION = '0.13.alpha.1'
+  VERSION = '0.14.alpha.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clc-fork-chef-metal
 version: !ruby/object:Gem::Version
-  version: 0.13.alpha.1
+  version: 0.14.alpha.1
 platform: ruby
 authors:
 - John Keiser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-01 00:00:00.000000000 Z
+date: 2014-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -155,13 +155,16 @@ files:
 - lib/chef/provider/machine_batch.rb
 - lib/chef/provider/machine_execute.rb
 - lib/chef/provider/machine_file.rb
+- lib/chef/provider/machine_image.rb
 - lib/chef/resource/machine.rb
 - lib/chef/resource/machine_batch.rb
 - lib/chef/resource/machine_execute.rb
 - lib/chef/resource/machine_file.rb
+- lib/chef/resource/machine_image.rb
 - lib/chef_metal.rb
 - lib/chef_metal/action_handler.rb
 - lib/chef_metal/add_prefix_action_handler.rb
+- lib/chef_metal/chef_image_spec.rb
 - lib/chef_metal/chef_machine_spec.rb
 - lib/chef_metal/chef_provider_action_handler.rb
 - lib/chef_metal/chef_run_data.rb
@@ -172,6 +175,7 @@ files:
 - lib/chef_metal/convergence_strategy/no_converge.rb
 - lib/chef_metal/convergence_strategy/precreate_chef_objects.rb
 - lib/chef_metal/driver.rb
+- lib/chef_metal/image_spec.rb
 - lib/chef_metal/machine.rb
 - lib/chef_metal/machine/basic_machine.rb
 - lib/chef_metal/machine/unix_machine.rb