cluster_chef 3.0.5

data/lib/cluster_chef/discovery.rb

@@ -0,0 +1,158 @@
+module ClusterChef
+  class Cluster
+
+    def discover!
+      @aws_instance_hash = {}
+      discover_cluster_chef!
+      discover_chef_nodes!
+      discover_fog_servers!
+      discover_chef_clients!
+      discover_volumes!
+    end
+
+    def chef_clients
+      return @chef_clients if @chef_clients
+      @chef_clients = []
+      Chef::Search::Query.new.search(:client, "clientname:#{cluster_name}-*") do |client_hsh|
+        # Return values from Chef::Search seem to be inconsistent across chef
+        # versions (sometimes a hash, sometimes an object). Fix if necessary.
+        client_hsh = Chef::ApiClient.json_create(client_hsh) unless client_hsh.is_a?(Chef::ApiClient)
+        @chef_clients.push( client_hsh )
+      end
+      @chef_clients
+    end
+
+    # returns client with the given name if in catalog, nil otherwise
+    def find_client(cl_name)
+      chef_clients.find{|ccl| ccl.name == cl_name }
+    end
+
+    def chef_nodes
+      return @chef_nodes if @chef_nodes
+      @chef_nodes = []
+      Chef::Search::Query.new.search(:node,"cluster_name:#{cluster_name}") do |n|
+        @chef_nodes.push(n) unless n.blank? || (n.cluster_name != cluster_name.to_s)
+      end
+      @chef_nodes
+    end
+
+    # returns node with the given name if in catalog, nil otherwise
+    def find_node(nd_name)
+      chef_nodes.find{|nd| nd.name == nd_name }
+    end
+
+  protected
+
+    def fog_servers
+      @fog_servers ||= ClusterChef.fog_servers.select{|fs| fs.key_name == cluster_name.to_s && (fs.state != "terminated") }
+    end
+
+    # Walk the list of chef nodes and
+    # * vivify the server,
+    # * associate the chef node
+    # * if the chef node knows about its instance id, memorize that for lookup
+    #   when we discover cloud instances.
+    def discover_chef_nodes!
+      chef_nodes.each do |chef_node|
+        if chef_node["cluster_name"] && chef_node["facet_name"] && chef_node["facet_index"]
+          cluster_name = chef_node["cluster_name"]
+          facet_name   = chef_node["facet_name"]
+          facet_index  = chef_node["facet_index"]
+        elsif chef_node.name
+          ( cluster_name, facet_name, facet_index ) = chef_node.name.split(/-/)
+        else
+          next
+        end
+        svr = ClusterChef::Server.get(cluster_name, facet_name, facet_index)
+        svr.chef_node = chef_node
+        @aws_instance_hash[ chef_node.ec2.instance_id ] = svr if chef_node[:ec2] && chef_node.ec2.instance_id
+      end
+    end
+
+    # Walk the list of servers, asking each to discover its chef client.
+    def discover_chef_clients!
+      servers.each(&:chef_client)
+    end
+
+    # calling #servers vivifies each facet's ClusterChef::Server instances
+    def discover_cluster_chef!
+      self.servers
+    end
+
+    def discover_fog_servers!
+      # If the fog server is tagged with cluster/facet/index, then try to
+      # locate the corresponding machine in the cluster def
+      # Otherwise, try to get to it through mapping the aws instance id
+      # to the chef node name found in the chef node
+      fog_servers.each do |fs|
+        if fs.tags["cluster"] && fs.tags["facet"] && fs.tags["index"] && fs.tags["cluster"] == cluster_name.to_s
+          svr = ClusterChef::Server.get(fs.tags["cluster"], fs.tags["facet"], fs.tags["index"])
+        elsif @aws_instance_hash[fs.id]
+          svr = @aws_instance_hash[fs.id]
+        else
+          next
+        end
+
+        # If there already is a fog server there, then issue a warning and slap
+        # the just-discovered one onto a server with an arbitrary index, and
+        # mark both bogus
+        if existing_fs = svr.fog_server
+          if existing_fs.id != fs.id
+            ui.warn "Duplicate fog instance found for #{svr.fullname}: #{fs.id} and #{existing_fs.id}!!"
+            old_svr = svr
+            svr     = old_svr.facet.server(1_000 + svr.facet_index.to_i)
+            old_svr.bogosity :duplicate
+            svr.bogosity     :duplicate
+          end
+        end
+        svr.fog_server = fs
+      end
+    end
+
+    def discover_volumes!
+      servers.each(&:discover_volumes!)
+    end
+
+    def discover_addresses!
+      servers.each(&:discover_addresses!)
+    end
+  end
+
+  def self.fog_connection
+    @fog_connection ||= Fog::Compute.new({
+        :provider              => 'AWS',
+        :aws_access_key_id     => Chef::Config[:knife][:aws_access_key_id],
+        :aws_secret_access_key => Chef::Config[:knife][:aws_secret_access_key],
+        :region                => Chef::Config[:knife][:region]
+      })
+  end
+
+  def self.fog_servers
+    return @fog_servers if @fog_servers
+    Chef::Log.debug("Using fog to catalog all servers")
+    @fog_servers = ClusterChef.fog_connection.servers.all
+  end
+
+  def self.fog_volumes
+    return @fog_volumes if @fog_volumes
+    Chef::Log.debug("Using fog to catalog all volumes")
+    @fog_volumes ||= ClusterChef.fog_connection.volumes
+  end
+
+  def self.fog_addresses
+    return @fog_addresses if @fog_addresses
+    Chef::Log.debug("Using fog to catalog all addresses")
+    @fog_addresses = {}.tap{|hsh| ClusterChef.fog_connection.addresses.each{|fa| hsh[fa.public_ip] = fa } }
+  end
+
+  def self.fog_keypairs
+    return @fog_keypairs if @fog_keypairs
+    Chef::Log.debug("Using fog to catalog all keypairs")
+    @fog_keypairs = {}.tap{|hsh| ClusterChef.fog_connection.key_pairs.each{|kp| hsh[kp.name] = kp } }
+  end
+
+  def safely *args, &block
+    ClusterChef.safely(*args, &block)
+  end
+
+end

data/lib/cluster_chef/dsl_object.rb

@@ -0,0 +1,123 @@
+Mash.class_eval do
+  def reverse_merge!(other_hash)
+    # stupid mash doesn't take a block arg, which breaks the implementation of
+    # reverse_merge!
+    other_hash.each_pair do |key, value|
+      key = convert_key(key)
+      regular_writer(key, convert_value(value)) unless has_key?(key)
+    end
+    self
+  end
+  def to_mash
+    self.dup
+  end unless method_defined?(:to_mash)
+end
+
+Hash.class_eval do
+  def to_mash
+    Mash.new(self)
+  end unless method_defined?(:to_mash)
+end
+
+module ClusterChef
+  #
+  # Provides magic methods, defined with has_keys
+  #
+  # @example
+  #   class Mom < ClusterChef::DslObject
+  #     has_keys(:college, :combat_boots, :fat, :so_fat)
+  #   end
+  #
+  #   class Person
+  #     def momma &block
+  #       @momma ||= Mom.new
+  #       @momma.configure(&block) if block
+  #     end
+  #   end
+  #
+  #   yo = Person.new
+  #   yo.mamma.combat_boots :wears
+  #   yo.momma do
+  #     fat    true
+  #     so_fat 'When she sits around the house, she sits *AROUND* the house'
+  #   end
+  #
+  class DslObject
+    class_attribute :keys
+    self.keys = []
+
+    def initialize(attrs={})
+      @settings = attrs.to_mash || Mash.new
+    end
+
+    #
+    # Defines DSL attributes
+    #
+    # @params [Array(String)] key_names DSL attribute names
+    #
+    # @example
+    #   class Mom < ClusterChef::DslObject
+    #     has_keys(:fat, :so_fat)
+    #   end
+    #   yer_mom = Mom.new
+    #   yer_mom.fat :quite
+    #
+    def self.has_keys(*key_names)
+      key_names.map!(&:to_sym)
+      self.keys += key_names
+      self.keys.uniq!
+      key_names.each do |key|
+        next if method_defined?(key)
+        define_method(key){|*args| set(key, *args) }
+      end
+    end
+
+    #
+    # Sets the DSL attribute, unless the given value is nil.
+    #
+    def set(key, val=nil)
+      @settings[key.to_s] = val unless val.nil?
+      @settings[key.to_s]
+    end
+
+    def to_hash
+      @settings.to_hash
+    end
+
+    def to_mash
+      @settings.dup
+    end
+
+    def to_s
+      "<#{self.class} #{to_hash.inspect}>"
+    end
+
+    def reverse_merge!(hsh)
+      @settings.reverse_merge!(hsh.to_hash)
+    end
+
+    def configure(hsh={}, &block)
+      @settings.merge!(hsh.to_hash)
+      instance_eval(&block) if block
+      self
+    end
+
+    # delegate to the knife ui presenter
+    def ui()      ClusterChef.ui ; end
+    # delegate to the knife ui presenter
+    def self.ui() ClusterChef.ui ; end
+
+    def step(desc, *style)
+      ui.info("  #{"%-15s" % (name.to_s+":")}\t#{ui.color(desc.to_s, *style)}")
+    end
+
+    # helper method for bombing out of a script
+    def die(*args) ClusterChef.die(*args) ; end
+
+    # helper method for turning exceptions into warnings
+    def safely(*args, &block) ClusterChef.safely(*args, &block) ; end
+
+    # helper method for debugging only
+    def dump(*args) args.each{|arg| Chef::Log.debug( arg.inspect ) } end
+  end
+end

data/lib/cluster_chef/facet.rb

@@ -0,0 +1,144 @@
+module ClusterChef
+  class Facet < ClusterChef::ComputeBuilder
+    attr_reader :cluster
+    has_keys  :instances
+
+    def initialize cluster, facet_name, attrs={}
+      super(facet_name.to_sym, attrs)
+      @cluster    = cluster
+      @servers    = Mash.new
+      @chef_roles = []
+      @settings[:instances] ||= 0
+      create_facet_role
+      create_facet_security_group unless attrs[:no_security_group]
+    end
+
+    def cluster_name
+      cluster.name
+    end
+
+    def facet_name
+      name
+    end
+
+    # The auto-generated role for this facet.
+    # Instance-evals the given block in the context of that role,
+    #
+    # @example
+    #   facet_role do
+    #     override_attributes({
+    #       :time_machine => { :transition_speed => 88 },
+    #     })
+    #   end
+    #
+    # @return [Chef::Role] The auto-generated role for this facet.
+    def facet_role(&block)
+      @facet_role.instance_eval( &block ) if block_given?
+      @facet_role
+    end
+
+    def assign_volume_ids(volume_name, *volume_ids)
+      volume_ids.flatten.zip(servers).each do |volume_id, server|
+        server.volume(volume_name){ volume_id(volume_id) } if server
+      end
+    end
+
+    #
+    # Retrieve or define the given server
+    #
+    # @param [Integer] idx  -- the index of the desired server
+    # @param [Hash] attrs -- attributes to configure on the object
+    # @yield a block to execute in the context of the object
+    #
+    # @return [ClusterChef::Facet]
+    #
+    def server(idx, attrs={}, &block)
+      idx = idx.to_i
+      @servers[idx] ||= ClusterChef::Server.new(self, idx)
+      @servers[idx].configure(attrs, &block)
+      @servers[idx]
+    end
+
+    # if the server has been added to this facet or is in range
+    def has_server? idx
+      (idx.to_i < instances) || @servers.include?(idx.to_i)
+    end
+
+    #
+    # Slicing
+    #
+
+    # All servers in this facet
+    #
+    # @return [ClusterChef::ServerSlice] slice containing all servers
+    def servers
+      slice(indexes)
+    end
+
+    #
+    # A slice of servers from this facet, in index order
+    #
+    # If +slice_indexes+ is nil, returns all servers.
+    # Otherwise, takes slice (given by +*args+) from the requested facet.
+    #
+    # @param [Array, String] slice_indexes -- servers in that facet (or nil for all in facet).
+    #
+    # @return [ClusterChef::ServerSlice] the requested slice
+    def slice(slice_indexes=nil)
+      slice_indexes = self.indexes if slice_indexes.blank?
+      slice_indexes = indexes_from_intervals(slice_indexes) if slice_indexes.is_a?(String)
+      svrs = Array(slice_indexes).map(&:to_i).sort!.select{|idx| has_server?(idx) }.map{|idx| server(idx) }
+      ClusterChef::ServerSlice.new(self.cluster, svrs)
+    end
+
+    # all valid server indexes
+    def valid_indexes
+      (0 ... instances).to_a # note the '...'
+    end
+
+    # indexes in the 0...instances range plus bogus ones that showed up
+    # (probably from chef or fog)
+    def indexes
+      [@servers.keys, valid_indexes].flatten.compact.uniq.sort
+    end
+
+    #
+    # Resolve:
+    #
+    def resolve!
+      servers.each(&:resolve!)
+    end
+
+  protected
+
+    def create_facet_security_group
+      cloud.security_group("#{cluster_name}-#{facet_name}")
+    end
+
+    # Creates a chef role named for the facet
+    def create_facet_role
+      @facet_role_name = "#{cluster_name}_#{facet_name}"
+      @facet_role      = new_chef_role(@facet_role_name, cluster, self)
+      role(@facet_role_name, :last)
+    end
+
+    #
+    # Given a string enumerating indexes to select returns a flat array of
+    # indexes. The indexes will be unique but in an arbitrary order.
+    #
+    # @example
+    #   facet = ClusterChef::Facet.new('foo', 'bar')
+    #   facet.indexes_from_intervals('1,2-3,8-9,7') # [1, 2, 3, 8, 9, 7]
+    #   facet.indexes_from_intervals('1,3-5,4,7')   # [1, 3, 4, 5, 7]
+    #
+    def indexes_from_intervals intervals
+      intervals.split(",").map do |term|
+        if    term =~ /^(\d+)-(\d+)$/ then ($1.to_i .. $2.to_i).to_a
+        elsif term =~ /^(\d+)$/       then  $1.to_i
+        else  ui.warn("Bad interval: #{term}") ; nil
+        end
+      end.flatten.compact.uniq
+    end
+
+  end
+end

data/lib/cluster_chef/fog_layer.rb

@@ -0,0 +1,134 @@
+module ClusterChef
+  #
+  # ClusterChef::Server methods that handle Fog action
+  #
+  Server.class_eval do
+
+    def fog_create_server
+      step(" creating cloud server", :green)
+      fog_description = fog_description_for_launch
+      Chef::Log.debug(JSON.pretty_generate(fog_description))
+      safely do
+        @fog_server = ClusterChef.fog_connection.servers.create(fog_description)
+      end
+    end
+
+    def fog_description_for_launch
+      {
+        :image_id          => cloud.image_id,
+        :flavor_id         => cloud.flavor,
+        #
+        :groups            => cloud.security_groups.keys,
+        :key_name          => cloud.keypair.to_s,
+        # Fog does not actually create tags when it creates a server.
+        :tags              => {
+          :cluster => cluster_name,
+          :facet   => facet_name,
+          :index   => facet_index, },
+        :user_data         => JSON.pretty_generate(cloud.user_data),
+        :block_device_mapping    => block_device_mapping,
+        # :disable_api_termination => cloud.permanent,
+        # :instance_initiated_shutdown_behavior => instance_initiated_shutdown_behavior,
+        :availability_zone => self.default_availability_zone,
+        :monitoring => cloud.monitoring,
+      }
+    end
+
+    #
+    # Takes key-value pairs and idempotently sets those tags on the cloud machine
+    #
+    def fog_create_tags(fog_obj, desc, tags)
+      tags.each do |key, value|
+        next if fog_obj.tags[key] == value.to_s
+        Chef::Log.debug( "tagging #{key} = #{value} on #{desc}" )
+        safely do
+          ClusterChef.fog_connection.tags.create({
+            :key => key, :value => value.to_s, :resource_id => fog_obj.id })
+        end
+      end
+    end
+
+    def fog_address
+      address_str = self.cloud.public_ip or return
+      ClusterChef.fog_addresses[address_str]
+    end
+
+    def discover_volumes!
+      composite_volumes.each do |vol_name, vol|
+        my_vol = volumes[vol_name]
+        next if my_vol.fog_volume
+        my_vol.fog_volume = ClusterChef.fog_volumes.find do |fv|
+          ( # matches the explicit volume id
+            (vol.volume_id && (fv.id == vol.volume_id)    ) ||
+            # OR this server's machine exists, and this volume is attached to
+            # it, and in the right place
+            ( fog_server && fv.server_id && vol.device  &&
+              (fv.server_id   == fog_server.id)         &&
+              (fv.device.to_s == vol.device.to_s)         ) ||
+            # OR this volume is tagged as belonging to this machine
+            ( fv.tags.present?                         &&
+              (fv.tags['server'] == self.fullname)     &&
+              (fv.tags['device'] == vol.device.to_s) )
+            )
+        end
+        next unless my_vol.fog_volume
+        my_vol.volume_id(my_vol.fog_volume.id)                        unless my_vol.volume_id.present?
+        my_vol.availability_zone(my_vol.fog_volume.availability_zone) unless my_vol.availability_zone.present?
+        check_server_id_pairing(my_vol.fog_volume, my_vol.desc)
+      end
+    end
+
+    def attach_volumes
+      return unless in_cloud?
+      discover_volumes!
+      return if composite_volumes.empty?
+      step("  attaching volumes")
+      composite_volumes.each do |vol_name, vol|
+        next if vol.volume_id.blank? || (vol.attachable != :ebs)
+        if (not vol.in_cloud?) then  Chef::Log.debug("Volume not found: #{vol.desc}") ; next ; end
+        if (vol.has_server?)   then check_server_id_pairing(vol.fog_volume, vol.desc) ; next ; end
+        step("  - attaching #{vol.desc} -- #{vol.inspect}", :blue)
+        safely do
+          vol.fog_volume.device = vol.device
+          vol.fog_volume.server = fog_server
+        end
+      end
+    end
+
+    def associate_public_ip
+      address = self.cloud.public_ip
+      return unless self.in_cloud? && address
+      desc = "elastic ip #{address} for #{self.fullname}"
+      if (fog_address && fog_address.server_id) then check_server_id_pairing(fog_address, desc) ; return ; end
+      safely do
+        step("  assigning #{desc}", :blue)
+        ClusterChef.fog_connection.associate_address(self.fog_server.id, address)
+      end
+    end
+
+    def check_server_id_pairing thing, desc
+      return unless thing && thing.server_id && self.in_cloud?
+      type_of_thing = thing.class.to_s.gsub(/.*::/,"")
+      if thing.server_id != self.fog_server.id
+        ui.warn "#{type_of_thing} mismatch: #{desc} is on #{thing.server_id} not #{self.fog_server.id}: #{thing.inspect.gsub(/\s+/m,' ')}"
+        false
+      else
+        Chef::Log.debug("#{type_of_thing} paired: #{desc}")
+        true
+      end
+    end
+
+  end
+
+  class ServerSlice
+    def sync_keypairs
+      step("ensuring keypairs exist")
+      keypairs  = servers.map{|svr| [svr.cluster.cloud.keypair, svr.cloud.keypair] }.flatten.map(&:to_s).reject(&:blank?).uniq
+      keypairs  = keypairs - ClusterChef.fog_keypairs.keys
+      keypairs.each do |keypair_name|
+        keypair_obj = ClusterChef::Ec2Keypair.create!(keypair_name)
+        ClusterChef.fog_keypairs[keypair_name] = keypair_obj
+      end
+    end
+  end
+end

data/lib/cluster_chef/private_key.rb

@@ -0,0 +1,110 @@
+require 'fileutils'
+
+module ClusterChef
+  #
+  # A private key -- chef client key, ssh key, etc.
+  #
+  # The key is a pro
+  class PrivateKey < ClusterChef::DslObject
+    attr_reader   :name
+    attr_reader   :proxy
+    attr_reader   :on_update
+
+    #
+    # PrivateKey.new('bob')
+    #
+    # @yield      a block, executed in caller's context, when the body is updated
+    # @yieldparam the updated body
+    def initialize(name, proxy=nil, &on_update)
+      super()
+      @name      = name
+      @proxy     = proxy
+      @on_update = on_update
+    end
+
+    def filename
+      File.join(key_dir, "#{name}.pem")
+    end
+
+    def save
+      return unless @body
+      if ClusterChef.chef_config[:dry_run]
+        Chef::Log.debug("    key #{name} - dry run, not writing out key")
+        return
+      end
+      ui.info( "    key #{name} - writing to #{filename}" )
+      FileUtils.mkdir_p(File.dirname(filename))
+      File.open(filename, "w", 0600){|f| f.print( @body ) }
+    end
+
+    def load
+      return unless File.exists?(filename)
+      self.body = File.read(filename).chomp
+    end
+
+    def body=(content)
+      @body = content
+      on_update.call(content) if on_update
+      content
+    end
+
+    def self.create!(name, *args, &block)
+      obj = self.new(name, *args, &block)
+      obj.create_proxy!
+      obj
+    end
+
+    def to_s
+      [super[0..-2], @name, @proxy, @body.to_s[32..64], '...', @body.to_s[-60..-30]].join(" ").gsub(/[\r\n\t]+/,'') + '>'
+    end
+  end
+
+  class ChefClientKey < PrivateKey
+    def body
+      return @body if @body
+      if proxy && proxy.private_key && (not proxy.private_key.empty?)
+        @body = proxy.private_key
+      else
+        load
+      end
+      @body
+    end
+
+    def key_dir
+      Chef::Config.client_key_dir || '/tmp/client_keys'
+    end
+  end
+
+  class Ec2Keypair < PrivateKey
+    def body
+      return @body if @body
+      if proxy && proxy.private_key && (not proxy.private_key.empty?)
+        @body = proxy.private_key
+      else
+        load
+      end
+      @body
+    end
+
+    def create_proxy!
+      safely do
+        step("    key #{name} - creating", :green)
+        @proxy = ClusterChef.fog_connection.key_pairs.create(:name => name.to_s)
+      end
+      ClusterChef.fog_keypairs[name] = proxy
+      self.body = proxy.private_key
+      save
+    end
+
+    def key_dir
+      if Chef::Config.ec2_key_dir
+        return Chef::Config.ec2_key_dir
+      else
+        dir = "#{ENV['HOME']}/.chef/ec2_keys"
+        warn "Please set 'ec2_key_dir' in your knife.rb -- using #{dir} as a default"
+        dir
+      end
+    end
+  end
+
+end