ironfan 3.1.0.rc1

data/lib/ironfan/cluster.rb

@@ -0,0 +1,118 @@
+module Ironfan
+  #
+  # A cluster has many facets. Any setting applied here is merged with the facet
+  # at resolve time; if the facet explicitly sets any attributes they will win out.
+  #
+  class Cluster < Ironfan::ComputeBuilder
+    attr_reader :facets, :undefined_servers
+
+    def initialize(name, attrs={})
+      super(name.to_sym, attrs)
+      @facets            = Mash.new
+      @chef_roles        = []
+      environment          :_default if environment.blank?
+      create_cluster_role
+      create_cluster_security_group unless attrs[:no_security_group]
+    end
+
+    def cluster
+      self
+    end
+
+    def cluster_name
+      name
+    end
+
+    # The auto-generated role for this cluster.
+    # Instance-evals the given block in the context of that role
+    #
+    # @example
+    #   cluster_role do
+    #     override_attributes({
+    #       :time_machine => { :transition_speed => 88 },
+    #     })
+    #   end
+    #
+    # @return [Chef::Role] The auto-generated role for this facet.
+    def cluster_role(&block)
+      @cluster_role.instance_eval( &block ) if block_given?
+      @cluster_role
+    end
+
+    #
+    # Retrieve or define the given facet
+    #
+    # @param [String] facet_name -- name of the desired facet
+    # @param [Hash] attrs -- attributes to configure on the object
+    # @yield a block to execute in the context of the object
+    #
+    # @return [Ironfan::Facet]
+    #
+    def facet(facet_name, attrs={}, &block)
+      facet_name = facet_name.to_sym
+      @facets[facet_name] ||= Ironfan::Facet.new(self, facet_name)
+      @facets[facet_name].configure(attrs, &block)
+      @facets[facet_name]
+    end
+
+    def has_facet? facet_name
+      @facets.include?(facet_name)
+    end
+
+    def find_facet(facet_name)
+      @facets[facet_name] or raise("Facet '#{facet_name}' is not defined in cluster '#{cluster_name}'")
+    end
+
+    # All servers in this facet, sorted by facet name and index
+    #
+    # @return [Ironfan::ServerSlice] slice containing all servers
+    def servers
+      svrs = @facets.sort.map{|name, facet| facet.servers.to_a }
+      Ironfan::ServerSlice.new(self, svrs.flatten)
+    end
+
+    #
+    # A slice of a cluster:
+    #
+    # If +facet_name+ is nil, returns all servers.
+    # Otherwise, takes slice (given by +*args+) from the requested facet.
+    #
+    # @param [String] facet_name -- facet to slice (or nil for all in cluster)
+    # @param [Array, String] slice_indexes -- servers in that facet (or nil for all in facet).
+    #   You must specify a facet if you use slice_indexes.
+    #
+    # @return [Ironfan::ServerSlice] the requested slice
+    def slice facet_name=nil, slice_indexes=nil
+      return Ironfan::ServerSlice.new(self, self.servers) if facet_name.nil?
+      find_facet(facet_name).slice(slice_indexes)
+    end
+
+    def to_s
+      "#{super[0..-3]} @facets=>#{@facets.keys.inspect}}>"
+    end
+
+    #
+    # Resolve:
+    #
+    def resolve!
+      facets.values.each(&:resolve!)
+    end
+
+  protected
+
+    # Create a security group named for the cluster
+    # that is friends with everything in the cluster
+    def create_cluster_security_group
+      clname = self.name # put it in scope
+      cloud.security_group(clname){ authorize_group(clname) }
+    end
+
+    # Creates a chef role named for the cluster
+    def create_cluster_role
+      @cluster_role_name = "#{name}_cluster"
+      @cluster_role      = new_chef_role(@cluster_role_name, cluster)
+      role(@cluster_role_name, :own)
+    end
+
+  end
+end

data/lib/ironfan/compute.rb

@@ -0,0 +1,153 @@
+module Ironfan
+  #
+  # Base class allowing us to layer settings for facet over cluster
+  #
+  class ComputeBuilder < Ironfan::DslObject
+    attr_reader :cloud, :volumes, :chef_roles
+    has_keys :name, :bogosity, :environment
+    @@role_implications ||= Mash.new
+    @@run_list_rank     ||= 0
+
+    def initialize(builder_name, attrs={})
+      super(attrs)
+      set :name, builder_name
+      @run_list_info = attrs[:run_list] || Mash.new
+      @volumes = Mash.new
+    end
+
+    # set the bogosity to a descriptive reason. Anything truthy implies bogusness
+    def bogus?
+      !! self.bogosity
+    end
+
+    # Magic method to produce cloud instance:
+    # * returns the cloud instance, creating it if necessary.
+    # * executes the block in the cloud's object context
+    #
+    # @example
+    #   cloud do
+    #     image_name     'maverick'
+    #     security_group :nagios
+    #   end
+    #
+    #   # defines ec2-specific behavior
+    #   cloud(:ec2) do
+    #     public_ip      '1.2.3.4'
+    #     region         'us-east-1d'
+    #   end
+    #
+    def cloud(cloud_provider=nil, attrs={}, &block)
+      raise "Only have ec2 so far" if cloud_provider && (cloud_provider != :ec2)
+      @cloud ||= Ironfan::Cloud::Ec2.new(self)
+      @cloud.configure(attrs, &block)
+      @cloud
+    end
+
+    # sugar for cloud(:ec2)
+    def ec2(attrs={}, &block)
+      cloud(:ec2, attrs, &block)
+    end
+
+    # Magic method to describe a volume
+    # * returns the named volume, creating it if necessary.
+    # * executes the block (if any) in the volume's context
+    #
+    # @example
+    #   # a 1 GB volume at '/data' from the given snapshot
+    #   volume(:data) do
+    #     size        1
+    #     mount_point '/data'
+    #     snapshot_id 'snap-12345'
+    #   end
+    #
+    # @param volume_name [String] an arbitrary handle -- you can use the device
+    #   name, or a descriptive symbol.
+    # @param attrs [Hash] a hash of attributes to pass down.
+    #
+    def volume(volume_name, attrs={}, &block)
+      volumes[volume_name] ||= Ironfan::Volume.new(:parent => self, :name => volume_name)
+      volumes[volume_name].configure(attrs, &block)
+      volumes[volume_name]
+    end
+
+    def raid_group(rg_name, attrs={}, &block)
+      volumes[rg_name] ||= Ironfan::RaidGroup.new(:parent => self, :name => rg_name)
+      volumes[rg_name].configure(attrs, &block)
+      volumes[rg_name].sub_volumes.each do |sv_name|
+        volume(sv_name){ in_raid(rg_name) ; mountable(false) ; tags({}) }
+      end
+      volumes[rg_name]
+    end
+
+    def root_volume(attrs={}, &block)
+      volume(:root, attrs, &block)
+    end
+
+    #
+    # Adds the given role to the run list, and invokes any role_implications it
+    # implies (for instance, defining and applying the 'ssh' security group if
+    # the 'ssh' role is applied.)
+    #
+    # You can specify placement of `:first`, `:normal` (or nil) or `:last`; the
+    # final runlist is assembled as
+    #
+    # * run_list :first  items -- cluster, then facet, then server
+    # * run_list :normal items -- cluster, then facet, then server
+    # * run_list :last   items -- cluster, then facet, then server
+    #
+    # (see Ironfan::Server#combined_run_list for full details though)
+    #
+    def role(role_name, placement=nil)
+      add_to_run_list("role[#{role_name}]", placement)
+      self.instance_eval(&@@role_implications[role_name]) if @@role_implications[role_name]
+    end
+
+    # Add the given recipe to the run list. You can specify placement of
+    # `:first`, `:normal` (or nil) or `:last`; the final runlist is assembled as
+    #
+    # * run_list :first  items -- cluster, then facet, then server
+    # * run_list :normal items -- cluster, then facet, then server
+    # * run_list :last   items -- cluster, then facet, then server
+    #
+    # (see Ironfan::Server#combined_run_list for full details though)
+    #
+    def recipe(name, placement=nil)
+      add_to_run_list(name, placement)
+    end
+
+    # Roles and recipes for this element only.
+    #
+    # See Ironfan::Server#combined_run_list for run_list order resolution
+    def run_list
+      groups = run_list_groups
+      [ groups[:first], groups[:normal], groups[:last] ].flatten.compact.uniq
+    end
+
+    # run list elements grouped into :first, :normal and :last
+    def run_list_groups
+      @run_list_info.keys.sort_by{|item| @run_list_info[item][:rank] }.group_by{|item| @run_list_info[item][:placement] }
+    end
+
+    #
+    # Some roles imply aspects of the machine that have to exist at creation.
+    # For instance, on an ec2 machine you may wish the 'ssh' role to imply a
+    # security group explicity opening port 22.
+    #
+    # @param [String] role_name -- the role that triggers the block
+    # @yield block will be instance_eval'd in the object that calls 'role'
+    #
+    def self.role_implication(name, &block)
+      @@role_implications[name] = block
+    end
+
+  protected
+
+    def add_to_run_list(item, placement)
+      raise "run_list placement must be one of :first, :normal, :last or nil (also means :normal)" unless [:first, :last, :own, nil].include?(placement)
+      @@run_list_rank += 1
+      placement ||= :normal
+      @run_list_info[item] ||= { :rank => @@run_list_rank, :placement => placement }
+    end
+
+  end
+end

data/lib/ironfan/deprecated.rb

@@ -0,0 +1,33 @@
+module Ironfan
+
+  class Cluster
+    #
+    # **DEPRECATED**: This doesn't really work -- use +reverse_merge!+ instead
+    #
+    def use(*clusters)
+      ui.warn "The 'use' statement is deprecated #{caller.inspect}"
+      clusters.each do |c|
+        other_cluster =  Ironfan.load_cluster(c)
+        reverse_merge! other_cluster
+      end
+      self
+    end
+
+  end
+
+  class Server
+    # **DEPRECATED**: Please use +fullname+ instead.
+    def chef_node_name name
+      ui.warn "[DEPRECATION] `chef_node_name` is deprecated.  Please use `fullname` instead."
+      fullname name
+    end
+  end
+
+  class Cloud::Ec2
+    # **DEPRECATED**: Please use +public_ip+ instead.
+    def elastic_ip(*args, &block)
+      public_ip(*args, &block)
+    end
+  end
+
+end

data/lib/ironfan/discovery.rb

@@ -0,0 +1,177 @@
+module Ironfan
+  class Cluster
+
+    def discover!
+      @aws_instance_hash = {}
+      discover_ironfan!
+      discover_chef_nodes!
+      discover_fog_servers!  unless Ironfan.chef_config[:cloud] == false
+      discover_chef_clients!
+      discover_volumes!
+    end
+
+    def chef_clients
+      return @chef_clients if @chef_clients
+      @chef_clients = []
+
+      # Oh for fuck's sake -- the key used to index clients changed from
+      # 'clientname' in 0.10.4-and-prev to 'name' in 0.10.8. Rather than index
+      # both 'clientname' and 'name', they switched it -- so we have to fall
+      # back.  FIXME: While the Opscode platform is 0.10.4 I have clientname
+      # first (sorry, people of the future). When it switches to 0.10.8 we'll
+      # reverse them (suck it people of the past).
+      # Also sometimes the server returns results that are nil on
+      # recently-expired clients, so that's annoying too.
+      clients, wtf, num = Chef::Search::Query.new.search(:client, "clientname:#{cluster_name}-*") ; clients.compact!
+      clients, wtf, num = Chef::Search::Query.new.search(:client, "name:#{cluster_name}-*") if clients.blank?
+      clients.each do |client_hsh|
+        next if client_hsh.nil?
+        # Return values from Chef::Search seem to be inconsistent across chef
+        # versions (sometimes a hash, sometimes an object). Fix if necessary.
+        client_hsh = Chef::ApiClient.json_create(client_hsh) unless client_hsh.is_a?(Chef::ApiClient)
+        @chef_clients.push( client_hsh )
+      end
+      @chef_clients
+    end
+
+    # returns client with the given name if in catalog, nil otherwise
+    def find_client(cl_name)
+      chef_clients.find{|ccl| ccl.name == cl_name }
+    end
+
+    def chef_nodes
+      return @chef_nodes if @chef_nodes
+      @chef_nodes = []
+      Chef::Search::Query.new.search(:node,"cluster_name:#{cluster_name}") do |n|
+        @chef_nodes.push(n) unless n.blank? || (n.cluster_name != cluster_name.to_s)
+      end
+      @chef_nodes
+    end
+
+    # returns node with the given name if in catalog, nil otherwise
+    def find_node(nd_name)
+      chef_nodes.find{|nd| nd.name == nd_name }
+    end
+
+  protected
+
+    def fog_servers
+      @fog_servers ||= Ironfan.fog_servers.select{|fs| fs.key_name == cluster_name.to_s && (fs.state != "terminated") }
+    end
+
+    # Walk the list of chef nodes and
+    # * vivify the server,
+    # * associate the chef node
+    # * if the chef node knows about its instance id, memorize that for lookup
+    #   when we discover cloud instances.
+    def discover_chef_nodes!
+      chef_nodes.each do |chef_node|
+        if chef_node["cluster_name"] && chef_node["facet_name"] && chef_node["facet_index"]
+          cluster_name = chef_node["cluster_name"]
+          facet_name   = chef_node["facet_name"]
+          facet_index  = chef_node["facet_index"]
+        elsif chef_node.name
+          ( cluster_name, facet_name, facet_index ) = chef_node.name.split(/-/)
+        else
+          next
+        end
+        svr = Ironfan::Server.get(cluster_name, facet_name, facet_index)
+        svr.chef_node = chef_node
+        @aws_instance_hash[ chef_node.ec2.instance_id ] = svr if chef_node[:ec2] && chef_node.ec2.instance_id
+      end
+    end
+
+    # Walk the list of servers, asking each to discover its chef client.
+    def discover_chef_clients!
+      servers.each(&:chef_client)
+    end
+
+    # calling #servers vivifies each facet's Ironfan::Server instances
+    def discover_ironfan!
+      self.servers
+    end
+
+    def discover_fog_servers!
+      # If the fog server is tagged with cluster/facet/index, then try to
+      # locate the corresponding machine in the cluster def
+      # Otherwise, try to get to it through mapping the aws instance id
+      # to the chef node name found in the chef node
+      fog_servers.each do |fs|
+        if fs.tags["cluster"] && fs.tags["facet"] && fs.tags["index"] && fs.tags["cluster"] == cluster_name.to_s
+          svr = Ironfan::Server.get(fs.tags["cluster"], fs.tags["facet"], fs.tags["index"])
+        elsif @aws_instance_hash[fs.id]
+          svr = @aws_instance_hash[fs.id]
+        else
+          next
+        end
+
+        # If there already is a fog server there, then issue a warning and slap
+        # the just-discovered one onto a server with an arbitrary index, and
+        # mark both bogus
+        if existing_fs = svr.fog_server
+          if existing_fs.id != fs.id
+            ui.warn "Duplicate fog instance found for #{svr.fullname}: #{fs.id} and #{existing_fs.id}!!"
+            old_svr = svr
+            svr     = old_svr.facet.server(1_000 + svr.facet_index.to_i)
+            old_svr.bogosity :duplicate
+            svr.bogosity     :duplicate
+          end
+        end
+        svr.fog_server = fs
+      end
+    end
+
+    def discover_volumes!
+      servers.each(&:discover_volumes!)
+    end
+
+    def discover_addresses!
+      servers.each(&:discover_addresses!)
+    end
+
+  end # Ironfan::Cluster
+end
+
+module Ironfan
+
+  def self.fog_connection
+    @fog_connection ||= Fog::Compute.new({
+        :provider              => 'AWS',
+        :aws_access_key_id     => Chef::Config[:knife][:aws_access_key_id],
+        :aws_secret_access_key => Chef::Config[:knife][:aws_secret_access_key],
+        :region                => Chef::Config[:knife][:region]
+      })
+  end
+
+  def self.fog_servers
+    return @fog_servers if @fog_servers
+    Chef::Log.debug("Using fog to catalog all servers")
+    @fog_servers = Ironfan.fog_connection.servers.all
+  end
+
+  def self.fog_addresses
+    return @fog_addresses if @fog_addresses
+    Chef::Log.debug("Using fog to catalog all addresses")
+    @fog_addresses = {}.tap{|hsh| Ironfan.fog_connection.addresses.each{|fa| hsh[fa.public_ip] = fa } }
+  end
+
+  def self.fog_volumes
+    @fog_volumes || fetch_fog_volumes
+  end
+
+  def self.fetch_fog_volumes
+    Chef::Log.debug("Using fog to catalog all volumes")
+    @fog_volumes = Ironfan.fog_connection.volumes
+  end
+
+  def self.fog_keypairs
+    return @fog_keypairs if @fog_keypairs
+    Chef::Log.debug("Using fog to catalog all keypairs")
+    @fog_keypairs = {}.tap{|hsh| Ironfan.fog_connection.key_pairs.each{|kp| hsh[kp.name] = kp } }
+  end
+
+  def safely *args, &block
+    Ironfan.safely(*args, &block)
+  end
+
+end