cluster_chef 3.0.5

data/lib/cluster_chef/role_implications.rb

@@ -0,0 +1,49 @@
+module ClusterChef
+  ComputeBuilder.class_eval do
+
+    role_implication "nfs_server" do
+      self.cloud.security_group "nfs_server" do
+        authorize_group "nfs_client"
+      end
+    end
+
+    role_implication "nfs_client" do
+      self.cloud.security_group "nfs_client"
+    end
+
+    role_implication "ssh" do
+      self.cloud.security_group 'ssh' do
+        authorize_port_range 22..22
+      end
+    end
+
+    role_implication "chef_server" do
+      self.cloud.security_group "chef_server" do
+        authorize_port_range 4000..4000  # chef-server-api
+        authorize_port_range 4040..4040  # chef-server-webui
+      end
+    end
+
+    # web server? add the group "web_server" to open the web holes
+    role_implication "web_server" do
+      self.cloud.security_group("#{cluster_name}-web_server") do
+        authorize_port_range  80..80
+        authorize_port_range 443..443
+      end
+    end
+
+    # if you're a redis server, open the port and authorize redis clients in your group to talk to you
+    role_implication("redis_server") do
+      cluster_name = self.cluster_name # hack: put cluster_name is in scope
+      self.cloud.security_group("#{cluster_name}-redis_server") do
+        authorize_group("#{cluster_name}-redis_client")
+      end
+    end
+
+    # redis_clients gain rights to the redis_server
+    role_implication("redis_client") do
+      self.cloud.security_group("#{cluster_name}-redis_client")
+    end
+
+  end
+end

data/lib/cluster_chef/security_group.rb

@@ -0,0 +1,103 @@
+module ClusterChef
+  module Cloud
+
+    class SecurityGroup < DslObject
+      has_keys :name, :description, :owner_id
+      attr_reader :group_authorizations
+      attr_reader :range_authorizations
+
+      def initialize cloud, group_name, group_description=nil, group_owner_id=nil
+        super()
+        set :name, group_name.to_s
+        description group_description || "cluster_chef generated group #{group_name}"
+        @cloud         = cloud
+        @group_authorizations = []
+        @range_authorizations = []
+        owner_id group_owner_id || Chef::Config[:knife][:aws_account_id]
+      end
+
+      @@all = nil
+      def all
+        self.class.all
+      end
+      def self.all
+        return @@all if @@all
+        get_all
+      end
+      def self.get_all
+        groups_list = ClusterChef.fog_connection.security_groups.all
+        @@all = groups_list.inject(Mash.new) do |hsh, group|
+          hsh[group.name] = group ; hsh
+        end
+      end
+
+      def get
+        all[name] || ClusterChef.fog_connection.security_groups.get(name)
+      end
+
+      def self.get_or_create group_name, description
+        group = all[group_name] || ClusterChef.fog_connection.security_groups.get(group_name)
+        if ! group
+          self.step(group_name, "creating (#{description})", :blue)
+          group = all[group_name] = ClusterChef.fog_connection.security_groups.new(:name => group_name, :description => description, :connection => ClusterChef.fog_connection)
+          group.save
+        end
+        group
+      end
+
+      def authorize_group_and_owner group, owner_id=nil
+        @group_authorizations << [group.to_s, owner_id]
+      end
+
+      # Alias for authorize_group_and_owner
+      def authorize_group *args
+        authorize_group_and_owner *args
+      end
+
+      def authorize_port_range range, cidr_ip = '0.0.0.0/0', ip_protocol = 'tcp'
+        range = (range .. range) if range.is_a?(Integer)
+        @range_authorizations << [range, cidr_ip, ip_protocol]
+      end
+
+      def group_permission_already_set? group, authed_group, authed_owner
+        return false if group.ip_permissions.nil?
+        group.ip_permissions.any? do |existing_permission|
+          existing_permission["groups"].include?({"userId"=>authed_owner, "groupName"=>authed_group}) &&
+            existing_permission["fromPort"] == 1 &&
+            existing_permission["toPort"] == 65535
+        end
+      end
+
+      def range_permission_already_set? group, range, cidr_ip, ip_protocol
+        return false if group.ip_permissions.nil?
+        group.ip_permissions.include?({"groups"=>[], "ipRanges"=>[{"cidrIp"=>cidr_ip}], "ipProtocol"=>ip_protocol, "fromPort"=>range.first, "toPort"=>range.last})
+      end
+
+      def run
+        group = self.class.get_or_create name, description
+        @group_authorizations.uniq.each do |authed_group, authed_owner|
+          authed_owner ||= self.owner_id
+          next if group_permission_already_set?(group, authed_group, authed_owner)
+          step("authorizing access from all machines in #{authed_group}", :blue)
+          self.class.get_or_create(authed_group, "Authorized to access nfs server")
+          begin  group.authorize_group_and_owner(authed_group, authed_owner)
+          rescue StandardError => e ; ui.warn e ; end
+        end
+        @range_authorizations.uniq.each do |range, cidr_ip, ip_protocol|
+          next if range_permission_already_set?(group, range, cidr_ip, ip_protocol)
+          step("opening #{ip_protocol} ports #{range} to #{cidr_ip}", :blue)
+          begin  group.authorize_port_range(range, { :cidr_ip => cidr_ip, :ip_protocol => ip_protocol })
+          rescue StandardError => e ; ui.warn e ; end
+        end
+      end
+
+      def self.step(group_name, desc, *style)
+        ui.info("  group #{"%-15s" % (group_name+":")}\t#{ui.color(desc.to_s, *style)}")
+      end
+      def step(desc, *style)
+        self.class.step(self.name, desc, *style)
+      end
+
+    end
+  end
+end

data/lib/cluster_chef/server.rb

@@ -0,0 +1,265 @@
+module ClusterChef
+
+  #
+  # A server is a specific (logical) member of a facet within a cluster.
+  #
+  # It may have extra attributes if it also exists in the Chef server,
+  # or if it exists in the real world (as revealed by Fog)
+  #
+  class Server < ClusterChef::ComputeBuilder
+    attr_reader   :cluster, :facet, :facet_index, :tags
+    attr_accessor :chef_node, :fog_server
+
+    @@all ||= Mash.new
+
+    def initialize facet, idx
+      @cluster     = facet.cluster
+      @facet       = facet
+      @facet_index = idx
+      @fullname    = [cluster_name, facet_name, facet_index].join('-')
+      super(@fullname)
+      @tags = { "name" => name, "cluster" => cluster_name, "facet"   => facet_name, "index" => facet_index, }
+      ui.warn("Duplicate server #{[self, facet.name, idx]} vs #{@@all[fullname]}") if @@all[fullname]
+      @@all[fullname] = self
+    end
+
+    def fullname fn=nil
+      @fullname = fn if fn
+      @fullname
+    end
+
+    def cluster_name
+      cluster.name
+    end
+
+    def facet_name
+      facet.name
+    end
+
+    def servers
+      ClusterChef::ServerGroup.new(cluster, [self])
+    end
+
+    def bogosity val=nil
+      @settings[:bogosity] = val  if not val.nil?
+      return @settings[:bogosity] if not @settings[:bogosity].nil?
+      return :bogus_facet         if facet.bogus?
+      # return :out_of_range      if (self.facet_index.to_i >= facet.instances)
+      false
+    end
+
+    def in_cloud?
+      !! fog_server
+    end
+
+    def in_chef?
+      chef_node || chef_client
+    end
+
+    def has_cloud_state?(*states)
+      in_cloud? && states.flatten.include?(fog_server.state)
+    end
+
+    def exists?
+      created? || in_chef?
+    end
+
+    def created?
+      in_cloud? && (not ['terminated', 'shutting-down'].include?(fog_server.state))
+    end
+
+    def running?
+      has_cloud_state?('running')
+    end
+
+    def startable?
+      has_cloud_state?('stopped')
+    end
+
+    def launchable?
+      not created?
+    end
+
+    def sshable?
+      in_chef?
+    end
+
+    def killable?
+      in_chef? || created?
+    end
+
+    def to_s
+      super[0..-3] + " chef: #{in_chef? && chef_node.name} fog: #{in_cloud? && fog_server.id}}>"
+    end
+
+    #
+    # Attributes
+    #
+
+    def tag key, value=nil
+      if value then @tags[key] = value ; end
+      @tags[key]
+    end
+
+    #
+    # Resolve:
+    #
+    def resolve!
+      reverse_merge!(facet)
+      reverse_merge!(cluster)
+      @settings[:run_list] = combined_run_list
+      #
+      cloud.reverse_merge!(facet.cloud)
+      cloud.reverse_merge!(cluster.cloud)
+      #
+      cloud.user_data({
+          :chef_server            => Chef::Config.chef_server_url,
+          :validation_client_name => Chef::Config.validation_client_name,
+          #
+          :node_name              => fullname,
+          :cluster_name           => cluster_name,
+          :facet_name             => facet_name,
+          :facet_index            => facet_index,
+          #
+          :run_list               => run_list,
+        })
+      #
+      if client_key.body then cloud.user_data({ :client_key     => client_key.body, })
+      else                    cloud.user_data({ :validation_key => cloud.validation_key }) ; end
+      cloud.keypair(cluster_name) if cloud.keypair.nil?
+      #
+      self
+    end
+
+    #
+    # Assembles the combined runlist.
+    #
+    # * run_list :first  items -- cluster then facet then server
+    # * run_list :normal items -- cluster then facet then server
+    # * run_list :last   items -- cluster then facet then server
+    #
+    #    ClusterChef.cluster(:my_cluster) do
+    #      role('f',  :last)
+    #      role('c')
+    #      facet(:my_facet) do
+    #        role('d')
+    #        role('e')
+    #        role('b', :first)
+    #        role('h',  :last)
+    #      end
+    #      role('a', :first)
+    #      role('g', :last)
+    #    end
+    #
+    # produces
+    #    cluster list  [a] [c] [fg]
+    #    facet list    [b] [de] [h]
+    #
+    # yielding run_list
+    #     ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']
+    #
+    # Avoid duplicate conflicting declarations. If you say define things more
+    # than once, the *earliest encountered* one wins, even if it is elsewhere
+    # marked :last.
+    #
+    def combined_run_list
+      cg = @cluster.run_list_groups
+      fg = @facet.run_list_groups
+      sg = self.run_list_groups
+      [ cg[:first],  fg[:first],  sg[:first],
+        cg[:normal], fg[:normal], sg[:normal],
+        cg[:last],   fg[:last],   sg[:last], ].flatten.compact.uniq
+    end
+
+    #
+    # This prepares a composited view of the volumes -- it shows the cluster
+    # definition overlaid by the facet definition overlaid by the server
+    # definition.
+    #
+    # This method *does* auto-vivify an empty volume declaration on the server,
+    # but doesn't modify it.
+    #
+    # This code is pretty smelly, but so is the resolve! behavior. advice welcome.
+    #
+    def composite_volumes
+      vols = {}
+      facet.volumes.each do |vol_name, vol|
+        self.volumes[vol_name] ||= ClusterChef::Volume.new(:parent => self, :name => vol_name)
+        vols[vol_name]         ||= self.volumes[vol_name].dup
+        vols[vol_name].reverse_merge!(vol)
+      end
+      cluster.volumes.each do |vol_name, vol|
+        self.volumes[vol_name] ||= ClusterChef::Volume.new(:parent => self, :name => vol_name)
+        vols[vol_name]         ||= self.volumes[vol_name].dup
+        vols[vol_name].reverse_merge!(vol)
+      end
+      vols.each{|vol_name, vol| vol.availability_zone self.default_availability_zone }
+      vols
+    end
+
+    # FIXME -- this will break on some edge case wehre a bogus node is
+    # discovered after everything is resolve!d
+    def default_availability_zone
+      cloud.default_availability_zone
+    end
+
+    #
+    # retrieval
+    #
+    def self.get(cluster_name, facet_name, facet_index)
+      cluster = ClusterChef.cluster(cluster_name)
+      had_facet = cluster.has_facet?(facet_name)
+      facet = cluster.facet(facet_name)
+      facet.bogosity true unless had_facet
+      had_server = facet.has_server?( facet_index )
+      server = facet.server(facet_index)
+      server.bogosity :not_defined_in_facet unless had_server
+      return server
+    end
+
+    def self.all
+      @@all
+    end
+
+    #
+    # Actions!
+    #
+
+    def sync_to_cloud
+      step "Syncing to cloud"
+      attach_volumes
+      create_tags
+      associate_public_ip
+    end
+
+    def sync_to_chef
+      step "Syncing to chef server"
+      sync_chef_node
+      true
+    end
+
+    # FIXME: a lot of AWS logic in here. This probably lives in the facet.cloud
+    # but for the one or two things that come from the facet
+    def create_server
+      return nil if created? # only create a server if it does not already exist
+      fog_create_server
+    end
+
+    def create_tags
+      return unless created?
+      step("  labeling servers and volumes")
+      fog_create_tags(fog_server, self.fullname, tags)
+      composite_volumes.each do |vol_name, vol|
+        if vol.fog_volume
+          fog_create_tags(vol.fog_volume, vol.desc,
+            { "server" => self.fullname, "name" => "#{name}-#{vol.name}", "device" => vol.device, "mount_point" => vol.mount_point, "cluster" => cluster_name, "facet"   => facet_name, "index"   => facet_index, })
+        end
+      end
+    end
+
+    def block_device_mapping
+      composite_volumes.values.map(&:block_device_mapping).compact
+    end
+
+  end
+end

data/lib/cluster_chef/server_slice.rb

@@ -0,0 +1,259 @@
+module ClusterChef
+  #
+  # A server group is a set of actual or implied servers.
+  #
+  # The idea is we want to be able to smoothly roll up settings
+  #
+  #
+  class ServerSlice < ClusterChef::DslObject
+    attr_reader :name, :servers, :cluster
+
+    def initialize cluster, servers
+      super()
+      @name    = "#{cluster.name} slice"
+      @cluster = cluster
+      @servers = servers
+    end
+
+    #
+    # Enumerable
+    #
+    include Enumerable
+    def each(&block)
+      @servers.each(&block)
+    end
+    def length
+      @servers.length
+    end
+    def empty?
+      length == 0
+    end
+    [:select, :find_all, :reject, :detect, :find, :drop_while].each do |method|
+      define_method(method) do |*args, &block|
+        ServerSlice.new cluster, @servers.send(method, *args, &block)
+      end
+    end
+
+    # Return the collection of servers that are not yet 'created'
+    def uncreated_servers
+      select{|svr| not svr.created? }
+    end
+
+    def bogus_servers
+      select(&:bogus?)
+    end
+
+    #
+    # Info!
+    #
+
+    def chef_nodes
+      servers.map(&:chef_node).compact
+    end
+
+    def fog_servers
+      servers.map(&:fog_server).compact
+    end
+
+    def security_groups
+      sg = {}
+      servers.each{|svr| sg.merge!(svr.cloud.security_groups) }
+      sg
+    end
+
+    def facets
+      servers.map(&:facet)
+    end
+
+    def chef_roles
+      [ cluster.chef_roles, facets.map(&:chef_roles) ].flatten.compact.uniq
+    end
+
+    # hack -- take the ssh_identity_file from the first server.
+    def ssh_identity_file
+      return if servers.empty?
+      servers.first.cloud.ssh_identity_file
+    end
+
+    #
+    # Actions!
+    #
+
+    def start
+      delegate_to_fog_servers( :start  )
+      delegate_to_fog_servers( :reload  )
+    end
+
+    def stop
+      delegate_to_fog_servers( :stop )
+      delegate_to_fog_servers( :reload  )
+    end
+
+    def destroy
+      delegate_to_fog_servers( :destroy )
+      delegate_to_fog_servers( :reload  )
+    end
+
+    def reload
+      delegate_to_fog_servers( :reload  )
+    end
+
+    def create_servers
+      delegate_to_servers( :create_server )
+    end
+
+    def delete_chef
+      delegate_to_servers( :delete_chef, true )
+    end
+
+    def sync_to_cloud
+      sync_keypairs
+      delegate_to_servers( :sync_to_cloud )
+    end
+
+    def sync_to_chef
+      sync_roles
+      delegate_to_servers( :sync_to_chef )
+    end
+
+    #
+    # Display!
+    #
+
+    # FIXME: this is a jumble. we need to pass it in some other way.
+
+    MINIMAL_HEADINGS  = ["Name", "Chef?", "State", "InstanceID", "Public IP", "Private IP", "Created At"].to_set.freeze
+    DEFAULT_HEADINGS  = (MINIMAL_HEADINGS + ['Flavor', 'AZ', 'Env']).freeze
+    EXPANDED_HEADINGS = DEFAULT_HEADINGS + ['Image', 'Volumes', 'Elastic IP', 'SSH Key']
+
+    MACHINE_STATE_COLORS  = {
+      'running'       => :green,
+      'pending'       => :yellow,
+      'stopping'      => :magenta,
+      'shutting-down' => :magenta,
+      'stopped'       => :cyan,
+      'terminated'    => :blue,
+      'not running'   => :blue,
+    }
+
+    #
+    # This is a generic display routine for cluster-like sets of nodes. If you
+    # call it with no args, you get the basic table that knife cluster show
+    # draws.  If you give it an array of strings, you can override the order and
+    # headings displayed. If you also give it a block you can add your own logic
+    # for generating content. The block is given a ClusterChef::Server instance
+    # for each item in the collection and should return a hash of Name,Value
+    # pairs to merge into the minimal fields.
+    #
+    def display hh = :default
+      headings =
+        case hh
+        when :minimal  then MINIMAL_HEADINGS
+        when :default  then DEFAULT_HEADINGS
+        when :expanded then EXPANDED_HEADINGS
+        else hh.to_set end
+      headings += ["Bogus"] if servers.any?(&:bogus?)
+      # probably not necessary any more
+      # servers = servers.sort{ |a,b| (a.facet_name <=> b.facet_name) *9 + (a.facet_index.to_i <=> b.facet_index.to_i)*3 + (a.facet_index <=> b.facet_index) }
+      defined_data = servers.map do |svr|
+        hsh = {
+          "Name"   => svr.fullname,
+          "Facet"  => svr.facet_name,
+          "Index"  => svr.facet_index,
+          "Chef?"  => (svr.chef_node? ? "yes" : "[red]no[reset]"),
+          "Bogus"  => (svr.bogus? ? "[red]#{svr.bogosity}[reset]" : ''),
+          "Env"    => svr.environment,
+        }
+        # if (cs = svr.chef_server)
+        #   hsh.merge!(
+        #     "Env"    => cs.environment,
+        #     )
+        # end
+        if (fs = svr.fog_server)
+          hsh.merge!(
+              "InstanceID" => (fs.id && fs.id.length > 0) ? fs.id : "???",
+              "Flavor"     => fs.flavor_id,
+              "Image"      => fs.image_id,
+              "AZ"         => fs.availability_zone,
+              "SSH Key"    => fs.key_name,
+              "State"      => "[#{MACHINE_STATE_COLORS[fs.state] || 'white'}]#{fs.state}[reset]",
+              "Public IP"  => fs.public_ip_address,
+              "Private IP" => fs.private_ip_address,
+              "Created At" => fs.created_at.strftime("%Y%m%d-%H%M%S")
+            )
+        else
+          hsh["State"] = "not running"
+        end
+        hsh['Volumes'] = []
+        svr.composite_volumes.each do |name, vol|
+          if    vol.ephemeral_device? then next
+          elsif vol.volume_id         then hsh['Volumes'] << vol.volume_id
+          elsif vol.create_at_launch? then hsh['Volumes'] << vol.snapshot_id
+          end
+        end
+        hsh['Volumes']    = hsh['Volumes'].join(',')
+        hsh['Elastic IP'] = svr.cloud.public_ip if svr.cloud.public_ip
+        if block_given?
+          extra_info = yield(svr)
+          hsh.merge!(extra_info)
+          headings += extra_info.keys
+        end
+        hsh
+      end
+      if defined_data.empty?
+        ui.info "Nothing to report"
+      else
+        Formatador.display_compact_table(defined_data, headings.to_a)
+      end
+    end
+
+    def to_s
+      str = super
+      str[0..-2] + " #{@servers.map(&:fullname)}>"
+    end
+
+    def joined_names
+      map(&:name).join(", ").gsub(/, ([^,]*)$/, ' and \1')
+    end
+
+    # Calls block on each server in parallel, each in its own thread
+    #
+    # @example
+    #   target = ClusterChef::Cluster.slice('web_server')
+    #   target.parallelize{|svr| svr.launch }
+    #
+    # @yield each server, in turn
+    #
+    # @return array (in same order as servers) of each block's result
+    def parallelize
+      servers.map do |svr|
+        sleep(0.1) # avoid hammering with simultaneous requests
+        Thread.new(svr){|svr| yield(svr) }
+      end
+    end
+
+  protected
+
+    # Helper methods for iterating through the servers to do things
+    #
+    # @param [Symbol]  method   -- method to call on each server
+    # @param [Boolean] threaded -- execute each call in own thread
+    #
+    # @return array (in same order as servers) of results for that method
+    def delegate_to_servers method, threaded = true
+      if threaded  # Call in threads
+        threads = parallelize{|svr| svr.send(method) }
+        threads.map{|t| t.join.value } # Wait, returning array of results
+      else         # Call the method for each server sequentially
+        servers.map{|svr| svr.send(method) }
+      end
+    end
+
+    def delegate_to_fog_servers method
+      fog_servers.compact.map do |fs|
+        fs.send(method)
+      end
+    end
+
+  end
+end