cluster_chef 3.0.10 → 3.0.11

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## v3.0.11: We Raid at Dawn
+
+* You can now assemble raid groups in the cluster definition:
+  - node metadata instructing the volumes recipe to build the raid volume
+  - marks the component volumes as non-mountable, in the appropriate raid group, etc
+* Changed the order of `cluster_role` and `facet_role` in the run list. It now goes:
+  - `:first`  roles (cluster then facet)
+  - `:normal` roles (cluster then facet)
+  - special roles: `cluster_role` then `facet_role`
+  - `:last` roles (cluster then facet)
+* knife cluster launch uses ClusterBootstrap, not knife's vanilla bootstrap.
+* can now do group('group_that_wants').authorized_by_group('group_that_grants') so that in cluster A I can request access to cluster B without gaining its group myself.
+* push the organization (if set) into the node metadata
+
 ## v3.0.10: Cloud fixes
 
 * security groups are now created/updated in knife cluster sync. This can't help you apply then to a node afer launch though -- nothing can, the API doesn't allow it.

data/TODO.md

@@ -6,3 +6,11 @@
 * knife cluster kick fails if service isn't running
 * make clear directions for installing `cluster_chef` and its initial use.
 * knife cluster launch should fail differently if you give it a facet that doesn't exist
+
+
+
+### ssh_user, ssh_identity_file, keypair, template should be set by cluster except when they shouldn't
+
+### Organization-specific homebase files
+
+The current organization of the homebase needs to better scope organization-specific customizations

data/VERSION

@@ -1 +1 @@
-3.0.10
+3.0.11

data/cluster_chef.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "cluster_chef"
-  s.version = "3.0.10"
+  s.version = "3.0.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-01-17"
+  s.date = "2012-01-24"
   s.description = "cluster_chef allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [
@@ -84,7 +84,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<rspec>, ["~> 2.5"])
       s.add_development_dependency(%q<yard>, ["~> 0.6"])
       s.add_development_dependency(%q<configliere>, ["~> 0.4.8"])
-      s.add_runtime_dependency(%q<cluster_chef-knife>, ["= 3.0.10"])
+      s.add_runtime_dependency(%q<cluster_chef-knife>, ["= 3.0.11"])
     else
       s.add_dependency(%q<chef>, ["~> 0.10.4"])
       s.add_dependency(%q<fog>, ["~> 1.1.1"])
@@ -95,7 +95,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<rspec>, ["~> 2.5"])
       s.add_dependency(%q<yard>, ["~> 0.6"])
       s.add_dependency(%q<configliere>, ["~> 0.4.8"])
-      s.add_dependency(%q<cluster_chef-knife>, ["= 3.0.10"])
+      s.add_dependency(%q<cluster_chef-knife>, ["= 3.0.11"])
     end
   else
     s.add_dependency(%q<chef>, ["~> 0.10.4"])
@@ -107,7 +107,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<rspec>, ["~> 2.5"])
     s.add_dependency(%q<yard>, ["~> 0.6"])
     s.add_dependency(%q<configliere>, ["~> 0.4.8"])
-    s.add_dependency(%q<cluster_chef-knife>, ["= 3.0.10"])
+    s.add_dependency(%q<cluster_chef-knife>, ["= 3.0.11"])
   end
 end
 

data/lib/cluster_chef/chef_layer.rb

@@ -198,13 +198,14 @@ module ClusterChef
     def set_chef_node_attributes
       step("  setting node runlist and essential attributes")
       @chef_node.run_list = Chef::RunList.new(*@settings[:run_list])
+      @chef_node.normal[  :organization] = Chef::Config[:organization] if Chef::Config[:organization]
       @chef_node.override[:cluster_name] = cluster_name
       @chef_node.override[:facet_name]   = facet_name
       @chef_node.override[:facet_index]  = facet_index
     end
 
     def set_chef_node_environment
-      @chef_node.chef_environment(environment.to_s)
+      @chef_node.chef_environment(environment.to_s) if environment.present?
     end
 
     #

data/lib/cluster_chef/cluster.rb

@@ -111,7 +111,7 @@ module ClusterChef
     def create_cluster_role
       @cluster_role_name = "#{name}_cluster"
       @cluster_role      = new_chef_role(@cluster_role_name, cluster)
-      role(@cluster_role_name, :last)
+      role(@cluster_role_name, :own)
     end
 
   end

data/lib/cluster_chef/compute.rb

@@ -70,6 +70,16 @@ module ClusterChef
       volumes[volume_name]
     end
 
+    def raid_group(rg_name, attrs={}, &block)
+      volumes[rg_name] ||= ClusterChef::RaidGroup.new(:parent => self, :name => rg_name)
+      volumes[rg_name].configure(attrs, &block)
+      p volumes
+      volumes[rg_name].sub_volumes.each do |sv_name|
+        volume(sv_name){ in_raid(rg_name) ; mountable(false) ; tags({}) }
+      end
+      volumes[rg_name]
+    end
+
     def root_volume(attrs={}, &block)
       volume(:root, attrs, &block)
     end
@@ -134,7 +144,7 @@ module ClusterChef
   protected
 
     def add_to_run_list(item, placement)
-      raise "run_list placement must be one of :first, :normal, :last or nil (also means :normal)" unless [:first, :last, nil].include?(placement)
+      raise "run_list placement must be one of :first, :normal, :last or nil (also means :normal)" unless [:first, :last, :own, nil].include?(placement)
       @@run_list_rank += 1
       placement ||= :normal
       @run_list_info[item] ||= { :rank => @@run_list_rank, :placement => placement }

data/lib/cluster_chef/facet.rb

@@ -119,7 +119,7 @@ module ClusterChef
     def create_facet_role
       @facet_role_name = "#{cluster_name}_#{facet_name}"
       @facet_role      = new_chef_role(@facet_role_name, cluster, self)
-      role(@facet_role_name, :last)
+      role(@facet_role_name, :own)
     end
 
     #

data/lib/cluster_chef/private_key.rb

@@ -74,6 +74,27 @@ module ClusterChef
     end
   end
 
+  class DataBagKey < PrivateKey
+    def body
+      return @body if @body
+      @body
+    end
+
+    def random_token
+      require "digest/sha2"
+      digest = Digest::SHA512.hexdigest(  Time.now.to_s + (1..10).collect{ rand.to_s }.join  )
+      5.times{ digest = Digest::SHA512.hexdigest(digest) }
+      digest
+    end
+
+    def key_dir
+      return Chef::Config.data_bag_key_dir if Chef::Config.data_bag_key_dir
+      dir = "#{ENV['HOME']}/.chef/data_bag_keys"
+      warn "Please set 'data_bag_key_dir' in your knife.rb. Will use #{dir} as a default"
+      dir
+    end
+  end
+
   class Ec2Keypair < PrivateKey
     def body
       return @body if @body
@@ -100,7 +121,7 @@ module ClusterChef
         return Chef::Config.ec2_key_dir
       else
         dir = "#{ENV['HOME']}/.chef/ec2_keys"
-        warn "Please set 'ec2_key_dir' in your knife.rb -- using #{dir} as a default"
+        warn "Please set 'ec2_key_dir' in your knife.rb. Will use #{dir} as a default"
         dir
       end
     end

data/lib/cluster_chef/security_group.rb

@@ -12,8 +12,9 @@ module ClusterChef
         description group_description || "cluster_chef generated group #{group_name}"
         @cloud         = cloud
         @group_authorizations = []
+        @group_authorized_by  = []
         @range_authorizations = []
-        owner_id group_owner_id || Chef::Config[:knife][:aws_account_id]
+        owner_id(group_owner_id || Chef::Config[:knife][:aws_account_id])
       end
 
       @@all = nil
@@ -26,8 +27,8 @@ module ClusterChef
       end
       def self.get_all
         groups_list = ClusterChef.fog_connection.security_groups.all
-        @@all = groups_list.inject(Mash.new) do |hsh, group|
-          hsh[group.name] = group ; hsh
+        @@all = groups_list.inject(Mash.new) do |hsh, fog_group|
+          hsh[fog_group.name] = fog_group ; hsh
         end
       end
 
@@ -35,58 +36,73 @@ module ClusterChef
         all[name] || ClusterChef.fog_connection.security_groups.get(name)
       end
 
-      def self.get_or_create group_name, description
-        group = all[group_name] || ClusterChef.fog_connection.security_groups.get(group_name)
-        if ! group
-          self.step(group_name, "creating (#{description})", :blue)
-          group = all[group_name] = ClusterChef.fog_connection.security_groups.new(:name => group_name, :description => description, :connection => ClusterChef.fog_connection)
-          group.save
+      def self.get_or_create(group_name, description)
+        # FIXME: the '|| ClusterChef.fog' part is probably unnecessary
+        fog_group = all[group_name] || ClusterChef.fog_connection.security_groups.get(group_name)
+        unless fog_group
+          self.step(group_name, "creating (#{description})", :green)
+          fog_group = all[group_name] = ClusterChef.fog_connection.security_groups.new(:name => group_name, :description => description, :connection => ClusterChef.fog_connection)
+          fog_group.save
         end
-        group
+        fog_group
       end
 
-      def authorize_group_and_owner group, owner_id=nil
-        @group_authorizations << [group.to_s, owner_id]
+      def authorize_group(group_name, owner_id=nil)
+        @group_authorizations << [group_name.to_s, owner_id]
       end
 
-      # Alias for authorize_group_and_owner
-      def authorize_group *args
-        authorize_group_and_owner *args
+      def authorized_by_group(other_name)
+        @group_authorized_by << [other_name.to_s, nil]
       end
 
-      def authorize_port_range range, cidr_ip = '0.0.0.0/0', ip_protocol = 'tcp'
+      def authorize_port_range(range, cidr_ip = '0.0.0.0/0', ip_protocol = 'tcp')
         range = (range .. range) if range.is_a?(Integer)
         @range_authorizations << [range, cidr_ip, ip_protocol]
       end
 
-      def group_permission_already_set? group, authed_group, authed_owner
-        return false if group.ip_permissions.nil?
-        group.ip_permissions.any? do |existing_permission|
-          existing_permission["groups"].include?({"userId"=>authed_owner, "groupName"=>authed_group}) &&
+      def group_permission_already_set?(fog_group, other_name, authed_owner)
+        return false if fog_group.ip_permissions.nil?
+        fog_group.ip_permissions.any? do |existing_permission|
+          existing_permission["groups"].include?({"userId" => authed_owner, "groupName" => other_name}) &&
             existing_permission["fromPort"] == 1 &&
-            existing_permission["toPort"] == 65535
+            existing_permission["toPort"]   == 65535
         end
       end
 
-      def range_permission_already_set? group, range, cidr_ip, ip_protocol
-        return false if group.ip_permissions.nil?
-        group.ip_permissions.include?({"groups"=>[], "ipRanges"=>[{"cidrIp"=>cidr_ip}], "ipProtocol"=>ip_protocol, "fromPort"=>range.first, "toPort"=>range.last})
+      def range_permission_already_set?(fog_group, range, cidr_ip, ip_protocol)
+        return false if fog_group.ip_permissions.nil?
+        fog_group.ip_permissions.include?(
+          { "groups"=>[], "ipRanges"=>[{"cidrIp"=>cidr_ip}],
+            "ipProtocol"=>ip_protocol, "fromPort"=>range.first, "toPort"=>range.last})
       end
 
+      # FIXME: so if you're saying to yourself, "self, this is some soupy gooey
+      # code right here" then you and your self are correct. Much of this is to
+      # work around old limitations in the EC2 api. You can now treat range and
+      # group permissions the same, and we should.
+
       def run
-        group = self.class.get_or_create name, description
-        @group_authorizations.uniq.each do |authed_group, authed_owner|
+        fog_group = self.class.get_or_create(name, description)
+        @group_authorizations.uniq.each do |other_name, authed_owner|
           authed_owner ||= self.owner_id
-          next if group_permission_already_set?(group, authed_group, authed_owner)
-          step("authorizing access from all machines in #{authed_group}", :blue)
-          self.class.get_or_create(authed_group, "Authorized to access nfs server")
-          begin  group.authorize_group_and_owner(authed_group, authed_owner)
+          next if group_permission_already_set?(fog_group, other_name, authed_owner)
+          step("authorizing access from all machines in #{other_name} to #{name}", :blue)
+          self.class.get_or_create(other_name, "Authorized to access #{name}")
+          begin  fog_group.authorize_group_and_owner(other_name, authed_owner)
+          rescue StandardError => e ; ui.warn e ; end
+        end
+        @group_authorized_by.uniq.each do |other_name|
+          authed_owner = self.owner_id
+          other_group = self.class.get_or_create(other_name, "Authorized for access by #{self.name}")
+          next if group_permission_already_set?(other_group, self.name, authed_owner)
+          step("authorizing access to all machines in #{other_name} from #{name}", :blue)
+          begin  other_group.authorize_group_and_owner(self.name, authed_owner)
           rescue StandardError => e ; ui.warn e ; end
         end
         @range_authorizations.uniq.each do |range, cidr_ip, ip_protocol|
-          next if range_permission_already_set?(group, range, cidr_ip, ip_protocol)
+          next if range_permission_already_set?(fog_group, range, cidr_ip, ip_protocol)
           step("opening #{ip_protocol} ports #{range} to #{cidr_ip}", :blue)
-          begin  group.authorize_port_range(range, { :cidr_ip => cidr_ip, :ip_protocol => ip_protocol })
+          begin  fog_group.authorize_port_range(range, { :cidr_ip => cidr_ip, :ip_protocol => ip_protocol })
           rescue StandardError => e ; ui.warn e ; end
         end
       end

data/lib/cluster_chef/server.rb

@@ -136,6 +136,7 @@ module ClusterChef
     #
     # * run_list :first  items -- cluster then facet then server
     # * run_list :normal items -- cluster then facet then server
+    # * own roles: cluster_role then facet_role
     # * run_list :last   items -- cluster then facet then server
     #
     #    ClusterChef.cluster(:my_cluster) do
@@ -152,11 +153,11 @@ module ClusterChef
     #    end
     #
     # produces
-    #    cluster list  [a] [c] [fg]
-    #    facet list    [b] [de] [h]
+    #    cluster list  [a] [c]  [cluster_role] [fg]
+    #    facet list    [b] [de] [facet_role]   [h]
     #
     # yielding run_list
-    #     ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']
+    #     ['a', 'b', 'c', 'd', 'e', 'cr', 'fr', 'f', 'g', 'h']
     #
     # Avoid duplicate conflicting declarations. If you say define things more
     # than once, the *earliest encountered* one wins, even if it is elsewhere
@@ -168,6 +169,7 @@ module ClusterChef
       sg = self.run_list_groups
       [ cg[:first],  fg[:first],  sg[:first],
         cg[:normal], fg[:normal], sg[:normal],
+        cg[:own],    fg[:own],
         cg[:last],   fg[:last],   sg[:last], ].flatten.compact.uniq
     end
 

data/lib/cluster_chef/volume.rb

@@ -9,6 +9,7 @@ module ClusterChef
       :name,
       # mountable volume attributes
       :device, :mount_point, :mount_options, :fstype, :mount_dump, :mount_pass,
+      :mountable, :formattable, :resizable, :in_raid,
       # cloud volume attributes
       :attachable, :create_at_launch, :volume_id, :snapshot_id, :size, :keep, :availability_zone,
       # arbitrary tags
@@ -18,11 +19,29 @@ module ClusterChef
     VOLUME_DEFAULTS = {
       :fstype          => 'xfs',
       :mount_options    => 'defaults,nouuid,noatime',
+      :keep             => true,
       :attachable       => :ebs,
       :create_at_launch => false,
-      :keep             => true,
+      #
+      :mountable        => true,
+      :resizable        => false,
+      :formattable      => false,
+      :in_raid          => false,
     }
 
+    # Snapshot for snapshot_name method.
+    # Set your own by adding
+    #
+    #     VOLUME_IDS = Mash.new unless defined?(VOLUME_IDS)
+    #     VOLUME_IDS.merge!({ :your_id => 'snap-whatever' })
+    #
+    # to your organization's knife.rb
+    #
+    VOLUME_IDS = Mash.new unless defined?(VOLUME_IDS)
+    VOLUME_IDS.merge!({
+      :blank_xfs       => 'snap-d9c1edb1',
+    })
+
     # Describes a volume
     #
     # @example
@@ -50,6 +69,13 @@ module ClusterChef
       volume_id =~ /^ephemeral/
     end
 
+    # Named snapshots, as defined in ClusterChef::Volume::VOLUME_IDS
+    def snapshot_name(name)
+      snap_id = VOLUME_IDS[name.to_sym]
+      raise "Unknown snapshot name #{name} - is it defined in ClusterChef::Volume::VOLUME_IDS?" unless snap_id
+      self.snapshot_id(snap_id)
+    end
+
     # With snapshot specified but volume missing, have it auto-created at launch
     #
     # Be careful with this -- you can end up with multiple volumes claiming to
@@ -79,10 +105,10 @@ module ClusterChef
       if ephemeral_device?
         hsh['VirtualName'] = volume_id
       elsif create_at_launch?
-        hsh.merge!({
-            'Ebs.SnapshotId' => snapshot_id,
-            'Ebs.VolumeSize' => size,
-            'Ebs.DeleteOnTermination' => (! keep).to_s })
+        raise "Must specify a size or a snapshot ID for #{self}" if snapshot_id.blank? && size.blank?
+        hsh['Ebs.SnapshotId'] = snapshot_id if snapshot_id.present?
+        hsh['Ebs.VolumeSize'] = size.to_s   if size.present?
+        hsh['Ebs.DeleteOnTermination'] = (! keep).to_s
       else
         return
       end
@@ -90,4 +116,42 @@ module ClusterChef
     end
 
   end
+
+
+  #
+  # Consider raising the chunk size to 256 and setting read_ahead 65536 if you are raid'ing EBS volumes
+  #
+  # * http://victortrac.com/EC2_Ephemeral_Disks_vs_EBS_Volumes
+  # * http://orion.heroku.com/past/2009/7/29/io_performance_on_ebs/
+  # * http://tech.blog.greplin.com/aws-best-practices-and-benchmarks
+  # * http://stu.mp/2009/12/disk-io-and-throughput-benchmarks-on-amazons-ec2.html
+  #
+  class RaidGroup < Volume
+    has_keys(
+      :sub_volumes,     # volumes that comprise this raid group
+      :level,           # RAID level (http://en.wikipedia.org/wiki/RAID#Standard_levels)
+      :chunk,           # Raid chunk size (https://raid.wiki.kernel.org/articles/r/a/i/RAID_setup_cbb2.html)
+      :read_ahead,      # read-ahead buffer
+      )
+
+    def desc
+      "#{name} on #{parent.fullname} (#{volume_id} @ #{device} from #{sub_volumes.join(',')})"
+    end
+
+    def defaults()
+      super
+      fstype            'xfs'
+      mount_options     "defaults,nobootwait,noatime,nouuid,comment=cluster_chef"
+      attachable        false
+      create_at_launch  false
+      #
+      mountable         true
+      resizable         false
+      formattable       true
+      #
+      in_raid           false
+      #
+      sub_volumes       []
+    end
+  end
 end

metadata

@@ -2,7 +2,7 @@
 name: cluster_chef
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 3.0.10
+  version: 3.0.11
 platform: ruby
 authors: 
 - Infochimps
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-17 00:00:00 Z
+date: 2012-01-24 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: chef
@@ -118,7 +118,7 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        version: 3.0.10
+        version: 3.0.11
   type: :runtime
   prerelease: false
   version_requirements: *id010
@@ -191,7 +191,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 905624580831723867
+      hash: 2883280125648369733
       segments: 
       - 0
       version: "0"