ironfan 4.6.2 → 4.7.0

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# v4.7.0:
+(@nickmarden rocks the house again)
+* Added support for "prepare" phase, prior to any machine-specific actions
+* Move security group creation and authorization assurance to prepare phase (fixes #189)
+* Allow user/group-style security group references (fixes #207)
+* Move keypair creation to prepare phase
+
 # v4.6.2:
 * Added a -f/--with-facet option to knife cluster list
 

data/VERSION

@@ -1 +1 @@
-4.6.2
+4.7.0

data/ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.6.2"
+  s.version = "4.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-12-13"
+  s.date = "2012-12-14"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [

data/lib/chef/knife/cluster_kill.rb

@@ -75,6 +75,10 @@ class Chef
         confirm_or_exit("Are you absolutely certain that you want to delete #{delete_message}? (Type 'Yes' to confirm) ", 'Yes')
       end
 
+      def prepares?
+        false
+      end
+
     end
   end
 end

data/lib/chef/knife/cluster_launch.rb

@@ -82,6 +82,11 @@ class Chef
         section("Syncing to chef")
         target.save :providers => :chef
 
+        unless target.empty?
+          ui.info "Preparing shared resources:"
+          all_computers(*@name_args).prepare
+        end
+
         # Launch computers
         ui.info("")
         section("Launching computers", :green)

data/lib/chef/knife/cluster_proxy.rb

@@ -119,6 +119,10 @@ class Chef
 }\n}
       end
 
+      def prepares?
+        false
+      end
+
       def aggregates?
         false
       end

data/lib/chef/knife/cluster_sync.rb

@@ -62,6 +62,10 @@ class Chef
         else Chef::Log.debug("Skipping sync to cloud") ; end
       end
 
+      def prepares_on_noop?
+        true
+      end
+
       def aggregates_on_noop?
         true
       end

data/lib/chef/knife/ironfan_script.rb

@@ -44,6 +44,11 @@ module Ironfan
 
       target = get_relevant_slice(* @name_args)
 
+      if prepares? and (prepares_on_noop? or not target.empty?)
+        ui.info "Preparing shared resources:"
+        all_computers(*@name_args).prepare
+      end
+
       unless target.empty?
         ui.info(["\n",
                  ui.color("Running #{sub_command}", :cyan),
@@ -76,6 +81,14 @@ module Ironfan
       target.send(sub_command)
     end
 
+    def prepares?
+      true
+    end
+
+    def prepares_on_noop?
+      false
+    end
+
     def aggregates?
       true
     end

data/lib/ironfan/broker/computer.rb

@@ -298,11 +298,19 @@ module Ironfan
         values.map {|c| c.providers.values}.flatten.uniq.each {|p| p.validate computers }
       end
 
-      def aggregate
+      def group_action(verb)
         computers = self
-        provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas})}.flatten.uniq
+        provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas })}.flatten.uniq
         providers     = provider_keys.map { |pk| values.map { |c| c.providers[pk] } }.flatten.compact.uniq
-        providers.each { |p| p.aggregate! computers }
+        providers.each { |p| p.send(verb, computers) }
+      end
+
+      def prepare
+        group_action(:prepare!)
+      end
+
+      def aggregate
+        group_action(:aggregate!)
       end
 
       #

data/lib/ironfan/provider.rb

@@ -42,6 +42,12 @@ module Ironfan
       resources.each {|r| r.validate_resources! computers }
     end
 
+    def self.prepare!(computers)
+      resources.each do |r|
+        r.prepare!(computers) if r.shared?
+      end
+    end
+
     def self.aggregate!(computers)
       resources.each do |r|
         r.aggregate!(computers) if r.shared?
@@ -87,14 +93,15 @@ module Ironfan
       #
       def self.create!(*p)              Ironfan.noop(self,__method__,*p);   end
       def self.save!(*p)                Ironfan.noop(self,__method__,*p);   end
+      def self.prepare!(*p)             Ironfan.noop(self,__method__,*p);   end
       def self.aggregate!(*p)           Ironfan.noop(self,__method__,*p);   end
       def self.destroy!(*p)             Ironfan.noop(self,__method__,*p);   end
 
       #
       # Utilities
       #
-      [:shared?, :multiple?, :load!,:validate_computer!,
-       :validate_resources!,:create!,:save!,:aggregate!,:destroy!].each do |method_name|
+      [:shared?, :multiple?, :load!,:validate_computer!, :validate_resources!,
+       :create!, :save!, :prepare!, :aggregate!, :destroy!].each do |method_name|
          define_method(method_name) {|*p| self.class.send(method_name,*p) }
        end
 

data/lib/ironfan/provider/ec2/keypair.rb

@@ -50,8 +50,9 @@ module Ironfan
         # Manipulation
         #
 
-        def self.create!(computer)
-          name = computer.server.cluster_name
+        def self.prepare!(computers)
+          return if computers.empty?
+          name = computers.values[0].server.cluster_name
           return if recall? name
           Ironfan.step(name, "creating key pair for #{name}", :blue)
           result = Ec2.connection.create_key_pair(name)

data/lib/ironfan/provider/ec2/security_group.rb

@@ -20,22 +20,18 @@ module Ironfan
         def self.resource_type()        :security_group;   end
         def self.expected_ids(computer)
           ec2 = computer.server.cloud(:ec2)
-          ec2.security_groups.keys.map do |name|
-            ec2.vpc ? "#{ec2.vpc}:#{name.to_s}" : name.to_s
-          end.uniq
+          ec2.security_groups.keys.map { |name| group_name_with_vpc(name,ec2.vpc) }.uniq
         end
 
         def name()
-          return adaptee.name if adaptee.vpc_id.nil?
-          "#{adaptee.vpc_id}:#{adaptee.name}"
+          self.class.group_name_with_vpc(adaptee.name, adaptee.vpc_id)
         end
 
         #
         # Discovery
         #
         def self.load!(cluster=nil)
-          Ec2.connection.security_groups.each do |raw|
-            next if raw.blank?
+          Ec2.connection.security_groups.reject { |raw| raw.blank? }.each do |raw|
             sg = SecurityGroup.new(:adaptee => raw)
             remember(sg)
             Chef::Log.debug("Loaded #{sg}: #{sg.inspect}")
@@ -66,75 +62,119 @@ module Ironfan
         # Manipulation
         #
 
-        def self.create!(computer)
-          return unless Ec2.applicable computer
+        def self.prepare!(computers)
+
+          # Create any groups that don't yet exist, and ensure any authorizations
+          # that are required for those groups
+          cluster_name             = nil
+          groups_to_create         = [ ]
+          authorizations_to_ensure = [ ]
+
+          # First, deduce the list of all groups to which at least one instance belongs
+          # We'll use this later to decide whether to create groups, or authorize access,
+          # using a VPC security group or an EC2 security group.
+          groups_that_should_exist = computers.map { |c| expected_ids(c) }.flatten.sort.uniq
+          groups_to_create << groups_that_should_exist
+
+          computers.select { |computer| Ec2.applicable computer }.each do |computer|
+            ensure_groups(computer) # Add facet and cluster security groups for the computer
+            cloud           = computer.server.cloud(:ec2)
+            cluster_name    = computer.server.cluster_name
+
+            # Iterate over all of the security group information, keeping track of
+            # any groups that must exist and any authorizations that must be ensured
+            cloud.security_groups.values.each do |dsl_group|
+
+              groups_to_create << dsl_group.name
+
+              groups_to_create << dsl_group.group_authorized.map do |other_group|
+                most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist)
+              end
+
+              groups_to_create << dsl_group.group_authorized_by.map do |other_group|
+                most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist)
+              end
+
+              authorizations_to_ensure << dsl_group.group_authorized.map do |other_group|
+                {
+                  :grantor      => most_appropriate_group_name(dsl_group.name, cloud.vpc, groups_that_should_exist),
+                  :grantee      => most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist),
+                  :grantee_type => :group,
+                  :range        => WIDE_OPEN,
+                }
+              end
+
+              authorizations_to_ensure << dsl_group.group_authorized_by.map do |other_group|
+                {
+                  :grantor      => most_appropriate_group_name(other_group, cloud.vpc, groups_that_should_exist),
+                  :grantee      => most_appropriate_group_name(dsl_group.name, cloud.vpc, groups_that_should_exist),
+                  :grantee_type => :group,
+                  :range        => WIDE_OPEN,
+                }
+              end
+
+              authorizations_to_ensure << dsl_group.range_authorizations.map do |range_auth|
+                range, cidr, protocol = range_auth
+                {
+                  :grantor      => group_name_with_vpc(dsl_group.name, cloud.vpc),
+                  :grantee      => { :cidr_ip => cidr, :ip_protocol => protocol },
+                  :grantee_type => :cidr,
+                  :range        => range,
+                }
+              end
+            end
+          end
+          groups_to_create         = groups_to_create.flatten.uniq.reject { |group| recall? group.to_s }.sort
+          authorizations_to_ensure = authorizations_to_ensure.flatten.uniq.sort { |a,b| a[:grantor] <=> b[:grantor] }
+
+          Ironfan.step(cluster_name, "creating security groups", :blue) unless groups_to_create.empty?
+          groups_to_create.each do |group|
+            if group =~ /\//
+              Ironfan.step(group, "  assuming that owner/group pair #{group} already exists", :blue)
+            else
+              Ironfan.step(group, "  creating #{group} security group", :blue)
+              begin
+                tokens    = group.to_s.split(':')
+                group_id  = tokens.pop
+                vpc_id    = tokens.pop
+                Ec2.connection.create_security_group(group_id,"Ironfan created group #{group_id}",vpc_id)
+              rescue Fog::Compute::AWS::Error => e # InvalidPermission.Duplicate
+                Chef::Log.info("ignoring security group error: #{e}")
+              end
+            end
+          end
 
-          ensure_groups(computer)
-          groups = self.expected_ids(computer)
-          # Only handle groups that don't already exist
-          groups.delete_if {|group| recall? group.to_s }
-          return if groups.empty?
-
-          Ironfan.step(computer.server.cluster_name, "creating security groups", :blue)
-          groups.each do |group|
-            Ironfan.step(group, "  creating #{group} security group", :blue)
-            begin
-              tokens    = group.to_s.split(':')
-              group_id  = tokens.pop
-              vpc_id    = tokens.pop
-              Ec2.connection.create_security_group(group_id,"Ironfan created group #{group_id}",vpc_id)
-            rescue Fog::Compute::AWS::Error => e # InvalidPermission.Duplicate
-              Chef::Log.info("ignoring security group error: #{e}")
-              sleep 0.5  # quit racing so hard
+          # Re-load everything so that we have a @@known list of security groups to manipulate
+          load! unless groups_to_create.empty?
+
+          # Now make sure that all required authorizations are present
+          Ironfan.step(cluster_name, "ensuring security group permissions", :blue) unless authorizations_to_ensure.empty?
+          authorizations_to_ensure.each do |auth|
+            grantor_fog = recall(auth[:grantor])
+            if :group == auth[:grantee_type]
+              if fog_grantee = recall(auth[:grantee])
+                options = { :group => fog_grantee.group_id }
+              elsif auth[:grantee] =~ /\//
+                options = { :group_alias => auth[:grantee] }
+              else
+                raise "Don't know what to do with authorization grantee #{auth[:grantee]}"
+              end
+              message = "  ensuring access from #{auth[:grantee]} to #{auth[:grantor]}"
+            else
+              options = auth[:grantee]
+              message = "  ensuring #{auth[:grantee][:ip_protocol]} access from #{auth[:grantee][:cidr_ip]} to #{auth[:range]}"
             end
+            Ironfan.step(auth[:grantor], message, :blue)
+            safely_authorize(grantor_fog, auth[:range], options)
           end
-          load! # Get the native groups via reload
         end
 
-        def self.recall_with_vpc(name,vpc_id=nil)
-          group_name = vpc_id.nil? ? name : "#{vpc_id}:#{name}"
-          recall(group_name)
+        def self.group_name_with_vpc(name,vpc_id=nil)
+          vpc_id.nil? ? name.to_s : "#{vpc_id}:#{name.to_s}"
         end
 
-        def self.save!(computer)
-          return unless Ec2.applicable computer
-          cloud = computer.server.cloud(:ec2)
-
-          create!(computer)            # Make sure the security groups exist
-          security_groups = cloud.security_groups.values
-          dsl_groups = security_groups.select do |dsl_group|
-            not (recall_with_vpc(dsl_group,cloud.vpc)) and \
-            not (dsl_group.range_authorizations +
-                 dsl_group.group_authorized_by +
-                 dsl_group.group_authorized).empty?
-          end.compact
-          return if dsl_groups.empty?
-
-          Ironfan.step(computer.server.cluster_name, "ensuring security group permissions", :blue)
-          dsl_groups.each do |dsl_group|
-            dsl_group_fog = recall_with_vpc(dsl_group.name,cloud.vpc)
-            dsl_group.group_authorized.each do |other_group|
-              other_group_fog = recall_with_vpc(other_group,cloud.vpc)
-              Ironfan.step(dsl_group.name, "  ensuring access from #{other_group}", :blue)
-              options = {:group => other_group_fog.group_id}
-              safely_authorize(dsl_group_fog, WIDE_OPEN, options)
-            end
-
-            dsl_group.group_authorized_by.each do |other_group|
-              other_group_fog = recall_with_vpc(other_group,cloud.vpc)
-              Ironfan.step(dsl_group.name, "  ensuring access to #{other_group}", :blue)
-              options = {:group => dsl_group_fog.group_id}
-              safely_authorize(other_group_fog, WIDE_OPEN, options)
-            end
-
-            dsl_group.range_authorizations.each do |range_auth|
-              range, cidr, protocol = range_auth
-              step_message = "  ensuring #{protocol} access from #{cidr} to #{range}"
-              Ironfan.step(dsl_group.name, step_message, :blue)
-              options = {:cidr_ip => cidr, :ip_protocol => protocol}
-              safely_authorize(dsl_group_fog, range, options)
-            end
-          end
+        def self.most_appropriate_group_name(group, vpc_id, all_valid_groups)
+          all_valid_groups.include?(group_name_with_vpc(group, vpc_id)) ? group_name_with_vpc(group, vpc_id) : group
         end
 
         #
@@ -156,20 +196,27 @@ module Ironfan
         # Try an authorization, ignoring duplicates (this is easier than correlating).
         # Do so for both TCP and UDP, unless only one is specified
         def self.safely_authorize(fog_group,range,options)
-          unless options[:ip_protocol]
+          if options[:group_alias]
+            owner, group = options[:group_alias].split(/\//)
+            self.patiently(fog_group.name, Fog::Compute::AWS::Error, :ignore => Proc.new { |e| e.message =~ /InvalidPermission\.Duplicate/ }) do
+              Ec2.connection.authorize_security_group_ingress(
+                'GroupName'                   => fog_group.name,
+                'SourceSecurityGroupName'     => group,
+                'SourceSecurityGroupOwnerId'  => owner
+              )
+            end
+          elsif options[:ip_protocol]
+            self.patiently(fog_group.name, Fog::Compute::AWS::Error, :ignore => Proc.new { |e| e.message =~ /InvalidPermission\.Duplicate/ }) do
+              fog_group.authorize_port_range(range,options)
+            end
+          else
             safely_authorize(fog_group,range,options.merge(:ip_protocol => 'tcp'))
             safely_authorize(fog_group,range,options.merge(:ip_protocol => 'udp'))
             safely_authorize(fog_group,Range.new(-1,-1),options.merge(:ip_protocol => 'icmp')) if(range == WIDE_OPEN)
             return
           end
-
-          self.patiently(fog_group.name, Fog::Compute::AWS::Error, :ignore => Proc.new { |e| e.message =~ /InvalidPermission\.Duplicate/ }) do
-            fog_group.authorize_port_range(range,options)
-          end
-
         end
       end
-
     end
   end
 end

data/spec/integration/spec/simple_cluster_spec.rb

@@ -17,6 +17,10 @@ Ironfan.cluster "simple" do
 
   facet :web do
     instances 1
+    cloud(:ec2).security_group(:web) do
+      authorize_group :web_clients
+      authorize_group 'amazon-elb/amazon-elb-sg'
+    end
   end
 
   facet :db do
@@ -35,7 +39,7 @@ launch_cluster 'simple' do |cluster, computers|
 
     describe "the web facet security groups" do
       subject { cluster.facets[:web].server(0).cloud(:ec2).security_groups.keys.map(&:to_s).sort }
-      it { should == %w[ simple simple-web ssh systemwide ] }
+      it { should == %w[ simple simple-web ssh systemwide web ] }
     end
 
     describe "the db facet security groups" do
@@ -43,6 +47,12 @@ launch_cluster 'simple' do |cluster, computers|
       it { should == %w[ simple simple-db ssh systemwide ] }
     end
 
+    describe "the passively created security groups" do
+      it "should include the :web_clients group" do
+        Ironfan::Provider::Ec2::SecurityGroup.recall('web_clients').should_not be_nil
+      end
+    end
+
     describe "the cluster-wide security group" do
       before :each do
         @sg = Ironfan::Provider::Ec2::SecurityGroup.recall('simple')
@@ -76,7 +86,17 @@ launch_cluster 'simple' do |cluster, computers|
         @ordered_ipp['icmp']['fromPort'].to_i.should              == -1
         @ordered_ipp['icmp']['toPort'].to_i.should                == -1
       end
+    end
+
+    describe "the web security group" do
+      before :each do
+        @sg = Ironfan::Provider::Ec2::SecurityGroup.recall('web')
+        @ordered_ipp = Hash[ @sg.ip_permissions.map { |s| [ s['ipProtocol'], s ] } ]
+      end
 
+      it "allows TCP connections to web_clients and to amazon-elb-sg" do
+        @ordered_ipp['tcp']['groups'].map { |g| g['groupName'] }.sort.should == %w[ amazon-elb-sg web_clients ]
+      end
     end
   end
 end

metadata

@@ -2,7 +2,7 @@
 name: ironfan
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 4.6.2
+  version: 4.7.0
 platform: ruby
 authors: 
 - Infochimps
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-12-13 00:00:00 Z
+date: 2012-12-14 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: chef
@@ -282,7 +282,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 4084772978397561372
+      hash: -2214564911911745674
       segments: 
       - 0
       version: "0"