ironfan 5.0.11 → 6.0.0

data/lib/ironfan/provider/openstack/security_group.rb

@@ -0,0 +1,224 @@
+module Ironfan
+  class Provider
+    class OpenStack
+
+      class SecurityGroup < Ironfan::Provider::Resource
+
+        WIDE_OPEN = Range.new(1,65535)
+
+        delegate :_dump, :authorize_group_and_owner, :authorize_port_range,
+            :collection, :collection=, :connection, :connection=, :description,
+            :description=, :destroy, :group_id, :group_id=, :identity,
+            :identity=, :ip_permissions=, :name, :name=,
+            :new_record?, :owner_id, :owner_id=, :reload, :requires,
+            :requires_one, :revoke_group_and_owner, :revoke_port_range, :save,
+            :symbolize_keys, :wait_for, :name,
+            :create_security_group_rule,
+            :rules,
+          :to => :adaptee
+
+        def self.shared?()      true;   end
+        def self.multiple?()    true;   end
+        def self.resource_type()        :security_group;   end
+
+        def self.expected_ids(computer)
+          return unless computer.server
+          openstack = computer.server.cloud(:openstack)
+          server_groups = computer.server.security_groups
+          cloud_groups = openstack.security_groups
+
+          result = []
+          [server_groups, cloud_groups].each do |container|
+            container.each { |g| result.push(g.name) }
+          end
+          return result.uniq
+        end
+
+        def ip_permissions
+        end
+
+        def group_id
+          @adaptee.id
+        end
+
+        #
+        # Discovery
+        #
+        def self.load!(cluster=nil)
+          OpenStack.connection.security_groups.reject { |raw| raw.blank? }.each do |raw|
+            remember SecurityGroup.new(:adaptee => raw)
+          end
+        end
+
+        def receive_adaptee(obj)
+          obj = OpenStack.connection.security_groups.new(obj) if obj.is_a?(Hash)
+          super
+        end
+
+        def to_s
+          if ip_permissions.present?
+            perm_str = ip_permissions.map{|perm|
+              "%s:%s-%s (%s | %s)" % [
+                perm['ipProtocol'], perm['fromPort'], perm['toPort'],
+                perm['groups'  ].map{|el| el['groupName'] }.join(','),
+                perm['ipRanges'].map{|el| el['cidrIp']    }.join(','),
+              ]
+            }
+            return "<%-15s %-12s %-25s %s>" % [ self.class.handle, group_id, name, perm_str]
+          else
+            return "<%-15s %-12s %s>" % [ self.class.handle, group_id, name ]
+          end
+        end
+
+        #
+        # Manipulation
+        #
+        def self.prepare!(computers)
+          # Create any groups that don't yet exist, and ensure any authorizations
+          # that are required for those groups
+          cluster_name             = nil
+          groups_to_create         = [ ]
+          authorizations_to_ensure = [ ]
+
+          computers.each{|comp| ensure_groups(comp) if OpenStack.applicable(comp) } # Add facet and cluster security groups for the computer
+
+          # First, deduce the list of all groups to which at least one instance belongs
+          # We'll use this later to decide whether to create groups, or authorize access,
+          # using a VPC security group or an Openstack security group.
+          groups_that_should_exist = [] #computers.map{|comp| expected_ids(comp) }.flatten.compact.sort.uniq
+          groups_to_create << groups_that_should_exist
+
+          computers.select { |computer| OpenStack.applicable computer }.each do |computer|
+            cloud           = computer.server.cloud(:openstack)
+            cluster_name    = computer.server.cluster_name
+
+            # Iterate over all of the security group information, keeping track of
+            # any groups that must exist and any authorizations that must be ensured
+            [computer.server.security_groups, cloud.security_groups].each do |container|
+              
+              container.values.each do |dsl_group|
+
+                groups_to_create << dsl_group.name
+                
+                groups_to_create << dsl_group.group_authorized
+
+                groups_to_create << dsl_group.group_authorized_by
+
+                authorizations_to_ensure << dsl_group.group_authorized.map do |other_group|
+                  {
+                    :grantor      => dsl_group.name,
+                    :grantee      => other_group,
+                    :grantee_type => :group,
+                    :range        => WIDE_OPEN,
+                  }
+                end
+
+                authorizations_to_ensure << dsl_group.group_authorized_by.map do |other_group|
+                  {
+                    :grantor      => other_group,
+                    :grantee      => dsl_group.name,
+                    :grantee_type => :group,
+                    :range        => WIDE_OPEN,
+                  }
+                end
+
+                authorizations_to_ensure << dsl_group.range_authorizations.map do |range_auth|
+                  range, cidr, protocol = range_auth
+                  {
+                    :grantor      => dsl_group.name,
+                    :grantee      => { :cidr_ip => cidr, :ip_protocol => protocol },
+                    :grantee_type => :cidr,
+                    :range        => range,
+                  }
+                end
+              end
+            end
+          end
+          groups_to_create         = groups_to_create.flatten.uniq.reject { |group| recall? group.to_s }.sort
+          authorizations_to_ensure = authorizations_to_ensure.flatten.uniq.sort { |a,b| a[:grantor] <=> b[:grantor] }
+
+          Ironfan.step(cluster_name, "creating security groups", :blue) unless groups_to_create.empty?
+          groups_to_create.each do |group|
+            if group =~ /\//
+              Ironfan.step(group, "  assuming that owner/group pair #{group} already exists", :blue)
+            else
+              Ironfan.step(group, "  creating #{group} security group", :blue)
+              begin
+                tokens    = group.to_s.split(':')
+                group_id  = tokens.pop
+                OpenStack.connection.create_security_group(group_id,"Ironfan created group #{group_id}")
+              rescue Fog::Compute::OpenStack::Error => e # InvalidPermission.Duplicate
+                Chef::Log.info("ignoring security group error: #{e}")
+              end
+            end
+          end
+
+          # Re-load everything so that we have a @@known list of security groups to manipulate
+          load! unless groups_to_create.empty?
+
+          # Now make sure that all required authorizations are present
+          Ironfan.step(cluster_name, "ensuring security group permissions", :blue) unless authorizations_to_ensure.empty?
+          authorizations_to_ensure.each do |auth|
+            grantor_fog = recall(auth[:grantor])
+            if :group == auth[:grantee_type]
+              if fog_grantee = recall(auth[:grantee])
+                options = { :group => fog_grantee.group_id }
+              elsif auth[:grantee] =~ /\//
+                options = { :group_alias => auth[:grantee] }
+              else
+                raise "Don't know what to do with authorization grantee #{auth[:grantee]}"
+              end
+              message = "  ensuring access from #{auth[:grantee]} to #{auth[:grantor]}"
+            else
+              options = auth[:grantee]
+              message = "  ensuring #{auth[:grantee][:ip_protocol]} access from #{auth[:grantee][:cidr_ip]} to #{auth[:range]}"
+            end
+            Ironfan.step(auth[:grantor], message, :blue)
+            safely_authorize(grantor_fog, auth[:range], options)
+          end
+        end
+
+        #
+        # Utility
+        #
+        def self.ensure_groups(computer)
+          return unless OpenStack.applicable computer
+          # Ensure the security_groups include those for cluster & facet
+          # FIXME: This violates the DSL's immutability; it should be
+          #   something calculated from within the DSL construction
+          Ironfan.todo("CODE SMELL: violation of DSL immutability: #{caller}")
+          server = computer.server
+          c_group = server.security_group(computer.server.cluster_name)
+          c_group.authorized_by_group(c_group.name)
+          facet_name = "#{computer.server.cluster_name}-#{computer.server.facet_name}"
+          server.security_group(facet_name)
+        end
+
+
+        # Try an authorization, ignoring duplicates (this is easier than correlating).
+        # Do so for both TCP and UDP, unless only one is specified
+        def self.safely_authorize(fog_group,range,options)
+          if options[:group_alias]
+            owner, group = options[:group_alias].split(/\//)
+            self.patiently(fog_group.name, Fog::Compute::OpenStack::Error, :ignore => Proc.new { |e| e.message =~ /This rule already exists in group/ }) do
+              OpenStack.connection.authorize_security_group_ingress(
+                'GroupName'                   => fog_group.name,
+                'SourceSecurityGroupName'     => group,
+                'SourceSecurityGroupOwnerId'  => owner
+              )
+            end
+          elsif options[:ip_protocol]
+            self.patiently(fog_group.name, Excon::Errors::HTTPStatusError, :ignore => Proc.new { |e| e.message =~ /This rule already exists in group/ }) do
+              fog_group.create_security_group_rule(range.min, range.max, options[:ip_protocol], options[:cidr_ip], options[:group])
+            end 
+          else
+            safely_authorize(fog_group,range,options.merge(:ip_protocol => 'tcp'))
+            safely_authorize(fog_group,range,options.merge(:ip_protocol => 'udp'))
+            safely_authorize(fog_group,Range.new(-1,-1),options.merge(:ip_protocol => 'icmp')) if(range == WIDE_OPEN)
+            return
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ironfan/provider/openstack.rb

@@ -0,0 +1,69 @@
+module Ironfan
+  class Provider
+
+    class OpenStack < Ironfan::IaasProvider
+      self.handle = :openstack
+
+      def self.resources
+        [ Machine, Keypair, SecurityGroup, ElasticIp ]
+      end
+
+      #
+      # Utility functions
+      #
+      def self.connection
+        @@connection ||= Fog::Compute.new(self.openstack_credentials.merge({ :provider => 'openstack' }))
+      end
+
+
+      #
+      # Returns a hash that maps flavor names to flavors
+      #
+      def self.flavor_hash
+        @@flavors ||= self.connection.flavors.inject({}){|h,f| h[f.name]=f; h }
+      end
+
+      #
+      # Returns a hash that maps flavor ids to flavors
+      #
+      def self.flavor_id_hash
+        @@flavor_ids ||= self.connection.flavors.inject({}){|h,f| h[f.id]=f; h }
+      end
+
+
+      # Ensure that a fog object (machine, volume, etc.) has the proper tags on it
+      def self.ensure_tags(tags,fog)
+        # openstack does not have tags.
+
+        #tags.delete_if {|k, v| fog.tags[k] == v.to_s  rescue false }
+        #return if tags.empty?
+
+        #Ironfan.step(fog.name,"tagging with #{tags.inspect}", :green)
+        #tags.each do |k, v|
+        #  Chef::Log.debug( "tagging #{fog.name} with #{k} = #{v}" )
+        #  Ironfan.safely do
+        #    config = {:key => k, :value => v.to_s, :resource_id => fog.id }
+        #    connection.tags.create(config)
+        #  end
+        #end
+      end
+
+      def self.applicable(computer)
+        computer.server and computer.server.clouds.include?(:openstack)
+      end
+
+      private
+
+      def self.openstack_credentials
+        return {
+          :openstack_api_key   =>   Chef::Config[:knife][:openstack_api_key],
+          :openstack_username  =>   Chef::Config[:knife][:openstack_username],
+          :openstack_auth_url  =>   Chef::Config[:knife][:openstack_auth_url],
+          :openstack_tenant    =>   Chef::Config[:knife][:openstack_tenant],
+          :openstack_region    =>   Chef::Config[:knife][:openstack_region],
+        }
+      end
+
+    end
+  end
+end

data/lib/ironfan/provider/static/machine.rb

@@ -0,0 +1,192 @@
+module Ironfan
+  class Provider
+    class Static
+      class Machine < Ironfan::IaasProvider::Machine
+
+        def self.shared?()      false;  end
+        def self.multiple?()    false;  end
+        def self.resource_type()        :machine;   end
+        def self.expected_ids(computer) [computer.server.full_name];   end
+
+        def name()  adaptee.full_name; end
+        def tags
+          t = {"Name" => @adaptee.name}
+          return t.keys.inject({}) {|h,k| h[k]=t[k]; h[k.to_sym]=t[k]; h}
+        end
+
+        def vpc_id
+          return nil
+        end
+
+        def created_at
+          nil
+        end
+
+        def flavor_id
+          return nil
+        end
+
+        def flavor_name
+          "nil"
+        end
+
+        def image_id
+          "none"
+        end
+
+        def groups           ; []   ;   end
+
+        def public_hostname    ; adaptee.cloud(:static).public_hostname || public_ip_address ; end
+        def public_ip_address  ; adaptee.cloud(:static).public_ip || private_ip_address ; end
+        def dns_name            ; public_ip_address ; end
+
+        def keypair          ; adaptee.cloud(:static).keypair ; end
+
+        def created?
+          false
+        end
+        def pending?
+          false
+        end
+        def running?
+          true
+        end
+        def stopping?
+          false
+        end
+
+        def stopped?
+          false
+        end
+        
+        def error?
+          false
+        end
+
+        def start
+        end
+
+        def stop
+        end
+
+        def perform_after_launch_tasks?
+          true
+        end
+
+        def to_display(style,values={})
+          # style == :minimal
+          values["State"] =             "???"
+          values["MachineID"] =         private_ip_address
+          values["Public IP"] =         private_ip_address
+          values["Private IP"] =        public_ip_address
+          values["Created On"] =        "???"
+          return values if style == :minimal
+
+          # style == :default
+          values["Flavor"] =            nil
+          values["AZ"] =                nil
+          return values if style == :default
+
+          # style == :expanded
+          values["Image"] =             "none"
+          #values["Volumes"] =           volumes.map(&:id).join(', ')
+          values["SSH Key"] =           "none"
+          values
+        end
+
+        def ssh_key
+          keypair = cloud.keypair || computer.server.cluster_name
+        end
+
+        def key_name
+          keypair
+        end
+
+        def private_ip_address
+          adaptee.cloud(:static).private_ip
+        end
+
+        def availability_zone
+          'none'
+        end
+
+        def destroy
+        end
+
+        def to_s
+          "<%-15s %-12s %-25s %-25s %-15s %-15s %-12s %-12s %s:%s>" % [
+            self.class.handle, "", created_at, name, private_ip_address, public_ip_address, flavor_name, availability_zone, key_name, groups.join(',') ]
+        end
+
+        #
+        # Discovery
+        #
+        def self.load!(cluster=nil)
+          cluster.facets.each do |facet| 
+            facet.servers.each do |server|
+              machine = new(:adaptee => server)
+              remember machine
+            end
+          end
+        end
+
+        # Find active machines that haven't matched, but should have,
+        #   make sure all bogus machines have a computer to attach to
+        #   for display purposes
+        def self.validate_resources!(computers)
+          recall.each_value do |machine|
+            next unless machine.users.empty? and machine.name
+            if machine.name.match("^#{computers.cluster.name}-")
+              machine.bogus << :unexpected_machine
+            end
+            next unless machine.bogus?
+            fake           = Ironfan::Broker::Computer.new
+            fake[:machine] = machine
+            fake.name      = machine.name
+            machine.users << fake
+            computers     << fake
+          end
+        end
+
+        #
+        # Manipulation
+        #
+        def self.create!(computer)
+          nil
+        end
+
+        # @returns [Hash{String, Array}] of 'what you did wrong' => [relevant, info]
+        def self.lint(computer)
+          cloud = computer.server.cloud(:static)
+          info  = [computer.name, cloud.inspect]
+          errors = {}
+          server_errors = computer.server.lint
+          errors["No Private IP"]       = info if cloud.private_ip.blank?
+          errors
+        end
+
+        def self.launch_description(computer)
+          nil
+        end
+
+        # An array of hashes with dorky-looking keys, just like Fog wants it.
+        def self.block_device_mapping(computer)
+          []
+        end
+
+        def self.destroy!(computer)
+          return unless computer.machine?
+          forget computer.machine.name
+          computer.machine.destroy
+          computer.machine.reload            # show the node as shutting down
+        end
+
+        def self.save!(computer)
+          return unless computer.machine?
+          return unless computer.created?
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ironfan/provider/static.rb

@@ -0,0 +1,23 @@
+module Ironfan
+  class Provider
+    class Static < Ironfan::IaasProvider
+      self.handle = :static
+
+      def self.resources
+        [ Machine ]
+      end
+
+      def self.ensure_tags(tags,fog)
+        #  Ironfan.safely do
+        #    config = {:key => k, :value => v.to_s, :resource_id => fog.id }
+        #    connection.tags.create(config)
+        #  end
+        #end
+      end
+
+      def self.applicable(computer)
+        computer.server and computer.server.clouds.include?(:static)
+      end
+    end
+  end
+end

data/lib/ironfan/provider.rb

@@ -13,6 +13,8 @@ module Ironfan
           case obj[:name]
           when :chef        then Chef
           when :ec2         then Ec2
+          when :openstack   then OpenStack
+          when :static      then Static
           when :vsphere     then Vsphere
           when :virtualbox  then VirtualBox
           when :rds         then Rds
@@ -204,7 +206,62 @@ module Ironfan
     class Machine < Resource
       # A Machine lives and dies with its Computer
       def self.shared?()        false;                   end
+
+      def self.cloud_init_user_data(computer) 
+        return <<EOF
+#cloud-config
+# The line above is critical - without it cloud-init will not interpret the machine
+# user data as a cloud-init script.
+
+# The following logs everything cloud init does to /var/log
+output: {all: '| tee -a /var/log/cloud-init-output.log'}
+
+# Set our hostname
+manage_etc_hosts: True
+
+cloud_init_modules:
+ - bootcmd
+ - resizefs
+ - set_hostname
+ - update_hostname
+ - update_etc_hosts
+ - write-files
+ - ca-certs
+ - rsyslog
+ - ssh
+
+fqdn: #{computer.server.fqdn}
+
+bootcmd:
+  # note that writefiles is not supported on precise...
+  - |
+      mkdir -p /etc/chef
+      touch /etc/chef/client.pem
+      chmod 600 /etc/chef/client.pem
+      chown root:root /etc/chef/client.pem
+      cat > /etc/chef/client.pem << EOF
+#{computer.private_key.split("\n").map {|l| "      "+l}.join("\n")}
+      EOF
+      domainname #{computer.server.fqdn}
+ 
+chef:
+ install_type: "packages"
+ force_install: false
+ server_url: #{Chef::Config[:chef_server_url]}
+ node_name: #{computer.name}
+ initial_attributes:
+    chef_server: #{Chef::Config[:chef_server_url]}
+    node_name: #{computer.name}
+    organization: #{Chef::Config[:organization]}
+    realm_name: #{computer.server.realm_name}
+    cluster_name: #{computer.server.cluster_name}
+    facet_name: #{computer.server.facet_name}
+    facet_index: #{computer.server.index}
+ validation_name: "no-validator"
+ validation_key: |
+    We don't need no stinking validators.
+EOF
+      end
     end
   end
-
 end

data/lib/ironfan/requirements.rb

@@ -1,7 +1,10 @@
+# Built on top of chef
+require 'chef'
+
 # Gorillib core classes
 require 'gorillib/builder'
+require 'gorillib/diff'
 require 'gorillib/resolution'
-
 require 'gorillib/nil_check_delegate'
 
 # Pre-declaration of class hierarchy
@@ -11,10 +14,12 @@ require 'ironfan/headers'
 require 'ironfan/plugin/base'
 
 # DSL for cluster descriptions
+require 'ironfan/cookbook_requirements'
 require 'ironfan/dsl'
 require 'ironfan/builder'
 
 require 'ironfan/dsl/component'
+require 'ironfan/dsl/security_group'
 require 'ironfan/dsl/compute'
 require 'ironfan/dsl/server'
 require 'ironfan/dsl/facet'
@@ -26,6 +31,8 @@ require 'ironfan/dsl/volume'
 
 require 'ironfan/dsl/cloud'
 require 'ironfan/dsl/ec2'
+require 'ironfan/dsl/openstack'
+require 'ironfan/dsl/static'
 require 'ironfan/dsl/vsphere'
 require 'ironfan/dsl/rds'
 
@@ -48,6 +55,15 @@ require 'ironfan/provider/ec2/elastic_ip'
 require 'ironfan/provider/ec2/elastic_load_balancer'
 require 'ironfan/provider/ec2/iam_server_certificate'
 
+require 'ironfan/provider/openstack'
+require 'ironfan/provider/openstack/machine'
+require 'ironfan/provider/openstack/keypair'
+require 'ironfan/provider/openstack/security_group'
+require 'ironfan/provider/openstack/elastic_ip'
+
+require 'ironfan/provider/static'
+require 'ironfan/provider/static/machine'
+
 require 'ironfan/provider/virtualbox'
 require 'ironfan/provider/virtualbox/machine'
 

data/lib/ironfan/version.rb

@@ -0,0 +1,3 @@
+module Ironfan
+  VERSION = '6.0.0'
+end