ironfan 3.1.0.rc1

data/lib/ironfan/role_implications.rb

@@ -0,0 +1,58 @@
+module Ironfan
+  ComputeBuilder.class_eval do
+
+    # organization-wide security group
+    role_implication "systemwide" do
+      self.cloud.security_group "systemwide" do
+      end
+    end
+
+    # NFS server allows access from nfs_clients
+    role_implication "nfs_server" do
+      self.cloud.security_group "nfs_server" do
+        authorize_group "nfs_client"
+      end
+    end
+
+    role_implication "nfs_client" do
+      self.cloud.security_group "nfs_client"
+    end
+
+    # Opens port 22 to the world
+    role_implication "ssh" do
+      self.cloud.security_group 'ssh' do
+        authorize_port_range 22..22
+      end
+    end
+
+    # Open the Chef server API port (4000) and the webui (4040)
+    role_implication "chef_server" do
+      self.cloud.security_group "chef_server" do
+        authorize_port_range 4000..4000  # chef-server-api
+        authorize_port_range 4040..4040  # chef-server-webui
+      end
+    end
+
+    # web server? add the group "web_server" to open the web holes
+    role_implication "web_server" do
+      self.cloud.security_group("#{cluster_name}-web_server") do
+        authorize_port_range  80..80
+        authorize_port_range 443..443
+      end
+    end
+
+    # if you're a redis server, open the port and authorize redis clients in your group to talk to you
+    role_implication("redis_server") do
+      cluster_name = self.cluster_name # hack: put cluster_name is in scope
+      self.cloud.security_group("#{cluster_name}-redis_server") do
+        authorize_group("#{cluster_name}-redis_client")
+      end
+    end
+
+    # redis_clients gain rights to the redis_server
+    role_implication("redis_client") do
+      self.cloud.security_group("#{cluster_name}-redis_client")
+    end
+
+  end
+end

data/lib/ironfan/security_group.rb

@@ -0,0 +1,119 @@
+module Ironfan
+  module Cloud
+
+    class SecurityGroup < DslObject
+      has_keys :name, :description, :owner_id
+      attr_reader :group_authorizations
+      attr_reader :range_authorizations
+
+      def initialize cloud, group_name, group_description=nil, group_owner_id=nil
+        super()
+        set :name, group_name.to_s
+        description group_description || "ironfan generated group #{group_name}"
+        @cloud         = cloud
+        @group_authorizations = []
+        @group_authorized_by  = []
+        @range_authorizations = []
+        owner_id(group_owner_id || Chef::Config[:knife][:aws_account_id])
+      end
+
+      @@all = nil
+      def all
+        self.class.all
+      end
+      def self.all
+        return @@all if @@all
+        get_all
+      end
+      def self.get_all
+        groups_list = Ironfan.fog_connection.security_groups.all
+        @@all = groups_list.inject(Mash.new) do |hsh, fog_group|
+          hsh[fog_group.name] = fog_group ; hsh
+        end
+      end
+
+      def get
+        all[name] || Ironfan.fog_connection.security_groups.get(name)
+      end
+
+      def self.get_or_create(group_name, description)
+        # FIXME: the '|| Ironfan.fog' part is probably unnecessary
+        fog_group = all[group_name] || Ironfan.fog_connection.security_groups.get(group_name)
+        unless fog_group
+          self.step(group_name, "creating (#{description})", :green)
+          fog_group = all[group_name] = Ironfan.fog_connection.security_groups.new(:name => group_name, :description => description, :connection => Ironfan.fog_connection)
+          fog_group.save
+        end
+        fog_group
+      end
+
+      def authorize_group(group_name, owner_id=nil)
+        @group_authorizations << [group_name.to_s, owner_id]
+      end
+
+      def authorized_by_group(other_name)
+        @group_authorized_by << other_name.to_s
+      end
+
+      def authorize_port_range(range, cidr_ip = '0.0.0.0/0', ip_protocol = 'tcp')
+        range = (range .. range) if range.is_a?(Integer)
+        @range_authorizations << [range, cidr_ip, ip_protocol]
+      end
+
+      def group_permission_already_set?(fog_group, other_name, authed_owner)
+        return false if fog_group.ip_permissions.nil?
+        fog_group.ip_permissions.any? do |existing_permission|
+          existing_permission["groups"].include?({"userId" => authed_owner, "groupName" => other_name}) &&
+            existing_permission["fromPort"] == 1 &&
+            existing_permission["toPort"]   == 65535
+        end
+      end
+
+      def range_permission_already_set?(fog_group, range, cidr_ip, ip_protocol)
+        return false if fog_group.ip_permissions.nil?
+        fog_group.ip_permissions.include?(
+          { "groups"=>[], "ipRanges"=>[{"cidrIp"=>cidr_ip}],
+            "ipProtocol"=>ip_protocol, "fromPort"=>range.first, "toPort"=>range.last})
+      end
+
+      # FIXME: so if you're saying to yourself, "self, this is some soupy gooey
+      # code right here" then you and your self are correct. Much of this is to
+      # work around old limitations in the EC2 api. You can now treat range and
+      # group permissions the same, and we should.
+
+      def run
+        fog_group = self.class.get_or_create(name, description)
+        @group_authorizations.uniq.each do |other_name, authed_owner|
+          authed_owner ||= self.owner_id
+          next if group_permission_already_set?(fog_group, other_name, authed_owner)
+          step("authorizing access from all machines in #{other_name} to #{name}", :blue)
+          self.class.get_or_create(other_name, "Authorized to access #{name}")
+          begin  fog_group.authorize_group_and_owner(other_name, authed_owner)
+          rescue StandardError => e ; ui.warn e ; end
+        end
+        @group_authorized_by.uniq.each do |other_name|
+          authed_owner = self.owner_id
+          other_group = self.class.get_or_create(other_name, "Authorized for access by #{self.name}")
+          next if group_permission_already_set?(other_group, self.name, authed_owner)
+          step("authorizing access to all machines in #{other_name} from #{name}", :blue)
+          begin  other_group.authorize_group_and_owner(self.name, authed_owner)
+          rescue StandardError => e ; ui.warn e ; end
+        end
+        @range_authorizations.uniq.each do |range, cidr_ip, ip_protocol|
+          next if range_permission_already_set?(fog_group, range, cidr_ip, ip_protocol)
+          step("opening #{ip_protocol} ports #{range} to #{cidr_ip}", :blue)
+          begin  fog_group.authorize_port_range(range, { :cidr_ip => cidr_ip, :ip_protocol => ip_protocol })
+          rescue StandardError => e ; ui.warn e ; end
+        end
+      end
+
+      def self.step(group_name, desc, *style)
+        ui.info("  group #{"%-15s" % (group_name+":")}\t#{ui.color(desc.to_s, *style)}")
+      end
+      def step(desc, *style)
+        self.class.step(self.name, desc, *style)
+      end
+
+    end
+  end
+end

data/lib/ironfan/server.rb

@@ -0,0 +1,281 @@
+module Ironfan
+
+  #
+  # A server is a specific (logical) member of a facet within a cluster.
+  #
+  # It may have extra attributes if it also exists in the Chef server,
+  # or if it exists in the real world (as revealed by Fog)
+  #
+  class Server < Ironfan::ComputeBuilder
+    attr_reader   :cluster, :facet, :facet_index, :tags
+    attr_accessor :chef_node, :fog_server
+
+    @@all ||= Mash.new
+
+    def initialize facet, idx
+      @cluster     = facet.cluster
+      @facet       = facet
+      @facet_index = idx
+      @fullname    = [cluster_name, facet_name, facet_index].join('-')
+      super(@fullname)
+      @tags = { "name" => name, "cluster" => cluster_name, "facet"   => facet_name, "index" => facet_index, }
+      ui.warn("Duplicate server #{[self, facet.name, idx]} vs #{@@all[fullname]}") if @@all[fullname]
+      @@all[fullname] = self
+    end
+
+    def fullname fn=nil
+      @fullname = fn if fn
+      @fullname
+    end
+
+    def cluster_name
+      cluster.name
+    end
+
+    def facet_name
+      facet.name
+    end
+
+    def servers
+      Ironfan::ServerGroup.new(cluster, [self])
+    end
+
+    def bogosity val=nil
+      @settings[:bogosity] = val  if not val.nil?
+      return @settings[:bogosity] if not @settings[:bogosity].nil?
+      return :bogus_facet         if facet.bogus?
+      # return :out_of_range      if (self.facet_index.to_i >= facet.instances)
+      false
+    end
+
+    def in_cloud?
+      !! fog_server
+    end
+
+    def in_chef?
+      chef_node || chef_client
+    end
+
+    def has_cloud_state?(*states)
+      in_cloud? && states.flatten.include?(fog_server.state)
+    end
+
+    def exists?
+      created? || in_chef?
+    end
+
+    def created?
+      in_cloud? && (not ['terminated', 'shutting-down'].include?(fog_server.state))
+    end
+
+    def running?
+      has_cloud_state?('running')
+    end
+
+    def startable?
+      has_cloud_state?('stopped')
+    end
+
+    def launchable?
+      not created?
+    end
+
+    def sshable?
+      in_chef?
+    end
+
+    def killable?
+      in_chef? || created?
+    end
+
+    def to_s
+      super[0..-3] + " chef: #{in_chef? && chef_node.name} fog: #{in_cloud? && fog_server.id}}>"
+    end
+
+    #
+    # Attributes
+    #
+
+    def tag key, value=nil
+      if value then @tags[key] = value ; end
+      @tags[key]
+    end
+
+    #
+    # Resolve:
+    #
+    def resolve!
+      reverse_merge!(facet)
+      reverse_merge!(cluster)
+      @settings[:run_list] = combined_run_list
+      #
+      cloud.reverse_merge!(facet.cloud)
+      cloud.reverse_merge!(cluster.cloud)
+      #
+      cloud.user_data({
+          :chef_server            => Chef::Config.chef_server_url,
+          :validation_client_name => Chef::Config.validation_client_name,
+          #
+          :node_name              => fullname,
+          :cluster_name           => cluster_name,
+          :facet_name             => facet_name,
+          :facet_index            => facet_index,
+          #
+          :run_list               => run_list,
+        })
+      #
+      if client_key.body then cloud.user_data({ :client_key     => client_key.body, })
+      else                    cloud.user_data({ :validation_key => cloud.validation_key }) ; end
+      cloud.keypair(cluster_name) if cloud.keypair.nil?
+      #
+      self
+    end
+
+    #
+    # Assembles the combined runlist.
+    #
+    # * run_list :first  items -- cluster then facet then server
+    # * run_list :normal items -- cluster then facet then server
+    # * own roles: cluster_role then facet_role
+    # * run_list :last   items -- cluster then facet then server
+    #
+    #    Ironfan.cluster(:my_cluster) do
+    #      role('f',  :last)
+    #      role('c')
+    #      facet(:my_facet) do
+    #        role('d')
+    #        role('e')
+    #        role('b', :first)
+    #        role('h',  :last)
+    #      end
+    #      role('a', :first)
+    #      role('g', :last)
+    #    end
+    #
+    # produces
+    #    cluster list  [a] [c]  [cluster_role] [fg]
+    #    facet list    [b] [de] [facet_role]   [h]
+    #
+    # yielding run_list
+    #     ['a', 'b', 'c', 'd', 'e', 'cr', 'fr', 'f', 'g', 'h']
+    #
+    # Avoid duplicate conflicting declarations. If you say define things more
+    # than once, the *earliest encountered* one wins, even if it is elsewhere
+    # marked :last.
+    #
+    def combined_run_list
+      cg = @cluster.run_list_groups
+      fg = @facet.run_list_groups
+      sg = self.run_list_groups
+      [ cg[:first],  fg[:first],  sg[:first],
+        cg[:normal], fg[:normal], sg[:normal],
+        cg[:own],    fg[:own],
+        cg[:last],   fg[:last],   sg[:last], ].flatten.compact.uniq
+    end
+
+    #
+    # This prepares a composited view of the volumes -- it shows the cluster
+    # definition overlaid by the facet definition overlaid by the server
+    # definition.
+    #
+    # This method *does* auto-vivify an empty volume declaration on the server,
+    # but doesn't modify it.
+    #
+    # This code is pretty smelly, but so is the resolve! behavior. advice welcome.
+    #
+    def composite_volumes
+      vols = {}
+      facet.volumes.each do |vol_name, vol|
+        self.volumes[vol_name] ||= Ironfan::Volume.new(:parent => self, :name => vol_name)
+        vols[vol_name]         ||= self.volumes[vol_name].dup
+        vols[vol_name].reverse_merge!(vol)
+      end
+      cluster.volumes.each do |vol_name, vol|
+        self.volumes[vol_name] ||= Ironfan::Volume.new(:parent => self, :name => vol_name)
+        vols[vol_name]         ||= self.volumes[vol_name].dup
+        vols[vol_name].reverse_merge!(vol)
+      end
+      vols.each{|vol_name, vol| vol.availability_zone self.default_availability_zone }
+      vols
+    end
+
+    # FIXME -- this will break on some edge case wehre a bogus node is
+    # discovered after everything is resolve!d
+    def default_availability_zone
+      cloud.default_availability_zone
+    end
+
+    #
+    # retrieval
+    #
+    def self.get(cluster_name, facet_name, facet_index)
+      cluster = Ironfan.cluster(cluster_name)
+      had_facet = cluster.has_facet?(facet_name)
+      facet = cluster.facet(facet_name)
+      facet.bogosity true unless had_facet
+      had_server = facet.has_server?( facet_index )
+      server = facet.server(facet_index)
+      server.bogosity :not_defined_in_facet unless had_server
+      return server
+    end
+
+    def self.all
+      @@all
+    end
+
+    #
+    # Actions!
+    #
+
+    def sync_to_cloud
+      step "Syncing to cloud"
+      attach_volumes
+      create_tags
+      associate_public_ip
+    end
+
+    def sync_to_chef
+      step "Syncing to chef server"
+      sync_chef_node
+      true
+    end
+
+    # FIXME: a lot of AWS logic in here. This probably lives in the facet.cloud
+    # but for the one or two things that come from the facet
+    def create_server
+      return nil if created? # only create a server if it does not already exist
+      fog_create_server
+    end
+
+    def create_tags
+      return unless created?
+      step("  labeling servers and volumes")
+      fog_create_tags(fog_server, self.fullname, tags)
+      composite_volumes.each do |vol_name, vol|
+        if vol.fog_volume
+          fog_create_tags(vol.fog_volume, vol.desc,
+            { "server" => self.fullname, "name" => "#{name}-#{vol.name}", "device" => vol.device, "mount_point" => vol.mount_point, "cluster" => cluster_name, "facet"   => facet_name, "index"   => facet_index, })
+        end
+      end
+    end
+
+    def block_device_mapping
+      composite_volumes.values.map(&:block_device_mapping).compact
+    end
+
+    # ugh. non-dry below.
+
+    def announce_as_started
+      return unless chef_node
+      announce_state('start')
+      chef_node.save
+    end
+
+    def announce_as_stopped
+      return unless chef_node
+      announce_state('stop')
+      chef_node.save
+    end
+
+  end
+end