ironfan 3.2.2 → 4.0.0

data/lib/ironfan/fog_layer.rb

@@ -1,196 +0,0 @@
-module Ironfan
-  #
-  # Ironfan::Server methods that handle Fog action
-  #
-  Server.class_eval do
-
-    def fog_create_server
-      step(" creating cloud server", :green)
-      lint_fog
-      launch_desc = fog_launch_description
-      Chef::Log.debug(JSON.pretty_generate(launch_desc))
-      safely do
-        @fog_server = Ironfan.fog_connection.servers.create(launch_desc)
-      end
-    end
-
-    def lint_fog
-      unless cloud.image_id then raise "No image ID found: nothing in Chef::Config[:ec2_image_info] for AZ #{self.default_availability_zone} flavor #{cloud.flavor} backing #{cloud.backing} image name #{cloud.image_name}, and cloud.image_id was not set directly. See https://github.com/infochimps-labs/ironfan/wiki/machine-image-(AMI)-lookup-by-name - #{cloud.list_images}" end
-      unless cloud.image_id then cloud.list_flavors ; raise "No machine flavor found" ; end
-    end
-
-    def fog_launch_description
-      user_data_hsh =
-        if client_key.body then cloud.user_data.merge({ :client_key     => client_key.body })
-        else                    cloud.user_data.merge({ :validation_key => cloud.validation_key }) ; end
-      #
-      description = {
-        :image_id             => cloud.image_id,
-        :flavor_id            => cloud.flavor,
-        :vpc_id               => cloud.vpc,
-        :subnet_id            => cloud.subnet,
-        :groups               => cloud.security_groups.keys,
-        :key_name             => cloud.keypair.to_s,
-        # Fog does not actually create tags when it creates a server.
-        :tags                 => {
-          :name               => self.fullname,
-          :cluster            => cluster_name,
-          :facet              => facet_name,
-          :index              => facet_index, },
-        :user_data            => JSON.pretty_generate(user_data_hsh),
-        :block_device_mapping => block_device_mapping,
-        :availability_zone    => self.default_availability_zone,
-        :monitoring           => cloud.monitoring,
-        # permanence is applied during sync
-      }
-      if needs_placement_group?
-        ui.warn "1.3.1 and earlier versions of Fog don't correctly support placement groups, so your nodes will land willy-nilly. We're working on a fix"
-        description[:placement] = { 'groupName' => cloud.placement_group.to_s }
-      end
-      description
-    end
-
-    #
-    # Takes key-value pairs and idempotently sets those tags on the cloud machine
-    #
-    def fog_create_tags(fog_obj, desc, tags)
-      tags['Name'] ||= tags['name'] if tags.has_key?('name')
-      tags_to_create = tags.reject{|key, val| fog_obj.tags[key] == val.to_s }
-      return if tags_to_create.empty?
-      step("  tagging #{desc} with #{tags_to_create.inspect}", :green)
-      tags_to_create.each do |key, value|
-        Chef::Log.debug( "tagging #{desc} with #{key} = #{value}" )
-        safely do
-          Ironfan.fog_connection.tags.create({
-            :key => key, :value => value.to_s, :resource_id => fog_obj.id })
-        end
-      end
-    end
-
-    def fog_address
-      address_str = self.cloud.public_ip or return
-      Ironfan.fog_addresses[address_str]
-    end
-
-    def discover_volumes!
-      result = self.class.fields[:volumes].type.new
-      volumes.each_pair do |vol_name, definition|
-        next if definition.fog_volume
-        next if Ironfan.chef_config[:cloud] == false
-        vol = definition.dup
-        vol.fog_volume = Ironfan.fog_volumes.find do |fv|
-          ( # matches the explicit volume id
-            (vol.volume_id && (fv.id == vol.volume_id)    ) ||
-            # OR this server's machine exists, and this volume is attached to
-            # it, and in the right place
-            ( fog_server && fv.server_id && vol.device  &&
-              (fv.server_id   == fog_server.id)         &&
-              (fv.device.to_s == vol.device.to_s)         ) ||
-            # OR this volume is tagged as belonging to this machine
-            ( fv.tags.present?                         &&
-              (fv.tags['server'] == self.fullname)     &&
-              (fv.tags['device'] == vol.device.to_s) )
-            )
-        end
-        next unless vol.fog_volume
-        vol.volume_id(vol.fog_volume.id)                        unless vol.volume_id.present?
-        vol.availability_zone(vol.fog_volume.availability_zone) unless vol.availability_zone.present?
-        check_server_id_pairing(vol.fog_volume, vol.desc)
-        result[vol.name] = vol
-      end
-      write_attribute(:volumes,result)
-    end
-
-    def attach_volumes
-      return unless in_cloud?
-      discover_volumes!
-      return if volumes.empty?
-      step("  attaching volumes")
-      volumes.each_pair do |vol_name, vol|
-        next if vol.volume_id.blank? || (vol.attachable != :ebs)
-        if (not vol.in_cloud?) then  Chef::Log.debug("Volume not found: #{vol.desc}") ; next ; end
-        if (vol.has_server?)   then check_server_id_pairing(vol.fog_volume, vol.desc) ; next ; end
-        step("  - attaching #{vol.desc} -- #{vol.inspect}", :blue)
-        safely do
-          vol.fog_volume.device = vol.device
-          vol.fog_volume.server = fog_server
-        end
-      end
-    end
-
-    def ensure_placement_group
-      return unless needs_placement_group?
-      pg_name = cloud.placement_group.to_s
-      desc = "placement group #{pg_name} for #{self.fullname} (vs #{Ironfan.placement_groups.inspect}"
-      return if Ironfan.placement_groups.include?(pg_name)
-      safely do
-        step("  creating #{desc}", :blue)
-        unless_dry_run{ Ironfan.fog_connection.create_placement_group(pg_name, 'cluster') }
-        Ironfan.placement_groups[pg_name] = { 'groupName' => pg_name, 'strategy' => 'cluster' }
-      end
-      pg_name
-    end
-
-    def needs_placement_group?
-      cloud.flavor_info[:placement_groupable]
-    end
-
-    def associate_public_ip
-      address = self.cloud.public_ip
-      return unless self.in_cloud? && address
-      desc = "elastic ip #{address} for #{self.fullname}"
-      if (fog_address && fog_address.server_id) then check_server_id_pairing(fog_address, desc) ; return ; end
-      safely do
-        step("  assigning #{desc}", :blue)
-        Ironfan.fog_connection.associate_address(self.fog_server.id, address)
-      end
-    end
-
-    def check_server_id_pairing thing, desc
-      return unless thing && thing.server_id && self.in_cloud?
-      type_of_thing = thing.class.to_s.gsub(/.*::/,"")
-      if thing.server_id != self.fog_server.id
-        ui.warn "#{type_of_thing} mismatch: #{desc} is on #{thing.server_id} not #{self.fog_server.id}: #{thing.inspect.gsub(/\s+/m,' ')}"
-        false
-      else
-        Chef::Log.debug("#{type_of_thing} paired: #{desc}")
-        true
-      end
-    end
-
-    def set_instance_attributes
-      return unless self.in_cloud? && (not self.cloud.permanent.nil?)
-      desc = "termination flag #{permanent?} for #{self.fullname}"
-      # the EC2 API does not surface disable_api_termination as a value, so we
-      # have to set it every time.
-      safely do
-        step("  setting #{desc}", :blue)
-        unless_dry_run do
-          Ironfan.fog_connection.modify_instance_attribute(self.fog_server.id, {
-              'DisableApiTermination.Value' => permanent?, })
-        end
-        true
-      end
-    end
-
-  end
-
-  class ServerSlice
-    def sync_keypairs
-      step("ensuring keypairs exist")
-      keypairs  = servers.map{|svr| [svr.cluster.cloud.keypair, svr.cloud.keypair] }.flatten.map(&:to_s).reject(&:blank?).uniq
-      keypairs  = keypairs - Ironfan.fog_keypairs.keys
-      keypairs.each do |keypair_name|
-        keypair_obj = Ironfan::Ec2Keypair.create!(keypair_name)
-        Ironfan.fog_keypairs[keypair_name] = keypair_obj
-      end
-    end
-
-    # Create security groups, their dependencies, and synchronize their permissions
-    def sync_security_groups
-      step("ensuring security groups exist and are correct")
-      security_groups.each{|name,group| group.run }
-    end
-
-  end
-end

data/lib/ironfan/private_key.rb

@@ -1,130 +0,0 @@
-require 'fileutils'
-
-module Ironfan
-  #
-  # A private key -- chef client key, ssh key, etc.
-  #
-  # The key is a pro
-  class PrivateKey < Ironfan::DslBuilder
-    attr_reader   :name
-    attr_reader   :proxy
-    attr_reader   :on_update
-
-    #
-    # PrivateKey.new('bob')
-    #
-    # @yield      a block, executed in caller's context, when the body is updated
-    # @yieldparam the updated body
-    def initialize(name, proxy=nil, &on_update)
-      @name      = name
-      @proxy     = proxy
-      @on_update = on_update
-    end
-
-    def filename
-      File.join(key_dir, "#{name}.pem")
-    end
-
-    def save
-      return unless @body
-      if Ironfan.chef_config[:dry_run]
-        Chef::Log.debug("    key #{name} - dry run, not writing out key")
-        return
-      end
-      ui.info( "    key #{name} - writing to #{filename}" )
-      FileUtils.mkdir_p(File.dirname(filename))
-      File.open(filename, "w", 0600){|f| f.print( @body ) }
-    end
-
-    def load
-      return unless File.exists?(filename)
-      self.body = File.read(filename).chomp
-    end
-
-    def body=(content)
-      @body = content
-      on_update.call(content) if on_update
-      content
-    end
-
-    def self.create!(name, *args, &block)
-      obj = self.new(name, *args, &block)
-      obj.create_proxy!
-      obj
-    end
-
-    def to_s
-      [super[0..-2], @name, @proxy, @body.to_s[32..64], '...', @body.to_s[-60..-30]].join(" ").gsub(/[\r\n\t]+/,'') + '>'
-    end
-  end
-
-  class ChefClientKey < PrivateKey
-    def body
-      return @body if @body
-      if proxy && proxy.private_key && (not proxy.private_key.empty?)
-        @body = proxy.private_key
-      else
-        load
-      end
-      @body
-    end
-
-    def key_dir
-      Chef::Config.client_key_dir || "/tmp/#{ENV['USER']}-client_keys"
-    end
-  end
-
-  class DataBagKey < PrivateKey
-    def body
-      return @body if @body
-      @body
-    end
-
-    def random_token
-      require "digest/sha2"
-      digest = Digest::SHA512.hexdigest(  Time.now.to_s + (1..10).collect{ rand.to_s }.join  )
-      5.times{ digest = Digest::SHA512.hexdigest(digest) }
-      digest
-    end
-
-    def key_dir
-      return Chef::Config.data_bag_key_dir if Chef::Config.data_bag_key_dir
-      dir = "#{ENV['HOME']}/.chef/credentials/data_bag_keys"
-      warn "Please set 'data_bag_key_dir' in your knife.rb. Will use #{dir} as a default"
-      dir
-    end
-  end
-
-  class Ec2Keypair < PrivateKey
-    def body
-      return @body if @body
-      if proxy && proxy.private_key && (not proxy.private_key.empty?)
-        @body = proxy.private_key
-      else
-        load
-      end
-      @body
-    end
-
-    def create_proxy!
-      safely do
-        step("    key #{name} - creating", :green)
-        @proxy = Ironfan.fog_connection.key_pairs.create(:name => name.to_s)
-      end
-      Ironfan.fog_keypairs[name] = proxy
-      self.body = proxy.private_key
-      save
-    end
-
-    def key_dir
-      if Chef::Config.ec2_key_dir
-        return Chef::Config.ec2_key_dir
-      else
-        dir = "#{ENV['HOME']}/.chef/credentials/ec2_keys"
-        warn "Please set 'ec2_key_dir' in your knife.rb. Will use #{dir} as a default"
-        dir
-      end
-    end
-  end
-
-end

data/lib/ironfan/role_implications.rb

@@ -1,58 +0,0 @@
-module Ironfan
-  ComputeBuilder.class_eval do
-
-    # organization-wide security group
-    role_implication "systemwide" do
-      self.cloud.security_group "systemwide" do
-      end
-    end
-
-    # NFS server allows access from nfs_clients
-    role_implication "nfs_server" do
-      self.cloud.security_group "nfs_server" do
-        authorize_group "nfs_client"
-      end
-    end
-
-    role_implication "nfs_client" do
-      self.cloud.security_group "nfs_client"
-    end
-
-    # Opens port 22 to the world
-    role_implication "ssh" do
-      self.cloud.security_group 'ssh' do
-        authorize_port_range 22..22
-      end
-    end
-
-    # Open the Chef server API port (4000) and the webui (4040)
-    role_implication "chef_server" do
-      self.cloud.security_group "chef_server" do
-        authorize_port_range 4000..4000  # chef-server-api
-        authorize_port_range 4040..4040  # chef-server-webui
-      end
-    end
-
-    # web server? add the group "web_server" to open the web holes
-    role_implication "web_server" do
-      self.cloud.security_group("#{cluster_name}-web_server") do
-        authorize_port_range  80..80
-        authorize_port_range 443..443
-      end
-    end
-
-    # if you're a redis server, open the port and authorize redis clients in your group to talk to you
-    role_implication("redis_server") do
-      cluster_name = self.cluster_name # hack: put cluster_name is in scope
-      self.cloud.security_group("#{cluster_name}-redis_server") do
-        authorize_group("#{cluster_name}-redis_client")
-      end
-    end
-
-    # redis_clients gain rights to the redis_server
-    role_implication("redis_client") do
-      self.cloud.security_group("#{cluster_name}-redis_client")
-    end
-
-  end
-end

data/lib/ironfan/security_group.rb

@@ -1,133 +0,0 @@
-module Ironfan
-  module CloudDsl
-    class SecurityGroup < Ironfan::DslBuilder
-      magic :name, String
-      magic :description, String
-      magic :group_authorizations, Array
-      magic :group_authorized_by, Array
-      magic :range_authorizations, Array
-
-      def initialize params
-        name                    params[:name].to_s
-        description             "ironfan generated group #{name}"
-        group_authorizations    []
-        group_authorized_by     []
-        range_authorizations    []
-      end
-
-      def to_key
-        name
-      end
-
-      @@all = nil
-      def all
-        self.class.all
-      end
-      def self.all
-        return @@all if @@all
-        get_all
-      end
-      def self.get_all
-        groups_list = Ironfan.fog_connection.security_groups.all
-        @@all = groups_list.inject(Mash.new) do |hsh, fog_group|
-          # AWS security_groups are strangely case sensitive, allowing upper-case but colliding regardless
-          #  of the case. This forces all names to lowercase, and matches against that below.
-          #  See https://github.com/infochimps-labs/ironfan/pull/86 for more details.
-          hsh[fog_group.name.downcase] = fog_group ; hsh
-        end
-      end
-
-      def get
-        all[name] || Ironfan.fog_connection.security_groups.get(name)
-      end
-
-      def self.get_or_create(group_name, description)
-        group_name = group_name.to_s.downcase
-        # FIXME: the '|| Ironfan.fog' part is probably unnecessary
-        fog_group = all[group_name] || Ironfan.fog_connection.security_groups.get(group_name)
-        unless fog_group
-          self.step(group_name, "creating (#{description})", :green)
-          fog_group = all[group_name] = Ironfan.fog_connection.security_groups.new(:name => group_name, :description => description, :connection => Ironfan.fog_connection)
-          fog_group.save
-        end
-        fog_group
-      end
-
-      def authorize_group(group_name, owner_id=nil)
-        group_authorizations << [group_name.to_s, owner_id]
-      end
-
-      def authorized_by_group(other_name)
-        group_authorized_by << other_name.to_s
-      end
-
-      def authorize_port_range(range, cidr_ip = '0.0.0.0/0', ip_protocol = 'tcp')
-        range = (range .. range) if range.is_a?(Integer)
-        range_authorizations << [range, cidr_ip, ip_protocol]
-      end
-# 
-#       def group_permission_already_set?(fog_group, other_name, authed_owner)
-#         return false if fog_group.ip_permissions.nil?
-#         fog_group.ip_permissions.any? do |existing_permission|
-#           existing_permission["groups"].include?({"userId" => authed_owner, "groupName" => other_name}) &&
-#             existing_permission["fromPort"] == 1 &&
-#             existing_permission["toPort"]   == 65535
-#         end
-#       end
-# 
-#       def range_permission_already_set?(fog_group, range, cidr_ip, ip_protocol)
-#         return false if fog_group.ip_permissions.nil?
-#         fog_group.ip_permissions.include?(
-#           { "groups"=>[], "ipRanges"=>[{"cidrIp"=>cidr_ip}],
-#             "ipProtocol"=>ip_protocol, "fromPort"=>range.first, "toPort"=>range.last})
-#       end
-# 
-      # FIXME: so if you're saying to yourself, "self, this is some soupy gooey
-      # code right here" then you and your self are correct. Much of this is to
-      # work around old limitations in the EC2 api. You can now treat range and
-      # group permissions the same, and we should.
-
-      def run
-        fog_group = self.class.get_or_create(name, description)
-        @group_authorizations.uniq.each do |other_name, authed_owner|
-          authed_owner ||= self.owner_id
-          next if group_permission_already_set?(fog_group, other_name, authed_owner)
-          step("authorizing access from all machines in #{other_name} to #{name}", :blue)
-          self.class.get_or_create(other_name, "Authorized to access #{name}")
-          begin  fog_group.authorize_group_and_owner(other_name, authed_owner)
-          rescue StandardError => err ; handle_security_group_error(err) ; end
-        end
-        @group_authorized_by.uniq.each do |other_name|
-          authed_owner = self.owner_id
-          other_group = self.class.get_or_create(other_name, "Authorized for access by #{self.name}")
-          next if group_permission_already_set?(other_group, self.name, authed_owner)
-          step("authorizing access to all machines in #{other_name} from #{name}", :blue)
-          begin  other_group.authorize_group_and_owner(self.name, authed_owner)
-          rescue StandardError => err ; handle_security_group_error(err) ; end
-        end
-        @range_authorizations.uniq.each do |range, cidr_ip, ip_protocol|
-          next if range_permission_already_set?(fog_group, range, cidr_ip, ip_protocol)
-          step("opening #{ip_protocol} ports #{range} to #{cidr_ip}", :blue)
-          begin  fog_group.authorize_port_range(range, { :cidr_ip => cidr_ip, :ip_protocol => ip_protocol })
-          rescue StandardError => err ; handle_security_group_error(err) ; end
-        end
-      end
-# 
-#       def handle_security_group_error(err)
-#         if (/has already been authorized/ =~ err.to_s)
-#           Chef::Log.debug err
-#         else
-#           ui.warn(err)
-#         end
-#       end
-# 
-      def self.step(group_name, desc, *style)
-        ui.info("  group #{"%-15s" % (group_name+":")}\t#{ui.color(desc.to_s, *style)}")
-      end
-      def step(desc, *style)
-        self.class.step(self.name, desc, *style)
-      end
-    end
-
-  end
-end