ironfan 3.1.0.rc1

data/lib/ironfan/dsl_object.rb

@@ -0,0 +1,124 @@
+Mash.class_eval do
+  def reverse_merge!(other_hash)
+    # stupid mash doesn't take a block arg, which breaks the implementation of
+    # reverse_merge!
+    other_hash.each_pair do |key, value|
+      key = convert_key(key)
+      regular_writer(key, convert_value(value)) unless has_key?(key)
+    end
+    self
+  end
+  def to_mash
+    self.dup
+  end unless method_defined?(:to_mash)
+end
+
+Hash.class_eval do
+  def to_mash
+    Mash.new(self)
+  end unless method_defined?(:to_mash)
+end
+
+module Ironfan
+  #
+  # Provides magic methods, defined with has_keys
+  #
+  # @example
+  #   class Mom < Ironfan::DslObject
+  #     has_keys(:college, :combat_boots, :fat, :so_fat)
+  #   end
+  #
+  #   class Person
+  #     def momma &block
+  #       @momma ||= Mom.new
+  #       @momma.configure(&block) if block
+  #     end
+  #   end
+  #
+  #   yo = Person.new
+  #   yo.mamma.combat_boots :wears
+  #   yo.momma do
+  #     fat    true
+  #     so_fat 'When she sits around the house, she sits *AROUND* the house'
+  #   end
+  #
+  class DslObject
+    class_attribute :keys
+    self.keys = []
+
+    def initialize(attrs={}, &block)
+      @settings = Mash.new
+      configure(attrs, &block)
+    end
+
+    #
+    # Defines DSL attributes
+    #
+    # @params [Array(String)] key_names DSL attribute names
+    #
+    # @example
+    #   class Mom < Ironfan::DslObject
+    #     has_keys(:fat, :so_fat)
+    #   end
+    #   yer_mom = Mom.new
+    #   yer_mom.fat :quite
+    #
+    def self.has_keys(*key_names)
+      key_names.map!(&:to_sym)
+      self.keys += key_names
+      self.keys.uniq!
+      key_names.each do |key|
+        next if method_defined?(key)
+        define_method(key){|*args| set(key, *args) }
+      end
+    end
+
+    #
+    # Sets the DSL attribute, unless the given value is nil.
+    #
+    def set(key, val=nil)
+      @settings[key.to_s] = val unless val.nil?
+      @settings[key.to_s]
+    end
+
+    def to_hash
+      @settings.to_hash
+    end
+
+    def to_mash
+      @settings.dup
+    end
+
+    def to_s
+      "<#{self.class} #{to_hash.inspect}>"
+    end
+
+    def reverse_merge!(hsh)
+      @settings.reverse_merge!(hsh.to_hash)
+    end
+
+    def configure(hsh={}, &block)
+      @settings.merge!(hsh.to_hash)
+      instance_eval(&block) if block
+      self
+    end
+
+    # delegate to the knife ui presenter
+    def ui()      Ironfan.ui ; end
+    # delegate to the knife ui presenter
+    def self.ui() Ironfan.ui ; end
+
+    def step(desc, *style)
+      ui.info("  #{"%-15s" % (name.to_s+":")}\t#{ui.color(desc.to_s, *style)}")
+    end
+
+    # helper method for bombing out of a script
+    def die(*args) Ironfan.die(*args) ; end
+
+    # helper method for turning exceptions into warnings
+    def safely(*args, &block) Ironfan.safely(*args, &block) ; end
+
+    # helper method for debugging only
+    def dump(*args) args.each{|arg| Chef::Log.debug( arg.inspect ) } end
+  end
+end

data/lib/ironfan/facet.rb

@@ -0,0 +1,144 @@
+module Ironfan
+  class Facet < Ironfan::ComputeBuilder
+    attr_reader :cluster
+    has_keys  :instances
+
+    def initialize cluster, facet_name, attrs={}
+      super(facet_name.to_sym, attrs)
+      @cluster    = cluster
+      @servers    = Mash.new
+      @chef_roles = []
+      @settings[:instances] ||= 1
+      create_facet_role
+      create_facet_security_group unless attrs[:no_security_group]
+    end
+
+    def cluster_name
+      cluster.name
+    end
+
+    def facet_name
+      name
+    end
+
+    # The auto-generated role for this facet.
+    # Instance-evals the given block in the context of that role,
+    #
+    # @example
+    #   facet_role do
+    #     override_attributes({
+    #       :time_machine => { :transition_speed => 88 },
+    #     })
+    #   end
+    #
+    # @return [Chef::Role] The auto-generated role for this facet.
+    def facet_role(&block)
+      @facet_role.instance_eval( &block ) if block_given?
+      @facet_role
+    end
+
+    def assign_volume_ids(volume_name, *volume_ids)
+      volume_ids.flatten.zip(servers).each do |volume_id, server|
+        server.volume(volume_name){ volume_id(volume_id) } if server
+      end
+    end
+
+    #
+    # Retrieve or define the given server
+    #
+    # @param [Integer] idx  -- the index of the desired server
+    # @param [Hash] attrs -- attributes to configure on the object
+    # @yield a block to execute in the context of the object
+    #
+    # @return [Ironfan::Facet]
+    #
+    def server(idx, attrs={}, &block)
+      idx = idx.to_i
+      @servers[idx] ||= Ironfan::Server.new(self, idx)
+      @servers[idx].configure(attrs, &block)
+      @servers[idx]
+    end
+
+    # if the server has been added to this facet or is in range
+    def has_server? idx
+      (idx.to_i < instances) || @servers.include?(idx.to_i)
+    end
+
+    #
+    # Slicing
+    #
+
+    # All servers in this facet
+    #
+    # @return [Ironfan::ServerSlice] slice containing all servers
+    def servers
+      slice(indexes)
+    end
+
+    #
+    # A slice of servers from this facet, in index order
+    #
+    # If +slice_indexes+ is nil, returns all servers.
+    # Otherwise, takes slice (given by +*args+) from the requested facet.
+    #
+    # @param [Array, String] slice_indexes -- servers in that facet (or nil for all in facet).
+    #
+    # @return [Ironfan::ServerSlice] the requested slice
+    def slice(slice_indexes=nil)
+      slice_indexes = self.indexes if slice_indexes.blank?
+      slice_indexes = indexes_from_intervals(slice_indexes) if slice_indexes.is_a?(String)
+      svrs = Array(slice_indexes).map(&:to_i).sort!.select{|idx| has_server?(idx) }.map{|idx| server(idx) }
+      Ironfan::ServerSlice.new(self.cluster, svrs)
+    end
+
+    # all valid server indexes
+    def valid_indexes
+      (0 ... instances).to_a # note the '...'
+    end
+
+    # indexes in the 0...instances range plus bogus ones that showed up
+    # (probably from chef or fog)
+    def indexes
+      [@servers.keys, valid_indexes].flatten.compact.uniq.sort
+    end
+
+    #
+    # Resolve:
+    #
+    def resolve!
+      servers.each(&:resolve!)
+    end
+
+  protected
+
+    def create_facet_security_group
+      cloud.security_group("#{cluster_name}-#{facet_name}")
+    end
+
+    # Creates a chef role named for the facet
+    def create_facet_role
+      @facet_role_name = "#{cluster_name}_#{facet_name}"
+      @facet_role      = new_chef_role(@facet_role_name, cluster, self)
+      role(@facet_role_name, :own)
+    end
+
+    #
+    # Given a string enumerating indexes to select returns a flat array of
+    # indexes. The indexes will be unique but in an arbitrary order.
+    #
+    # @example
+    #   facet = Ironfan::Facet.new('foo', 'bar')
+    #   facet.indexes_from_intervals('1,2-3,8-9,7') # [1, 2, 3, 8, 9, 7]
+    #   facet.indexes_from_intervals('1,3-5,4,7')   # [1, 3, 4, 5, 7]
+    #
+    def indexes_from_intervals intervals
+      intervals.split(",").map do |term|
+        if    term =~ /^(\d+)-(\d+)$/ then ($1.to_i .. $2.to_i).to_a
+        elsif term =~ /^(\d+)$/       then  $1.to_i
+        else  ui.warn("Bad interval: #{term}") ; nil
+        end
+      end.flatten.compact.uniq
+    end
+
+  end
+end

data/lib/ironfan/fog_layer.rb

@@ -0,0 +1,150 @@
+module Ironfan
+  #
+  # Ironfan::Server methods that handle Fog action
+  #
+  Server.class_eval do
+
+    def fog_create_server
+      step(" creating cloud server", :green)
+      lint_fog
+      launch_desc = fog_launch_description
+      Chef::Log.debug(JSON.pretty_generate(launch_desc))
+      safely do
+        @fog_server = Ironfan.fog_connection.servers.create(launch_desc)
+      end
+    end
+
+    def lint_fog
+      unless cloud.image_id then raise "No image ID found: nothing in Chef::Config[:ec2_image_info] for AZ #{self.default_availability_zone} flavor #{cloud.flavor} backing #{cloud.backing} image name #{cloud.image_name}, and cloud.image_id was not set directly. See https://github.com/infochimps-labs/ironfan/wiki/machine-image-(AMI)-lookup-by-name - #{cloud.list_images}" end
+      unless cloud.image_id then cloud.list_flavors ; raise "No machine flavor found" ; end
+    end
+
+    def fog_launch_description
+      {
+        :image_id          => cloud.image_id,
+        :flavor_id         => cloud.flavor,
+        #
+        :groups            => cloud.security_groups.keys,
+        :key_name          => cloud.keypair.to_s,
+        # Fog does not actually create tags when it creates a server.
+        :tags              => {
+          :cluster => cluster_name,
+          :facet   => facet_name,
+          :index   => facet_index, },
+        :user_data         => JSON.pretty_generate(cloud.user_data),
+        :block_device_mapping    => block_device_mapping,
+        # :disable_api_termination => cloud.permanent,
+        # :instance_initiated_shutdown_behavior => instance_initiated_shutdown_behavior,
+        :availability_zone => self.default_availability_zone,
+        :monitoring => cloud.monitoring,
+      }
+    end
+
+    #
+    # Takes key-value pairs and idempotently sets those tags on the cloud machine
+    #
+    def fog_create_tags(fog_obj, desc, tags)
+      tags_to_create = tags.reject{|key, val| fog_obj.tags[key] == val.to_s }
+      return if tags_to_create.empty?
+      step("  tagging #{desc} with #{tags_to_create.inspect}", :green)
+      tags_to_create.each do |key, value|
+        Chef::Log.debug( "tagging #{desc} with #{key} = #{value}" )
+        safely do
+          Ironfan.fog_connection.tags.create({
+            :key => key, :value => value.to_s, :resource_id => fog_obj.id })
+        end
+      end
+    end
+
+    def fog_address
+      address_str = self.cloud.public_ip or return
+      Ironfan.fog_addresses[address_str]
+    end
+
+    def discover_volumes!
+      composite_volumes.each do |vol_name, vol|
+        my_vol = volumes[vol_name]
+        next if my_vol.fog_volume
+        next if Ironfan.chef_config[:cloud] == false
+        my_vol.fog_volume = Ironfan.fog_volumes.find do |fv|
+          ( # matches the explicit volume id
+            (vol.volume_id && (fv.id == vol.volume_id)    ) ||
+            # OR this server's machine exists, and this volume is attached to
+            # it, and in the right place
+            ( fog_server && fv.server_id && vol.device  &&
+              (fv.server_id   == fog_server.id)         &&
+              (fv.device.to_s == vol.device.to_s)         ) ||
+            # OR this volume is tagged as belonging to this machine
+            ( fv.tags.present?                         &&
+              (fv.tags['server'] == self.fullname)     &&
+              (fv.tags['device'] == vol.device.to_s) )
+            )
+        end
+        next unless my_vol.fog_volume
+        my_vol.volume_id(my_vol.fog_volume.id)                        unless my_vol.volume_id.present?
+        my_vol.availability_zone(my_vol.fog_volume.availability_zone) unless my_vol.availability_zone.present?
+        check_server_id_pairing(my_vol.fog_volume, my_vol.desc)
+      end
+    end
+
+    def attach_volumes
+      return unless in_cloud?
+      discover_volumes!
+      return if composite_volumes.empty?
+      step("  attaching volumes")
+      composite_volumes.each do |vol_name, vol|
+        next if vol.volume_id.blank? || (vol.attachable != :ebs)
+        if (not vol.in_cloud?) then  Chef::Log.debug("Volume not found: #{vol.desc}") ; next ; end
+        if (vol.has_server?)   then check_server_id_pairing(vol.fog_volume, vol.desc) ; next ; end
+        step("  - attaching #{vol.desc} -- #{vol.inspect}", :blue)
+        safely do
+          vol.fog_volume.device = vol.device
+          vol.fog_volume.server = fog_server
+        end
+      end
+    end
+
+    def associate_public_ip
+      address = self.cloud.public_ip
+      return unless self.in_cloud? && address
+      desc = "elastic ip #{address} for #{self.fullname}"
+      if (fog_address && fog_address.server_id) then check_server_id_pairing(fog_address, desc) ; return ; end
+      safely do
+        step("  assigning #{desc}", :blue)
+        Ironfan.fog_connection.associate_address(self.fog_server.id, address)
+      end
+    end
+
+    def check_server_id_pairing thing, desc
+      return unless thing && thing.server_id && self.in_cloud?
+      type_of_thing = thing.class.to_s.gsub(/.*::/,"")
+      if thing.server_id != self.fog_server.id
+        ui.warn "#{type_of_thing} mismatch: #{desc} is on #{thing.server_id} not #{self.fog_server.id}: #{thing.inspect.gsub(/\s+/m,' ')}"
+        false
+      else
+        Chef::Log.debug("#{type_of_thing} paired: #{desc}")
+        true
+      end
+    end
+
+  end
+
+  class ServerSlice
+    def sync_keypairs
+      step("ensuring keypairs exist")
+      keypairs  = servers.map{|svr| [svr.cluster.cloud.keypair, svr.cloud.keypair] }.flatten.map(&:to_s).reject(&:blank?).uniq
+      keypairs  = keypairs - Ironfan.fog_keypairs.keys
+      keypairs.each do |keypair_name|
+        keypair_obj = Ironfan::Ec2Keypair.create!(keypair_name)
+        Ironfan.fog_keypairs[keypair_name] = keypair_obj
+      end
+    end
+
+    # Create security groups, their dependencies, and synchronize their permissions
+    def sync_security_groups
+      step("ensuring security groups exist and are correct")
+      security_groups.each{|name,group| group.run }
+    end
+
+  end
+end

data/lib/ironfan/private_key.rb

@@ -0,0 +1,130 @@
+require 'fileutils'
+
+module Ironfan
+  #
+  # A private key -- chef client key, ssh key, etc.
+  #
+  # The key is a pro
+  class PrivateKey < Ironfan::DslObject
+    attr_reader   :name
+    attr_reader   :proxy
+    attr_reader   :on_update
+
+    #
+    # PrivateKey.new('bob')
+    #
+    # @yield      a block, executed in caller's context, when the body is updated
+    # @yieldparam the updated body
+    def initialize(name, proxy=nil, &on_update)
+      @name      = name
+      @proxy     = proxy
+      @on_update = on_update
+    end
+
+    def filename
+      File.join(key_dir, "#{name}.pem")
+    end
+
+    def save
+      return unless @body
+      if Ironfan.chef_config[:dry_run]
+        Chef::Log.debug("    key #{name} - dry run, not writing out key")
+        return
+      end
+      ui.info( "    key #{name} - writing to #{filename}" )
+      FileUtils.mkdir_p(File.dirname(filename))
+      File.open(filename, "w", 0600){|f| f.print( @body ) }
+    end
+
+    def load
+      return unless File.exists?(filename)
+      self.body = File.read(filename).chomp
+    end
+
+    def body=(content)
+      @body = content
+      on_update.call(content) if on_update
+      content
+    end
+
+    def self.create!(name, *args, &block)
+      obj = self.new(name, *args, &block)
+      obj.create_proxy!
+      obj
+    end
+
+    def to_s
+      [super[0..-2], @name, @proxy, @body.to_s[32..64], '...', @body.to_s[-60..-30]].join(" ").gsub(/[\r\n\t]+/,'') + '>'
+    end
+  end
+
+  class ChefClientKey < PrivateKey
+    def body
+      return @body if @body
+      if proxy && proxy.private_key && (not proxy.private_key.empty?)
+        @body = proxy.private_key
+      else
+        load
+      end
+      @body
+    end
+
+    def key_dir
+      Chef::Config.client_key_dir || '/tmp/client_keys'
+    end
+  end
+
+  class DataBagKey < PrivateKey
+    def body
+      return @body if @body
+      @body
+    end
+
+    def random_token
+      require "digest/sha2"
+      digest = Digest::SHA512.hexdigest(  Time.now.to_s + (1..10).collect{ rand.to_s }.join  )
+      5.times{ digest = Digest::SHA512.hexdigest(digest) }
+      digest
+    end
+
+    def key_dir
+      return Chef::Config.data_bag_key_dir if Chef::Config.data_bag_key_dir
+      dir = "#{ENV['HOME']}/.chef/data_bag_keys"
+      warn "Please set 'data_bag_key_dir' in your knife.rb. Will use #{dir} as a default"
+      dir
+    end
+  end
+
+  class Ec2Keypair < PrivateKey
+    def body
+      return @body if @body
+      if proxy && proxy.private_key && (not proxy.private_key.empty?)
+        @body = proxy.private_key
+      else
+        load
+      end
+      @body
+    end
+
+    def create_proxy!
+      safely do
+        step("    key #{name} - creating", :green)
+        @proxy = Ironfan.fog_connection.key_pairs.create(:name => name.to_s)
+      end
+      Ironfan.fog_keypairs[name] = proxy
+      self.body = proxy.private_key
+      save
+    end
+
+    def key_dir
+      if Chef::Config.ec2_key_dir
+        return Chef::Config.ec2_key_dir
+      else
+        dir = "#{ENV['HOME']}/.chef/ec2_keys"
+        warn "Please set 'ec2_key_dir' in your knife.rb. Will use #{dir} as a default"
+        dir
+      end
+    end
+  end
+
+end