ironfan 3.1.0.rc1

data/lib/ironfan/server_slice.rb

@@ -0,0 +1,260 @@
+module Ironfan
+  #
+  # A server group is a set of actual or implied servers.
+  #
+  # The idea is we want to be able to smoothly roll up settings
+  #
+  #
+  class ServerSlice < Ironfan::DslObject
+    attr_reader :name, :servers, :cluster
+
+    def initialize cluster, servers
+      super()
+      @name    = "#{cluster.name} slice"
+      @cluster = cluster
+      @servers = servers
+    end
+
+    #
+    # Enumerable
+    #
+    include Enumerable
+    def each(&block)
+      @servers.each(&block)
+    end
+    def length
+      @servers.length
+    end
+    def empty?
+      length == 0
+    end
+    [:select, :find_all, :reject, :detect, :find, :drop_while].each do |method|
+      define_method(method) do |*args, &block|
+        ServerSlice.new cluster, @servers.send(method, *args, &block)
+      end
+    end
+
+    # Return the collection of servers that are not yet 'created'
+    def uncreated_servers
+      select{|svr| not svr.created? }
+    end
+
+    def bogus_servers
+      select(&:bogus?)
+    end
+
+    #
+    # Info!
+    #
+
+    def chef_nodes
+      servers.map(&:chef_node).compact
+    end
+
+    def fog_servers
+      servers.map(&:fog_server).compact
+    end
+
+    def security_groups
+      sg = {}
+      servers.each{|svr| sg.merge!(svr.cloud.security_groups) }
+      sg
+    end
+
+    def facets
+      servers.map(&:facet)
+    end
+
+    def chef_roles
+      [ cluster.chef_roles, facets.map(&:chef_roles) ].flatten.compact.uniq
+    end
+
+    # hack -- take the ssh_identity_file from the first server.
+    def ssh_identity_file
+      return if servers.empty?
+      servers.first.cloud.ssh_identity_file
+    end
+
+    #
+    # Actions!
+    #
+
+    def start
+      delegate_to_fog_servers( :start  )
+      delegate_to_fog_servers( :reload  )
+    end
+
+    def stop
+      delegate_to_fog_servers( :stop )
+      delegate_to_fog_servers( :reload  )
+    end
+
+    def destroy
+      delegate_to_fog_servers( :destroy )
+      delegate_to_fog_servers( :reload  )
+    end
+
+    def reload
+      delegate_to_fog_servers( :reload  )
+    end
+
+    def create_servers
+      delegate_to_servers( :create_server )
+    end
+
+    def delete_chef
+      delegate_to_servers( :delete_chef, true )
+    end
+
+    def sync_to_cloud
+      sync_keypairs
+      sync_security_groups
+      delegate_to_servers( :sync_to_cloud )
+    end
+
+    def sync_to_chef
+      sync_roles
+      delegate_to_servers( :sync_to_chef )
+    end
+
+    #
+    # Display!
+    #
+
+    # FIXME: this is a jumble. we need to pass it in some other way.
+
+    MINIMAL_HEADINGS  = ["Name", "Chef?", "State", "InstanceID", "Public IP", "Private IP", "Created At"].to_set.freeze
+    DEFAULT_HEADINGS  = (MINIMAL_HEADINGS + ['Flavor', 'AZ', 'Env']).freeze
+    EXPANDED_HEADINGS = DEFAULT_HEADINGS + ['Image', 'Volumes', 'Elastic IP', 'SSH Key']
+
+    MACHINE_STATE_COLORS  = {
+      'running'       => :green,
+      'pending'       => :yellow,
+      'stopping'      => :magenta,
+      'shutting-down' => :magenta,
+      'stopped'       => :cyan,
+      'terminated'    => :blue,
+      'not running'   => :blue,
+    }
+
+    #
+    # This is a generic display routine for cluster-like sets of nodes. If you
+    # call it with no args, you get the basic table that knife cluster show
+    # draws.  If you give it an array of strings, you can override the order and
+    # headings displayed. If you also give it a block you can add your own logic
+    # for generating content. The block is given a Ironfan::Server instance
+    # for each item in the collection and should return a hash of Name,Value
+    # pairs to merge into the minimal fields.
+    #
+    def display hh = :default
+      headings =
+        case hh
+        when :minimal  then MINIMAL_HEADINGS
+        when :default  then DEFAULT_HEADINGS
+        when :expanded then EXPANDED_HEADINGS
+        else hh.to_set end
+      headings += ["Bogus"] if servers.any?(&:bogus?)
+      # probably not necessary any more
+      # servers = servers.sort{ |a,b| (a.facet_name <=> b.facet_name) *9 + (a.facet_index.to_i <=> b.facet_index.to_i)*3 + (a.facet_index <=> b.facet_index) }
+      defined_data = servers.map do |svr|
+        hsh = {
+          "Name"   => svr.fullname,
+          "Facet"  => svr.facet_name,
+          "Index"  => svr.facet_index,
+          "Chef?"  => (svr.chef_node? ? "yes" : "[red]no[reset]"),
+          "Bogus"  => (svr.bogus? ? "[red]#{svr.bogosity}[reset]" : ''),
+          "Env"    => svr.environment,
+        }
+        # if (cs = svr.chef_server)
+        #   hsh.merge!(
+        #     "Env"    => cs.environment,
+        #     )
+        # end
+        if (fs = svr.fog_server)
+          hsh.merge!(
+              "InstanceID" => (fs.id && fs.id.length > 0) ? fs.id : "???",
+              "Flavor"     => fs.flavor_id,
+              "Image"      => fs.image_id,
+              "AZ"         => fs.availability_zone,
+              "SSH Key"    => fs.key_name,
+              "State"      => "[#{MACHINE_STATE_COLORS[fs.state] || 'white'}]#{fs.state}[reset]",
+              "Public IP"  => fs.public_ip_address,
+              "Private IP" => fs.private_ip_address,
+              "Created At" => fs.created_at.strftime("%Y%m%d-%H%M%S")
+            )
+        else
+          hsh["State"] = "not running"
+        end
+        hsh['Volumes'] = []
+        svr.composite_volumes.each do |name, vol|
+          if    vol.ephemeral_device? then next
+          elsif vol.volume_id         then hsh['Volumes'] << vol.volume_id
+          elsif vol.create_at_launch? then hsh['Volumes'] << vol.snapshot_id
+          end
+        end
+        hsh['Volumes']    = hsh['Volumes'].join(',')
+        hsh['Elastic IP'] = svr.cloud.public_ip if svr.cloud.public_ip
+        if block_given?
+          extra_info = yield(svr)
+          hsh.merge!(extra_info)
+          headings += extra_info.keys
+        end
+        hsh
+      end
+      if defined_data.empty?
+        ui.info "Nothing to report"
+      else
+        Formatador.display_compact_table(defined_data, headings.to_a)
+      end
+    end
+
+    def to_s
+      str = super
+      str[0..-2] + " #{@servers.map(&:fullname)}>"
+    end
+
+    def joined_names
+      map(&:name).join(", ").gsub(/, ([^,]*)$/, ' and \1')
+    end
+
+    # Calls block on each server in parallel, each in its own thread
+    #
+    # @example
+    #   target = Ironfan::Cluster.slice('web_server')
+    #   target.parallelize{|svr| svr.launch }
+    #
+    # @yield each server, in turn
+    #
+    # @return array (in same order as servers) of each block's result
+    def parallelize
+      servers.map do |svr|
+        sleep(0.1) # avoid hammering with simultaneous requests
+        Thread.new(svr){|svr| yield(svr) }
+      end
+    end
+
+  protected
+
+    # Helper methods for iterating through the servers to do things
+    #
+    # @param [Symbol]  method   -- method to call on each server
+    # @param [Boolean] threaded -- execute each call in own thread
+    #
+    # @return array (in same order as servers) of results for that method
+    def delegate_to_servers method, threaded = true
+      if threaded  # Call in threads
+        threads = parallelize{|svr| svr.send(method) }
+        threads.map{|t| t.join.value } # Wait, returning array of results
+      else         # Call the method for each server sequentially
+        servers.map{|svr| svr.send(method) }
+      end
+    end
+
+    def delegate_to_fog_servers method
+      fog_servers.compact.map do |fs|
+        fs.send(method)
+      end
+    end
+
+  end
+end

data/lib/ironfan/volume.rb

@@ -0,0 +1,157 @@
+module Ironfan
+  #
+  # Internal or external storage
+  #
+  class Volume < Ironfan::DslObject
+    attr_reader   :parent
+    attr_accessor :fog_volume
+    has_keys(
+      :name,
+      # mountable volume attributes
+      :device, :mount_point, :mount_options, :fstype, :mount_dump, :mount_pass,
+      :mountable, :formattable, :resizable, :in_raid,
+      # cloud volume attributes
+      :attachable, :create_at_launch, :volume_id, :snapshot_id, :size, :keep, :availability_zone,
+      # arbitrary tags
+      :tags
+      )
+
+    VOLUME_DEFAULTS = {
+      :fstype          => 'xfs',
+      :mount_options    => 'defaults,nouuid,noatime',
+      :keep             => true,
+      :attachable       => :ebs,
+      :create_at_launch => false,
+      #
+      :mountable        => true,
+      :resizable        => false,
+      :formattable      => false,
+      :in_raid          => false,
+    }
+
+    # Snapshot for snapshot_name method.
+    # Set your own by adding
+    #
+    #     VOLUME_IDS = Mash.new unless defined?(VOLUME_IDS)
+    #     VOLUME_IDS.merge!({ :your_id => 'snap-whatever' })
+    #
+    # to your organization's knife.rb
+    #
+    VOLUME_IDS = Mash.new unless defined?(VOLUME_IDS)
+    VOLUME_IDS.merge!({
+      :blank_xfs       => 'snap-d9c1edb1',
+    })
+
+    # Describes a volume
+    #
+    # @example
+    #   Ironfan::Volume.new( :name => 'redis',
+    #     :device => '/dev/sdp', :mount_point => '/data/redis', :fstype => 'xfs', :mount_options => 'defaults,nouuid,noatime'
+    #     :size => 1024, :snapshot_id => 'snap-66494a08', :volume_id => 'vol-12312',
+    #     :tags => {}, :keep => true )
+    #
+    def initialize attrs={}
+      @parent = attrs.delete(:parent)
+      super(attrs)
+      @settings[:tags] ||= {}
+    end
+
+    # human-readable description for logging messages and such
+    def desc
+      "#{name} on #{parent.fullname} (#{volume_id} @ #{device})"
+    end
+
+    def defaults
+      self.configure(VOLUME_DEFAULTS)
+    end
+
+    def ephemeral_device?
+      volume_id =~ /^ephemeral/
+    end
+
+    # Named snapshots, as defined in Ironfan::Volume::VOLUME_IDS
+    def snapshot_name(name)
+      snap_id = VOLUME_IDS[name.to_sym]
+      raise "Unknown snapshot name #{name} - is it defined in Ironfan::Volume::VOLUME_IDS?" unless snap_id
+      self.snapshot_id(snap_id)
+    end
+
+    # With snapshot specified but volume missing, have it auto-created at launch
+    #
+    # Be careful with this -- you can end up with multiple volumes claiming to
+    # be the same thing.
+    #
+    def create_at_launch?
+      volume_id.blank? && self.create_at_launch
+    end
+
+    def in_cloud?
+      !! fog_volume
+    end
+
+    def has_server?
+      in_cloud? && fog_volume.server_id.present?
+    end
+
+    def reverse_merge!(other_hsh)
+      super(other_hsh)
+      self.tags.reverse_merge!(other_hsh.tags) if other_hsh.respond_to?(:tags) && other_hsh.tags.present?
+      self
+    end
+
+    # An array of hashes with dorky-looking keys, just like Fog wants it.
+    def block_device_mapping
+      hsh = { 'DeviceName' => device }
+      if ephemeral_device?
+        hsh['VirtualName'] = volume_id
+      elsif create_at_launch?
+        raise "Must specify a size or a snapshot ID for #{self}" if snapshot_id.blank? && size.blank?
+        hsh['Ebs.SnapshotId'] = snapshot_id if snapshot_id.present?
+        hsh['Ebs.VolumeSize'] = size.to_s   if size.present?
+        hsh['Ebs.DeleteOnTermination'] = (! keep).to_s
+      else
+        return
+      end
+      hsh
+    end
+
+  end
+
+
+  #
+  # Consider raising the chunk size to 256 and setting read_ahead 65536 if you are raid'ing EBS volumes
+  #
+  # * http://victortrac.com/EC2_Ephemeral_Disks_vs_EBS_Volumes
+  # * http://orion.heroku.com/past/2009/7/29/io_performance_on_ebs/
+  # * http://tech.blog.greplin.com/aws-best-practices-and-benchmarks
+  # * http://stu.mp/2009/12/disk-io-and-throughput-benchmarks-on-amazons-ec2.html
+  #
+  class RaidGroup < Volume
+    has_keys(
+      :sub_volumes,     # volumes that comprise this raid group
+      :level,           # RAID level (http://en.wikipedia.org/wiki/RAID#Standard_levels)
+      :chunk,           # Raid chunk size (https://raid.wiki.kernel.org/articles/r/a/i/RAID_setup_cbb2.html)
+      :read_ahead,      # read-ahead buffer
+      )
+
+    def desc
+      "#{name} on #{parent.fullname} (#{volume_id} @ #{device} from #{sub_volumes.join(',')})"
+    end
+
+    def defaults()
+      super
+      fstype            'xfs'
+      mount_options     "defaults,nobootwait,noatime,nouuid,comment=ironfan"
+      attachable        false
+      create_at_launch  false
+      #
+      mountable         true
+      resizable         false
+      formattable       true
+      #
+      in_raid           false
+      #
+      sub_volumes       []
+    end
+  end
+end

data/spec/ironfan/cluster_spec.rb

@@ -0,0 +1,13 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require IRONFAN_DIR("lib/ironfan")
+
+describe Ironfan::Cluster do
+  describe 'discover!' do
+    let(:cluster){ get_example_cluster(:monkeyballs) }
+
+    it 'enumerates chef nodes' do
+      cluster.discover!
+    end
+  end
+end

data/spec/ironfan/facet_spec.rb

@@ -0,0 +1,69 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require IRONFAN_DIR("lib/ironfan")
+
+describe Ironfan::Facet do
+  let(:cluster){ Ironfan.cluster(:gibbon) }
+  let(:facet){
+    cluster.facet(:namenode) do
+      instances 5
+    end
+  }
+
+  describe 'slicing' do
+    it 'has servers' do
+      facet.indexes.should == [0, 1, 2, 3, 4]
+      facet.valid_indexes.should == [0, 1, 2, 3, 4]
+      facet.server(3){ name(:bob) }
+      svrs = facet.servers
+      svrs.length.should == 5
+      svrs.map{|svr| svr.name }.should == ["gibbon-namenode-0", "gibbon-namenode-1", "gibbon-namenode-2", :bob, "gibbon-namenode-4"]
+    end
+
+    it 'servers have bogosity if out of range' do
+      facet.server(69).should be_bogus
+      facet.servers.select(&:bogus?).map(&:facet_index).should == [69]
+      facet.indexes.should       == [0, 1, 2, 3, 4, 69]
+      facet.valid_indexes.should == [0, 1, 2, 3, 4]
+    end
+
+    it 'returns all on nil or "", but [] means none' do
+      facet.server(69)
+      facet.slice('' ).map(&:facet_index).should == [0, 1, 2, 3, 4, 69]
+      facet.slice(nil).map(&:facet_index).should == [0, 1, 2, 3, 4, 69]
+      facet.slice([] ).map(&:facet_index).should == []
+    end
+
+    it 'slice returns all by default' do
+      facet.server(69)
+      facet.slice().map(&:facet_index).should == [0, 1, 2, 3, 4, 69]
+    end
+
+    it 'with an array returns specified indexes (bogus or not) in sorted order' do
+      facet.server(69)
+      facet.slice( [3, 1, 0]     ).map(&:facet_index).should == [0, 1, 3]
+      facet.slice( [3, 1, 69, 0] ).map(&:facet_index).should == [0, 1, 3, 69]
+    end
+
+    it 'with an array does not create new dummy servers' do
+      facet.server(69)
+      facet.slice( [3, 1, 69, 0, 75, 123] ).map(&:facet_index).should == [0, 1, 3, 69]
+      facet.has_server?(75).should be_false
+      facet.has_server?(69).should be_true
+    end
+
+    it 'with a string, converts to intervals' do
+      facet.slice('1'      ).map(&:facet_index).should == [1]
+      facet.slice('5'      ).map(&:facet_index).should == []
+      facet.slice('1-1'    ).map(&:facet_index).should == [1]
+      facet.slice('0-1'    ).map(&:facet_index).should == [0,1]
+      facet.slice('0-1,3-4').map(&:facet_index).should == [0,1,3,4]
+      facet.slice('0-1,69' ).map(&:facet_index).should == [0,1,69]
+      facet.slice('0-2,1-3').map(&:facet_index).should == [0,1,2,3]
+      facet.slice('3-1'    ).map(&:facet_index).should == []
+      facet.slice('2-5'    ).map(&:facet_index).should == [2,3,4]
+      facet.slice(1).map(&:facet_index).should == [1]
+    end
+
+  end
+end