ironfan 3.2.2 → 4.0.0

data/lib/ironfan/provider/ec2/machine.rb

@@ -0,0 +1,258 @@
+module Ironfan
+  class Provider
+    class Ec2
+
+      class Machine < Ironfan::IaasProvider::Machine
+        delegate :_dump, :addresses, :ami_launch_index, :ami_launch_index=,
+            :architecture, :architecture=, :availability_zone,
+            :availability_zone=, :block_device_mapping, :block_device_mapping=,
+            :client_token, :client_token=, :collection, :collection=,
+            :connection, :connection=, :console_output, :created_at,
+            :created_at=, :destroy, :dns_name, :dns_name=, :flavor, :flavor=,
+            :flavor_id, :flavor_id=, :groups, :groups=, :iam_instance_profile,
+            :iam_instance_profile=, :iam_instance_profile_arn=,
+            :iam_instance_profile_name=, :id, :id=, :identity, :identity=,
+            :image_id, :image_id=, :instance_initiated_shutdown_behavior,
+            :instance_initiated_shutdown_behavior=, :ip_address, :kernel_id,
+            :kernel_id=, :key_name, :key_name=, :key_pair, :key_pair=,
+            :monitor=, :monitoring, :monitoring=, :network_interfaces,
+            :network_interfaces=, :new_record?, :password, :password=,
+            :placement_group, :placement_group=, :platform, :platform=,
+            :private_dns_name, :private_dns_name=, :private_ip_address,
+            :private_ip_address=, :private_key, :private_key=,
+            :private_key_path, :private_key_path=, :product_codes,
+            :product_codes=, :public_ip_address, :public_ip_address=,
+            :public_key, :public_key=, :public_key_path, :public_key_path=,
+            :ramdisk_id, :ramdisk_id=, :ready?, :reason, :reason=, :reboot,
+            :reload, :requires, :requires_one, :root_device_name,
+            :root_device_name=, :root_device_type, :root_device_type=, :save,
+            :scp, :scp_download, :scp_upload, :security_group_ids,
+            :security_group_ids=, :setup, :ssh, :ssh_port, :sshable?, :start,
+            :state, :state=, :state_reason, :state_reason=, :stop, :subnet_id,
+            :subnet_id=, :symbolize_keys, :tags, :tags=, :tenancy, :tenancy=,
+            :user_data, :user_data=, :username, :username=, :volumes, :vpc_id,
+            :vpc_id=, :wait_for,
+          :to => :adaptee
+
+
+        def self.shared?()      false;  end
+        def self.multiple?()    false;  end
+#        def self.resource_type()        Ironfan::IaasProvider::Machine;   end
+        def self.resource_type()        :machine;   end
+        def self.expected_ids(computer) [computer.server.fullname];   end
+
+        def name
+          return id if tags.empty?
+          tags["Name"] || tags["name"] || id
+        end
+
+        def public_hostname()           dns_name;      end
+
+        def created?
+          not ['terminated', 'shutting-down'].include? state
+        end
+        def running?
+          state == "running"
+        end
+        def stopped?
+          state == "stopped"
+        end
+
+        def start
+          adaptee.start
+          adaptee.wait_for{ state == 'pending' }
+        end
+
+        def stop
+          adaptee.stop
+          adaptee.wait_for{ state == 'stopping' }
+        end
+
+        def to_display(style,values={})
+          # style == :minimal
+          values["State"] =             state.to_sym
+          values["MachineID"] =        id
+          values["Public IP"] =         public_ip_address
+          values["Private IP"] =        private_ip_address
+          values["Created On"] =        created_at.to_date
+          return values if style == :minimal
+
+          # style == :default
+          values["Flavor"] =            flavor_id
+          values["AZ"] =                availability_zone
+          return values if style == :default
+
+          # style == :expanded
+          values["Image"] =             image_id
+          values["Volumes"] =           volumes.map(&:id).join(', ')
+          values["SSH Key"] =           key_name
+          values
+        end
+
+        #
+        # Discovery
+        #
+        def self.load!(cluster=nil)
+          Ec2.connection.servers.each do |fs|
+            machine = new(:adaptee => fs)
+            if recall? machine.name
+              raise 'duplicate'
+              machine.bogus <<                 :duplicate_machines
+              recall(machine.name).bogus <<    :duplicate_machines
+              remember machine, :append_id => "duplicate:#{machine.id}"
+            elsif machine.created?
+              remember machine
+            else
+              remember machine, :append_id => "terminated:#{machine.id}"
+            end
+          end
+        end
+
+        # Find active machines that haven't matched, but should have,
+        #   make sure all bogus machines have a computer to attach to
+        #   for display purposes
+        def self.validate_resources!(computers)
+          recall.each_value do |machine|
+            next unless machine.users.empty? and machine.name
+            if machine.name.match("^#{computers.cluster.name}")
+              machine.bogus << :unexpected_machine
+            end
+            next unless machine.bogus?
+            fake =                Ironfan::Broker::Computer.new
+            fake[:machine] =      machine
+            fake.name =           machine.name
+            machine.users <<      fake
+            computers <<          fake
+          end
+        end
+
+        #
+        # Manipulation
+        #
+        def self.create!(computer)
+          Chef::Log.warn("CODE SMELL: overly large method: #{caller}")
+          return if computer.machine? and computer.machine.created?
+          Ironfan.step(computer.name,"creating cloud machine", :green)
+          # lint_fog
+          launch_desc = fog_launch_description(computer)
+          Chef::Log.debug(JSON.pretty_generate(launch_desc))
+
+          Ironfan.safely do
+            fog_server = Ec2.connection.servers.create(launch_desc)
+            machine = Machine.new(:adaptee => fog_server)
+            computer.machine = machine
+            remember machine, :id => computer.name
+
+            fog_server.wait_for { ready? }
+          end
+
+          # tag the computer correctly
+          tags = {
+            'cluster' =>      computer.server.cluster_name,
+            'facet' =>        computer.server.facet_name,
+            'index' =>        computer.server.index,
+            'name' =>         computer.name,
+            'Name' =>         computer.name,
+          }
+          Ec2.ensure_tags(tags,computer.machine)
+
+          # register the new volumes for later save!, and tag appropriately
+          computer.machine.volumes.each do |v|
+            ebs_vol = Ec2::EbsVolume.register v
+            drive = computer.drives.values.select do |drive|
+              drive.volume.device == ebs_vol.device
+            end.first
+            drive.disk = ebs_vol
+
+            vol_name = "#{computer.name}-#{drive.volume.name}"
+            tags['name'] = vol_name
+            tags['Name'] = vol_name
+            Ec2.ensure_tags(tags,ebs_vol)
+          end
+        end
+        def self.fog_launch_description(computer)
+          cloud =                       computer.server.cloud(:ec2)
+          user_data_hsh =               {
+            :chef_server =>             Chef::Config[:chef_server_url],
+            #:validation_client_name =>  Chef::Config[:validation_client_name],
+            #
+            :node_name =>               computer.name,
+            :organization =>            Chef::Config[:organization],
+            :cluster_name =>            computer.server.cluster_name,
+            :facet_name =>              computer.server.facet_name,
+            :facet_index =>             computer.server.index,
+            :client_key =>              computer[:client].private_key
+          }
+
+          # Fog does not actually create tags when it creates a server;
+          #  they and permanence are applied during sync
+          keypair = cloud.keypair || computer.server.cluster_name
+          description = {
+            :image_id             => cloud.image_id,
+            :flavor_id            => cloud.flavor,
+            :vpc_id               => cloud.vpc,
+            :subnet_id            => cloud.subnet,
+            :groups               => cloud.security_groups.keys,
+            :key_name             => keypair.to_s,
+            :user_data            => JSON.pretty_generate(user_data_hsh),
+            :block_device_mapping => block_device_mapping(computer),
+            :availability_zone    => cloud.default_availability_zone,
+            :monitoring           => cloud.monitoring,
+          }
+
+          if cloud.flavor_info[:placement_groupable]
+            ui.warn "1.3.1 and earlier versions of Fog don't correctly support placement groups, so your nodes will land willy-nilly. We're working on a fix"
+            description[:placement] = { 'groupName' => cloud.placement_group.to_s }
+          end
+          description
+        end
+        # An array of hashes with dorky-looking keys, just like Fog wants it.
+        def self.block_device_mapping(computer)
+          computer.drives.values.map do |drive|
+            next if drive.disk  # Don't create any disc already satisfied
+            volume = drive.volume or next
+            hsh = { 'DeviceName' => volume.device }
+            if volume.attachable == 'ephemeral'
+              hsh['VirtualName'] = drive.name
+            # if set for creation at launch (and not already created)
+            elsif drive.node[:volume_id].blank? && volume.create_at_launch
+              if volume.snapshot_id.blank? && volume.size.blank?
+                raise "Must specify a size or a snapshot ID for #{volume}"
+              end
+              hsh['Ebs.SnapshotId'] = volume.snapshot_id if volume.snapshot_id.present?
+              hsh['Ebs.VolumeSize'] = volume.size.to_s   if volume.size.present?
+              hsh['Ebs.DeleteOnTermination'] = (not volume.keep).to_s
+            else
+              next
+            end
+            hsh
+          end.compact
+        end
+
+        def self.destroy!(computer)
+          return unless computer.machine?
+          forget computer.machine.name
+          computer.machine.destroy
+          computer.machine.reload            # show the node as shutting down
+        end
+
+        def self.save!(computer)
+          return unless computer.machine?
+          # the EC2 API does not surface disable_api_termination as a value, so we
+          # have to set it every time.
+          permanent = computer.server.cloud(:ec2).permanent
+          return unless computer.created?
+          Ironfan.step(computer.name, "setting termination flag #{permanent}", :blue)
+          Ironfan.unless_dry_run do
+            Ironfan.safely do
+              Ec2.connection.modify_instance_attribute( computer.machine.id,
+                {'DisableApiTermination.Value' => computer.permanent?, })
+            end
+          end
+        end
+      end
+
+    end
+  end
+end
+

data/lib/ironfan/provider/ec2/placement_group.rb

@@ -0,0 +1,24 @@
+module Ironfan
+  class Provider
+    class Ec2
+
+      # Fog::AWS doesn't seem to have native models for PlacementGroup,
+      #   using Hash semantics instead
+      class PlacementGroup < Ironfan::Provider::Resource
+        delegate :[],:[]=,      :to => :adaptee
+
+        def name()
+          self["groupName"]
+        end
+
+        def self.load!(cluster)
+          result = Ec2.connection.describe_placement_groups
+          result.body["placementGroupSet"].each do |group|
+            register group unless group.blank?
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/ironfan/provider/ec2/security_group.rb

@@ -0,0 +1,118 @@
+module Ironfan
+  class Provider
+    class Ec2
+
+      class SecurityGroup < Ironfan::Provider::Resource
+        delegate :_dump, :authorize_group_and_owner, :authorize_port_range,
+            :collection, :collection=, :connection, :connection=, :description,
+            :description=, :destroy, :group_id, :group_id=, :identity,
+            :identity=, :ip_permissions, :ip_permissions=, :name, :name=,
+            :new_record?, :owner_id, :owner_id=, :reload, :requires,
+            :requires_one, :revoke_group_and_owner, :revoke_port_range, :save,
+            :symbolize_keys, :vpc_id, :vpc_id=, :wait_for,
+          :to => :adaptee
+        field :ensured,        :boolean,       :default => false
+
+        def self.shared?()      true;   end
+        def self.multiple?()    true;   end
+        def self.resource_type()        :security_group;   end
+        def self.expected_ids(computer)
+          computer.server.cloud(:ec2).security_groups.keys.map{|k| k.to_s}.uniq
+        end
+
+        #
+        # Discovery
+        #
+        def self.load!(cluster=nil)
+          Ec2.connection.security_groups.each do |sg|
+            remember SecurityGroup.new(:adaptee => sg) unless sg.blank?
+          end
+        end
+
+        #
+        # Manipulation
+        #
+
+        def self.create!(computer)
+          return unless Ec2.applicable computer
+ 
+          ensure_groups(computer)
+          groups = computer.server.cloud(:ec2).security_groups.keys.map{|k| k.to_s}.uniq
+          # Only handle groups that don't already exist
+          groups.delete_if {|group| recall? group.to_s }
+          return if groups.empty?
+
+          Ironfan.step(computer.server.cluster_name, "creating security groups", :blue)
+          groups.each do |group|
+            Ironfan.step(group, "  creating #{group} security group", :blue)
+            Ec2.connection.create_security_group(group.to_s,"Ironfan created group #{group}")
+          end
+          load! # Get the native groups via reload
+        end
+
+        def self.save!(computer)
+          return unless Ec2.applicable computer
+
+          create!(computer)            # Make sure the security groups exist
+          security_groups = computer.server.cloud(:ec2).security_groups.values
+          dsl_groups = security_groups.select do |dsl_group|
+            not (recall? dsl_group or recall(dsl_group.name).ensured) and \
+            not (dsl_group.range_authorizations + dsl_group.group_authorized_by).empty?
+          end.compact
+          return if dsl_groups.empty?
+
+          Ironfan.step(computer.server.cluster_name, "ensuring security group permissions", :blue)
+          dsl_groups.each do |dsl_group|
+            dsl_group.group_authorized_by.each do |other_group|
+              Ironfan.step(dsl_group.name, "  ensuring access to #{other_group}", :blue)
+              options = {:group => "#{Ec2.aws_account_id}:#{dsl_group.name}"}
+              safely_authorize(other_group,1..65535,options)
+            end
+
+            dsl_group.range_authorizations.each do |range_auth|
+              range, cidr, protocol = range_auth
+              step_message = "  ensuring #{protocol} access from #{cidr} to #{range}"
+              Ironfan.step(dsl_group.name, step_message, :blue)
+              options = {:cidr_ip => cidr, :ip_protocol => protocol}
+              safely_authorize(dsl_group.name,range,options)
+            end
+          end
+        end
+
+        #
+        # Utility
+        #
+        def self.ensure_groups(computer)
+          return unless Ec2.applicable computer
+          # Ensure the security_groups include those for cluster & facet
+          # FIXME: This violates the DSL's immutability; it should be
+          #   something calculated from within the DSL construction
+          Chef::Log.warn("CODE SMELL: violation of DSL immutability: #{caller}")
+          cloud = computer.server.cloud(:ec2)
+          c_group = cloud.security_group(computer.server.cluster_name)
+          c_group.authorized_by_group(c_group.name)
+          facet_name = "#{computer.server.cluster_name}-#{computer.server.facet_name}"
+          cloud.security_group(facet_name)
+        end
+
+        # Try an authorization, ignoring duplicates (this is easier than correlating).
+        # Do so for both TCP and UDP, unless only one is specified
+        def self.safely_authorize(group_name,range,options)
+          unless options[:ip_protocol]
+            safely_authorize(group_name,range,options.merge(:ip_protocol => 'tcp'))
+            safely_authorize(group_name,range,options.merge(:ip_protocol => 'udp'))
+            return
+          end
+
+          fog_group = recall(group_name) or raise "unrecognized group: #{group_name}"
+          begin
+            fog_group.authorize_port_range(range,options)
+          rescue Fog::Compute::AWS::Error => e      # InvalidPermission.Duplicate
+            Chef::Log.debug("ignoring #{e}")
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/ironfan/provider/ec2.rb

@@ -0,0 +1,47 @@
+module Ironfan
+  class Provider
+
+    class Ec2 < Ironfan::IaasProvider
+
+      def self.resources
+        [ Machine, EbsVolume, KeyPair, SecurityGroup ]
+      end
+
+      #
+      # Utility functions
+      #
+      def self.connection
+        @@connection ||= Fog::Compute.new({
+          :provider              => 'AWS',
+          :aws_access_key_id     => Chef::Config[:knife][:aws_access_key_id],
+          :aws_secret_access_key => Chef::Config[:knife][:aws_secret_access_key],
+          :region                => Chef::Config[:knife][:region]
+        })
+      end
+      
+      def self.aws_account_id()
+        Chef::Config[:knife][:aws_account_id]
+      end
+
+      # Ensure that a fog object (machine, volume, etc.) has the proper tags on it
+      def self.ensure_tags(tags,fog)
+        tags.delete_if {|k, v| fog.tags[k] == v.to_s  rescue false }
+        return if tags.empty?
+
+        Ironfan.step(fog.name,"tagging with #{tags.inspect}", :green)
+        tags.each do |k, v|
+          Chef::Log.debug( "tagging #{fog.name} with #{k} = #{v}" )
+          Ironfan.safely do
+            config = {:key => k, :value => v.to_s, :resource_id => fog.id }
+            connection.tags.create(config)
+          end
+        end
+      end
+
+      def self.applicable(computer)
+        computer.server and computer.server.clouds.include?(:ec2)
+      end
+    end
+
+  end
+end

data/lib/ironfan/provider/virtualbox/machine.rb

@@ -0,0 +1,10 @@
+module Ironfan
+  class Provider
+    class VirtualBox
+
+      class Machine < Ironfan::IaasProvider::Machine
+      end
+
+    end
+  end
+end

data/lib/ironfan/provider/virtualbox.rb

@@ -0,0 +1,8 @@
+module Ironfan
+  class Provider
+
+    class VirtualBox < Ironfan::IaasProvider
+    end
+
+  end
+end

data/lib/ironfan/provider.rb

@@ -0,0 +1,139 @@
+# Providers present a lightweight wrapper for various third-party services,
+#   such as Chef's node and client APIs, and Amazon's EC2 APIs. This allows
+#   Ironfan ask specialized questions (such as whether a given resource 
+#   matches
+module Ironfan
+  class Provider < Builder
+
+    def self.receive(obj,&block)
+      obj[:_type] = case obj[:name]
+        when    :chef;          Chef
+        when    :ec2;           Ec2
+        when    :virtualbox;    VirtualBox
+        else;   raise "Unsupported provider #{obj[:name]}"
+      end unless native?(obj)
+      super
+    end
+
+    def resources()     self.class.resources;   end
+    def self.resources
+      raise "missing #{self.class}.resources declaration"
+    end
+
+    #
+    # Discovery
+    #
+    def self.load(cluster)
+      Ironfan.delegate_to(resources) { load! cluster }
+    end
+
+    def self.validate(computers)
+      Ironfan.delegate_to(resources) { validate_resources! computers }
+    end
+
+
+    class Resource < Builder
+      @@known = {}
+      field             :adaptee,       Whatever
+      field             :bogus,         Array,          :default => []
+      attr_accessor     :owner
+      attr_accessor     :users
+      def users()       @users ||= [];  end;
+
+      def bogus?()                      !bogus.empty?;          end
+
+      #
+      # Flags
+      # 
+      # Non-shared resources live and die with the computer
+      def self.shared?()                true;                   end
+      # Can multiple instances of this resource be associated with the computer?
+      def self.multiple?()              false;                  end
+
+      #
+      # Discovery
+      #
+      def self.load!(*p)                Ironfan.noop(self,__method__,*p);   end
+      def self.correlate!(*p)           Ironfan.noop(self,__method__,*p);   end
+      def self.validate_computer!(*p)   Ironfan.noop(self,__method__,*p);   end
+      def self.validate_resources!(*p)  Ironfan.noop(self,__method__,*p);   end
+
+      def on_correlate(*p)              Ironfan.noop(self,__method__,*p);   end
+
+      #
+      # Manipulation
+      #
+      def self.create!(*p)              Ironfan.noop(self,__method__,*p);   end
+      def self.save!(*p)                Ironfan.noop(self,__method__,*p);   end
+      def self.destroy!(*p)             Ironfan.noop(self,__method__,*p);   end
+
+      #
+      # Utilities
+      #
+      [:shared?, :multiple?, :load!,:correlate!,:validate_computer!,
+       :validate_resources!,:create!,:save!,:destroy!].each do |method_name|
+        define_method(method_name) {|*p| self.class.send(method_name,*p) }
+      end
+
+      def self.remember(resource,options={})
+        index = options[:id] || resource.name
+        index += options[:append_id] if options[:append_id]
+        self.known[index] = resource
+      end
+
+      # Register and return the (adapted) object with the collection
+      def self.register(native)
+        result = new(:adaptee => native)
+        remember result unless result.nil?
+      end
+
+      def self.recall?(id)
+        self.known.include? id
+      end
+
+      def self.recall(id=nil)
+        return self.known if id.nil?
+        self.known[id]
+      end
+
+      def self.forget(id)
+        self.known.delete(id)
+      end
+
+      # Provide a separate namespace in @@known for each subclass
+      def self.known
+        @@known[self.name] ||= {}
+      end
+    end
+
+  end
+
+  class IaasProvider < Provider
+    def self.machine_class
+      self.const_get(:Machine)
+    end
+
+    #
+    # Manipulation
+    #
+    def ensure_prerequisites!(computers)
+      # Create all things that aren't machines
+      targets = resources.reject {|type| type < IaasProvider::Machine}
+      computers.each do |computer|
+        delegate_to(targets) { create! computer }
+      end
+    end
+
+    def save!(computers)
+      computers.each do |computer|
+        delegate_to(resources) { save! computer }
+      end
+    end
+
+    class Machine < Resource
+      # A Machine lives and dies with its Computer
+      def self.shared?()        false;                   end
+    end
+  end
+
+end

data/lib/ironfan/requirements.rb

@@ -0,0 +1,52 @@
+# Gorillib core classes
+require 'gorillib/builder'
+require 'gorillib/resolution'
+
+
+# Pre-declaration of class hierarchy
+require 'ironfan/headers'
+
+
+# DSL for cluster descriptions
+require 'ironfan/dsl'
+require 'ironfan/builder'
+
+require 'ironfan/dsl/compute'
+require 'ironfan/dsl/server'
+require 'ironfan/dsl/facet'
+require 'ironfan/dsl/cluster'
+
+require 'ironfan/dsl/role'
+require 'ironfan/dsl/volume'
+
+require 'ironfan/dsl/cloud'
+require 'ironfan/dsl/ec2'
+
+
+# Providers for specific resources
+require 'ironfan/provider'
+
+require 'ironfan/provider/chef'
+require 'ironfan/provider/chef/client'
+require 'ironfan/provider/chef/node'
+require 'ironfan/provider/chef/role'
+
+require 'ironfan/provider/ec2'
+require 'ironfan/provider/ec2/ebs_volume'
+require 'ironfan/provider/ec2/machine'
+require 'ironfan/provider/ec2/key_pair'
+require 'ironfan/provider/ec2/placement_group'
+require 'ironfan/provider/ec2/security_group'
+
+require 'ironfan/provider/virtualbox'
+require 'ironfan/provider/virtualbox/machine'
+
+
+# Broker classes to coordinate DSL expectations and provider resources
+require 'ironfan/broker'
+require 'ironfan/broker/computer'
+require 'ironfan/broker/drive'
+
+
+# Calls that are slated to go away
+require 'ironfan/deprecated'