ironfan 5.0.11 → 6.0.0

data/lib/ironfan/dsl/openstack.rb

@@ -0,0 +1,147 @@
+require 'digest/md5'
+
+module Ironfan
+  class Dsl
+
+    class Compute < Ironfan::Dsl
+      def openstack(*attrs,&block)            cloud(:openstack,*attrs,&block);      end
+    end
+
+    class OpenStack < Cloud
+      magic :availability_zones,        Array,          :default => ['nova']
+      magic :backing,                   String,         :default => 'ebs'
+      magic :bits,                      Integer,        :default => ->{ flavor_info[:bits] }
+      magic :bootstrap_distro,          String,         :default => ->{ image_info[:bootstrap_distro] }
+      magic :chef_client_script,        String
+      magic :default_availability_zone, String,         :default => ->{ availability_zones.first }
+      #collection :elastic_load_balancers,  Ironfan::Dsl::Ec2::ElasticLoadBalancer, :key_method => :name
+      magic :ebs_optimized,             :boolean,       :default => false
+      magic :flavor,                    String,         :default => 't1.micro'
+      #collection :iam_server_certificates, Ironfan::Dsl::Ec2::IamServerCertificate, :key_method => :name
+      magic :image_id,                  String
+      magic :image_name,                String
+      magic :keypair,                   String
+      magic :monitoring,                String
+      magic :mount_ephemerals,          Hash,           :default => {}
+      magic :permanent,                 :boolean,       :default => false
+      magic :placement_group,           String
+      magic :provider,                  Whatever,       :default => Ironfan::Provider::OpenStack
+      magic :elastic_ip,                String
+      magic :auto_elastic_ip,           String
+      magic :allocation_id,             String
+      magic :region,                    String,         :default => ->{ default_region }
+      collection :security_groups,      Ironfan::Dsl::SecurityGroup, :key_method => :name
+      magic :ssh_user,                  String,         :default => ->{ image_info[:ssh_user] }
+      magic :ssh_identity_dir,          String,         :default => ->{ Chef::Config.openstack_key_dir }
+      magic :subnet,                    String
+      magic :validation_key,            String,         :default => ->{ IO.read(Chef::Config.validation_key) rescue '' }
+      magic :vpc,                       String
+      magic :dns_search_domain,         String,         :default => 'novalocal'
+
+      def domain;                       vpc.nil? ? 'standard' : 'vpc';       end
+
+      def image_info
+        bit_str = "#{self.bits.to_i}-bit" # correct for legacy image info.
+        keys = [region, bit_str, backing, image_name]
+        info = Chef::Config[:openstack_image_info][ keys ]
+        ui.warn("Can't find image for #{[region, bit_str, backing, image_name].inspect}") if info.blank?
+        return info || {}
+      end
+
+      def image_id
+        result = read_attribute(:image_id) || image_info[:image_id]
+      end
+
+      def ssh_key_name(computer)
+        keypair ? keypair.to_s : computer.server.cluster_name
+      end
+
+      def default_region
+        default_availability_zone ? default_availability_zone.gsub(/^(\w+-\w+-\d)[a-z]/, '\1') : nil
+      end
+
+      def to_display(style,values={})
+        return values if style == :minimal
+
+        values["Flavor"] =            flavor
+        values["AZ"] =                default_availability_zone
+        return values if style == :default
+
+        values["Public IP"] =        elastic_ip if elastic_ip
+        values
+      end
+
+      def flavor_info
+        if not Chef::Config[:openstack_flavor_info].has_key?(flavor)
+          ui.warn("Unknown machine image flavor '#{flavor}'")
+          list_flavors
+          return nil
+        end
+        Chef::Config[:openstack_flavor_info][flavor]
+      end
+
+      def implied_volumes
+        result = []
+        if backing == 'ebs'
+          result << Ironfan::Dsl::Volume.new(:name => 'root') do
+            device              '/dev/sda1'
+            fstype              'ext4'
+            keep                false
+            mount_point         '/'
+          end
+        end
+        return result unless (mount_ephemerals and (flavor_info[:ephemeral_volumes] > 0))
+
+        layout = {  0 => ['/dev/sdb','/mnt'],
+                    1 => ['/dev/sdc','/mnt2'],
+                    2 => ['/dev/sdd','/mnt3'],
+                    3 => ['/dev/sde','/mnt4']   }
+        ( 0 .. (flavor_info[:ephemeral_volumes]-1) ).each do |idx|
+          dev, mnt = layout[idx]
+          ephemeral = Ironfan::Dsl::Volume.new(:name => "ephemeral#{idx}") do
+            attachable          'ephemeral'
+            fstype              'ext3'
+            device              dev
+            mount_point         mnt
+            mount_options       'defaults,noatime'
+            tags({:bulk => true, :local => true, :fallback => true})
+          end
+          ephemeral_attrs = mount_ephemerals.clone
+          if ephemeral_attrs.has_key?(:disks)
+            disk_attrs = mount_ephemerals[:disks][idx] || { }
+            ephemeral_attrs.delete(:disks)
+            ephemeral_attrs.merge!(disk_attrs)
+          end
+          ephemeral.receive! ephemeral_attrs
+          result << ephemeral
+        end
+        result
+      end
+
+      def receive_provider(obj)
+        if obj.is_a?(String)
+          write_attribute :provider, Gorillib::Inflector.constantize(Gorillib::Inflector.camelize(obj.gsub(/\./, '/')))
+        else
+          super(obj)
+        end
+      end
+    end
+  end
+end
+
+Chef::Config[:openstack_flavor_info] ||= {}
+Chef::Config[:openstack_flavor_info].merge!({
+    # 32-or-64: m1.small, m1.medium, t1.micro, c1.medium
+    't1.micro'    => { :price => 0.02,  :bits => 64, :ram =>    686, :cores => 1, :core_size => 0.25, :inst_disks => 0, :inst_disk_size =>    0, :ephemeral_volumes => 0 },
+    'm1.small'    => { :price => 0.08,  :bits => 64, :ram =>   1740, :cores => 1, :core_size => 1,    :inst_disks => 1, :inst_disk_size =>  160, :ephemeral_volumes => 1 },
+    'm1.medium'   => { :price => 0.165, :bits => 64, :ram =>   3840, :cores => 2, :core_size => 1,    :inst_disks => 1, :inst_disk_size =>  410, :ephemeral_volumes => 1 },
+    'c1.medium'   => { :price => 0.17,  :bits => 64, :ram =>   1740, :cores => 2, :core_size => 2.5,  :inst_disks => 1, :inst_disk_size =>  350, :ephemeral_volumes => 1 },
+    #
+    'm1.large'    => { :price => 0.32,  :bits => 64, :ram =>   7680, :cores => 2, :core_size => 2,    :inst_disks => 1, :inst_disk_size =>  850, :ephemeral_volumes => 2, },
+    'm2.xlarge'   => { :price => 0.45,  :bits => 64, :ram =>  18124, :cores => 2, :core_size => 3.25, :inst_disks => 1, :inst_disk_size =>  420, :ephemeral_volumes => 1, },
+    'c1.xlarge'   => { :price => 0.64,  :bits => 64, :ram =>   7168, :cores => 8, :core_size => 2.5,  :inst_disks => 1, :inst_disk_size => 1690, :ephemeral_volumes => 4, },
+    'm1.xlarge'   => { :price => 0.66,  :bits => 64, :ram =>  15360, :cores => 4, :core_size => 2,    :inst_disks => 1, :inst_disk_size => 1690, :ephemeral_volumes => 4, },
+  })
+
+Chef::Config[:openstack_image_info] ||= {}
+

data/lib/ironfan/dsl/realm.rb

@@ -1,37 +1,44 @@
 module Ironfan
   class Dsl
-
     class Realm < Ironfan::Dsl::Compute
       collection :clusters,       Ironfan::Dsl::Cluster,   :resolver => :deep_resolve
 
-      def children
-        clusters.to_a
+      magic :cluster_suffixes,    Whatever
+
+      def self.definitions
+        @realms ||= {}
       end
 
-      magic :cluster_suffixes,    Whatever
+      def self.define(attrs = {}, &blk)
+        rlm = new(attrs)
+        rlm.receive!({}, &blk)
+        definitions[attrs[:name].to_sym] = rlm
+      end
 
-      def initialize(attrs={},&block)
-        cluster_names({})
+      def initialize(attrs = {}, &blk)
+        cluster_names Hash.new
         realm_name attrs[:name] if attrs[:name]
         attrs[:environment] = realm_name unless attrs.has_key?(:environment)
-        super(attrs, &block)
+        super(attrs, &blk)
       end
 
-      def cluster(label, attrs={},&blk)
-        new_name = [realm_name, label].join('_').to_sym
-
-        if clusters.keys.include? new_name
-          clusters[new_name].tap do |cl|
-            cl.receive!(attrs)
+      def cluster(label, attrs = {}, &blk)
+        if clusters.keys.include? label
+          clusters[label].tap do |cl|
+            cl.receive! attrs
             cl.instance_eval(&blk) if block_given?
           end
         else
-          cluster = Ironfan::Dsl::Cluster.new(name: new_name, owner: self, cluster_names: cluster_names)
-          cluster_names[label] = new_name
+          cluster = Ironfan::Dsl::Cluster.define(name: label, owner: self, cluster_names: cluster_names)
+          cluster_names[label] = label
           cluster.receive!(attrs, &blk)
-          super(new_name, cluster)
+          super(label, cluster)
         end
       end
+
+      def children
+        clusters.to_a
+      end
     end
   end
 end

data/lib/ironfan/dsl/security_group.rb

@@ -0,0 +1,29 @@
+module Ironfan
+  class Dsl
+    class SecurityGroup < Ironfan::Dsl
+      field :name,                    String
+      field :group_authorized,        Array, :default => []
+      field :group_authorized_by,     Array, :default => []
+      field :range_authorizations,    Array, :default => []
+      
+      def authorize_port_range(range, cidr_ip = '0.0.0.0/0', ip_protocol = 'tcp')
+        range = (range .. range) if range.is_a?(Integer)
+        range_authorizations << [range, cidr_ip, ip_protocol]
+        range_authorizations.compact!
+        range_authorizations.uniq!
+      end
+
+      def authorized_by_group(other_name)
+        group_authorized_by << other_name.to_s
+        group_authorized_by.compact!
+        group_authorized_by.uniq!
+      end
+
+      def authorize_group(other_name)
+        group_authorized << other_name.to_s
+        group_authorized.compact!
+        group_authorized.uniq!
+      end
+    end
+  end
+end

data/lib/ironfan/dsl/server.rb

@@ -8,6 +8,7 @@ module Ironfan
       field :environment, Symbol
       field :name, String
       field :cluster_name, String
+      field :realm_name, String
       field :facet_name, String
       field :components, Array, of: Ironfan::Dsl::Component, default: []
       field :run_list, Array, of: String, default: []
@@ -63,11 +64,13 @@ module Ironfan
       # field :validation_key,            String
       #-----------------------------------------------------------------------------------
 
+     
       # Reconstruct machine manifest from a computer, pulling
       # information from remote sources as necessary.
       def self.from_computer(computer)
         node = get_node(computer.name)
         cluster_name = node['cluster_name']
+        Chef::Log.info "name is whoa: #{cluster_name}"
         facet_name = node['facet_name']
         instance = node['facet_index']
 
@@ -228,16 +231,22 @@ module Ironfan
         super
       end
 
-      def full_name()           "#{cluster_name}-#{facet_name}-#{name}";        end
+      def full_name
+        "#{realm_name}-#{cluster_name}-#{facet_name}-#{name}"
+      end
+      def host_name()           full_name.gsub('_','-'); end;
+      def fqdn()                [self.host_name, self.dns_domain].compact.join(".");     end
+
       def index()               name.to_i;                                      end
-      def implied_volumes()     selected_cloud.implied_volumes;                 end
+      def implied_volumes()    selected_cloud.implied_volumes;                 end
 
       def to_display(style,values={})
         selected_cloud.to_display(style,values)
 
         return values if style == :minimal
 
-        values["Env"] =         environment
+        values["Env"]   = environment
+        values["Realm"] = realm_name
         values
       end
 
@@ -279,10 +288,10 @@ module Ironfan
                                 bootstrap_distro: cloud.bootstrap_distro,
                                 chef_client_script: cloud.chef_client_script,
                                 default_availability_zone: cloud.default_availability_zone,
-                                elastic_load_balancers: cloud.elastic_load_balancers,
+                                elastic_load_balancers: cloud.respond_to?(:elastic_load_balancers) ? cloud.elastic_load_balancers : nil,
                                 ebs_optimized: cloud.ebs_optimized,
                                 flavor: cloud.flavor,
-                                iam_server_certificates: cloud.iam_server_certificates,
+                                iam_server_certificates: cloud.respond_to?(:iam_server_certificates) ? cloud.iam_server_certificates : nil,
                                 image_id: cloud.image_id,
                                 image_name: cloud.image_name,
                                 keypair: cloud.keypair,

data/lib/ironfan/dsl/static.rb

@@ -0,0 +1,63 @@
+require 'digest/md5'
+
+module Ironfan
+  class Dsl
+
+    class Compute < Ironfan::Dsl
+      def static(*attrs,&block)            cloud(:static,*attrs,&block);      end
+    end
+
+    class Static < Cloud
+
+      magic :private_ip,                String
+      magic :public_ip,                 String
+      magic :private_hostname,          String
+      magic :public_hostname,           String
+      magic :keypair,                   String
+      magic :provider,                  Whatever,       :default => Ironfan::Provider::Static
+
+      magic :availability_zones,        Array,          :default => []
+      magic :backing,                   String,         :default => nil
+      magic :bits,                      Integer,        :default => 64
+      magic :bootstrap_distro,          String,         :default => "ubuntu12.04-ironfan"
+      magic :chef_client_script,        String
+      magic :default_availability_zone, String,         :default => "none"
+      magic :ebs_optimized,             :boolean,       :default => false
+      magic :flavor,                    String,         :default => 'whatever'
+      magic :image_id,                  String
+      magic :image_name,                String
+      magic :monitoring,                String
+      magic :mount_ephemerals,          Hash,           :default => {}
+      magic :permanent,                 :boolean,       :default => false
+      magic :placement_group,           String
+      magic :elastic_ip,                String
+      magic :auto_elastic_ip,           String
+      magic :allocation_id,             String
+      magic :region,                    String,         :default => 'none'
+      collection :security_groups,      Ironfan::Dsl::SecurityGroup, :key_method => :name
+      magic :ssh_user,                  String,         :default => 'ubuntu'
+      magic :ssh_identity_dir,          String,         :default => ->{ Chef::Config.openstack_key_dir }
+      magic :subnet,                    String
+      magic :validation_key,            String,         :default => ->{ IO.read(Chef::Config.validation_key) rescue '' }
+      magic :vpc,                       String
+      magic :dns_search_domain,         String,         :default => 'static'
+
+      def to_display(style,values={})
+        return values if style == :minimal
+
+        values["Private IP"] =   private_ip
+        values["Public IP"] =    public_ip
+        values
+      end
+
+      def flavor_info
+        return nil
+      end
+
+      def implied_volumes
+        []
+      end
+    end
+  end
+end
+

data/lib/ironfan/dsl/vsphere.rb

@@ -32,6 +32,7 @@ module Ironfan
       magic :virtual_disks,             Array,          :default => []
       magic :vsphere_datacenters,       Array,          :default => ['New Datacenter']
       magic :network, 			String,		:default => "VM Network"
+      magic :dns_search_domain,         String,         :default => 'internal'
 
       def image_info
         bit_str = "#{self.bits.to_i}-bit" # correct for legacy image info.

data/lib/ironfan/dsl.rb

@@ -1,139 +1,6 @@
 module Ironfan
-
   class Dsl < Builder
-    include Gorillib::Resolution
-
-    def self.default_cookbook_reqs
-      @default_cookbook_reqs ||= []
-    end
-
-    def self.cookbook_req name, constraint
-      default_cookbook_reqs << new_req(name, constraint)
-        
-    end
-
-    def join_req req1, req2
-      # order requirements by operation: =, >=, ~>
-      req1, req2 = (req1.constraint < req2.constraint) ? [req1, req2] : [req2, req1]
-      cn1, cn2 = [req1.constraint, req2.constraint]
-      vers1, vers2 = [cn1.split.last, cn2.split.last]
-      vers1_c, vers2_c = [vers1.split('.'), vers2.split('.')]
-      op1, op2 = [req1, req2].map{|x| x.constraint.split.first}
-
-      if op1 == '=' and op2 == '='
-        join_eq_eq(req1, req2)
-      elsif op1 == '>=' and op2 == '>='
-        join_geq_geq(req1, req2)
-      elsif op1 == '~>' and op2 == '~>'
-        join_agt_agt(req1, req2)
-      elsif op1 == '=' and op2 == '>='
-        join_eq_gte(req1, req2)
-      elsif op1 == '=' and op2 == '~>'
-        join_eq_agt(req1, req2)
-      elsif op1 == '>=' and op2 == '~>'
-        join_gte_agt(req1, req2)
-      end
-    end
-
-    def cookbook_req name, constraint
-      (@cookbook_reqs ||= []) << self.class.new_req(name, constraint)
-    end
-
-    def children() [] end
-
-    def cookbook_reqs
-      Hash[_cookbook_reqs.map{|x| [x.name, x.constraint]}]
-    end
-
-    def _cookbook_reqs
-      [
-       *shallow_cookbook_reqs,
-         *child_cookbook_reqs
-      ].group_by(&:name).values.map do |group|
-        group.inject{|result, req| join_req(result, req)}
-      end
-    end
-
-    private
-
-    #-----------------------------------------------------------------------------------------------
-
-    def join_eq_eq(req1, req2)
-      (vers(req1) == vers(req2)) ? req1 : bad_reqs(req1, req2)
-    end
-
-    def join_geq_geq(req1, req2)
-      (vers(req1) >= vers(req2)) ? req1 : req2
-    end
-
-    def join_agt_agt(req1, req2)
-      if vers_a(req1).size == vers_a(req2).size and
-          vers_a_head(req1).zip(vers_a_head(req2)).all?{|v1,v2| v1 == v2}
-        (req1.constraint > req2.constraint) ? req1 : req2
-      else
-        bad_reqs(req1, req2)
-      end
-    end
-
-    def join_eq_gte(req1, req2)
-      vers(req1) >= vers(req2) ? req1 : bad_reqs(req1, req2)
-    end
-
-    def join_eq_agt(req1, req2)
-      if match_v_head(req1, req2) and
-          vers_a(req1)[vers_a(req2).size - 1] >= vers_a(req2).last
-        req1
-      else
-        bad_reqs(req1, req2)
-      end
-    end
-
-    def join_gte_agt(req1, req2)
-      if match_v_head(req1, req2) and vers(req1) <= vers(req2)
-        req2
-      else
-        bad_reqs(req1, req2)
-      end
-    end
-
-    def match_v_head(req1, req2)
-      vers_a_head(req1, vers_a(req2).size).zip(vers_a_head(req2)).all?{|v1,v2| v1 == v2}      
-    end
-
-    def op(req)
-      req.constraint.split.first
-    end
-
-    def vers(req)
-      req.constraint.split.last
-    end
-
-    def vers_a(req)
-      vers(req).split('.')
-    end
-
-    def vers_a_head(req, last = 0)
-      vers_a(req)[0...last-1]
-    end
-
-    #-----------------------------------------------------------------------------------------------
-
-    def bad_reqs(req1, req2)
-      raise ArgumentError.new("#{req1.name}: cannot reconcile #{req1.constraint} with #{req2.constraint}")
-    end
-
-    def child_cookbook_reqs
-      children.map(&:_cookbook_reqs).flatten(1)
-    end
-
-    def self.new_req(name, constraint)
-      raise StandardError.new("Please don't use >= constraints. They're too vague!") if
-        constraint.start_with?('>=') and not (@@testing ||= false)
-      Ironfan::Plugin::CookbookRequirement.new(name: name, constraint: constraint)
-    end
+    include Gorillib::Resolution, CookbookRequirements
 
-    def shallow_cookbook_reqs
-      @cookbook_reqs || self.class.default_cookbook_reqs
-    end
   end
 end

data/lib/ironfan/headers.rb

@@ -31,6 +31,14 @@ module Ironfan
       class ElasticLoadBalancer < Ironfan::Dsl; end
       class IamServerCertificate < Ironfan::Dsl; end
     end
+
+    class OpenStack < Cloud
+      class SecurityGroup < Ironfan::Dsl; end
+    end
+
+    class Static < Cloud
+    end
+
     class VirtualBox < Cloud; end
     class Vsphere < Cloud; end
     class Rds < Cloud
@@ -65,6 +73,17 @@ module Ironfan
       class ElasticLoadBalancer < Ironfan::Provider::Resource; end
       class IamServerCertificate < Ironfan::Provider::Resource; end
     end
+
+    class OpenStack < Ironfan::IaasProvider
+      class Machine < Ironfan::IaasProvider::Machine; end
+      class Keypair < Ironfan::Provider::Resource; end
+      class SecurityGroup < Ironfan::Provider::Resource; end
+    end
+
+    class Static < Ironfan::IaasProvider
+      class Machine < Ironfan::IaasProvider::Machine; end
+    end
+
     class VirtualBox < Ironfan::IaasProvider
       class Machine < Ironfan::IaasProvider::Machine; end
     end

data/lib/ironfan/provider/chef/node.rb

@@ -67,6 +67,7 @@ module Ironfan
           server =                      computer.server
           chef_environment(server.environment.to_s)
           run_list.instance_eval        { @run_list_items = server.run_list }
+          normal[:realm_name] =         server.realm_name
           normal[:cluster_name] =       server.cluster_name
           normal[:facet_name] =         server.facet_name
           normal[:permanent] =          computer.permanent?
@@ -86,8 +87,8 @@ module Ironfan
         #
         # Discovery
         #
-        def self.load!(cluster=nil)
-          query = cluster && "name:#{cluster.name}-*"
+        def self.load!(cluster = nil)
+          query = cluster && "name:#{cluster.realm_name}-*"
           ChefServer.search(:node, query) do |raw|
             next unless raw.present?
             node = register(raw)

data/lib/ironfan/provider/ec2/machine.rb

@@ -48,7 +48,7 @@ module Ironfan
         def groups           ; Array(adaptee.groups)   ;   end
         def public_hostname  ; dns_name ; end
         def keypair          ; key_pair ; end
-
+        
         def created?
           not ['terminated', 'shutting-down'].include? state
         end
@@ -141,7 +141,7 @@ module Ironfan
         def self.validate_resources!(computers)
           recall.each_value do |machine|
             next unless machine.users.empty? and machine.name
-            if machine.name.match("^#{computers.cluster.name}-")
+            if computers.clusters.any?{ |comp| machine.name.match("^#{comp.name}-") }
               machine.bogus << :unexpected_machine
             end
             next unless machine.bogus?
@@ -230,15 +230,7 @@ module Ironfan
 
         def self.launch_description(computer)
           cloud = computer.server.cloud(:ec2)
-          user_data_hsh =               {
-            :chef_server =>             Chef::Config[:chef_server_url],
-            :node_name =>               computer.name,
-            :organization =>            Chef::Config[:organization],
-            :cluster_name =>            computer.server.cluster_name,
-            :facet_name =>              computer.server.facet_name,
-            :facet_index =>             computer.server.index,
-            :client_key =>              computer.private_key
-          }
+          user_data = self.cloud_init_user_data(computer)
 
           # main machine info
           # note that Fog does not actually create tags when it creates a
@@ -249,16 +241,20 @@ module Ironfan
             :vpc_id               => cloud.vpc,
             :subnet_id            => cloud.subnet,
             :key_name             => cloud.ssh_key_name(computer),
-            :user_data            => JSON.pretty_generate(user_data_hsh),
+            :user_data            => user_data,
             :block_device_mapping => block_device_mapping(computer),
             :availability_zone    => cloud.default_availability_zone,
             :monitoring           => cloud.monitoring,
           }
 
           # VPC security_groups can only be addressed by id (not name)
-          description[:security_group_ids] = cloud.security_groups.keys.map do |g|
-            SecurityGroup.recall( SecurityGroup.group_name_with_vpc(g,cloud.vpc) ).group_id
+          sec_group_ids = []
+          [computer.server.security_groups, cloud.security_groups].each do |container|
+            sec_group_ids += container.keys.map do |g|
+              SecurityGroup.recall( SecurityGroup.group_name_with_vpc(g,cloud.vpc) ).group_id
+            end
           end
+          description[:security_group_ids] = sec_group_ids.uniq
 
           description[:iam_server_certificates] = cloud.iam_server_certificates.values.map do |cert|
             IamServerCertificate.recall(IamServerCertificate.full_name(computer, cert))