ironfan 4.12.3 → 5.0.0

data/lib/ironfan/dsl/server.rb

@@ -1,6 +1,218 @@
 module Ironfan
   class Dsl
 
+    class MachineManifest
+      include Gorillib::Model
+
+      # base server fields
+      field :environment, Symbol
+      field :name, String
+      field :cluster_name, String
+      field :facet_name, String
+      field :components, Array, of: Ironfan::Dsl::Component, default: []
+      field :run_list, Array, of: String, default: []
+      field :cluster_default_attributes, Hash
+      field :cluster_override_attributes, Hash
+      field :facet_default_attributes, Hash
+      field :facet_override_attributes, Hash
+
+      # cloud fields
+      field :cloud_name,                String
+      field :availability_zones,        Array, default: []
+      field :backing,                   String
+      field :ebs_optimized,             :boolean
+      field :flavor,                    String
+      field :image_id,                  String
+      field :placement_group,           String
+      field :elastic_ip,                String
+      field :auto_elastic_ip,           String
+      field :allocation_id,             String
+      field :region,                    String
+      field :ssh_user,                  String
+      field :subnet,                    String
+      field :vpc,                       String
+
+      #-----------------------------------------------------------------------------------
+      # # FIXME: I haven't determined how to pull some of these fields
+      # #        in from the remote machines. In fact, some of these
+      # #        will have to be omitted when comparing. Since
+      # #        they'll only be necessary when we refactor the
+      # #        backend to accept manifests for launch, I'm going
+      # #        to leave these commented out for now. --josh
+      #
+      # # base server fields
+      #
+      # field :volumes, Array, of: Volume
+      #
+      # # cloud fields 
+      #
+      # field :bits,                      Integer
+      # field :bootstrap_distro,          String
+      # field :chef_client_script,        String
+      # field :default_availability_zone, String
+      # field :elastic_load_balancers,    Array, of: Ironfan::Dsl::Ec2::ElasticLoadBalancer, default: []
+      # field :iam_server_certificates,   Array, of: Ironfan::Dsl::Ec2::IamServerCertificate, default: []
+      # field :image_name,                String
+      # field :keypair,                   String
+      # field :monitoring,                String
+      # field :mount_ephemerals,          Hash
+      # field :permanent,                 :boolean
+      # field :provider,                  Whatever
+      # field :security_groups,           Array, of: Ironfan::Dsl::Ec2::SecurityGroup
+      # field :ssh_identity_dir,          String
+      # field :validation_key,            String
+      #-----------------------------------------------------------------------------------
+
+      # Reconstruct machine manifest from a computer, pulling
+      # information from remote sources as necessary.
+      def self.from_computer(computer)
+        node = get_node(computer.name)
+        cluster_name = node['cluster_name']
+        facet_name = node['facet_name']
+        instance = node['facet_index']
+
+        from_remote(
+                    cluster_name,
+                    facet_name,
+                    instance,
+                    node,
+                    computer.machine,
+                    computer.server.clouds.to_a.first,
+                    get_role("#{cluster_name}-cluster"),
+                    get_role("#{cluster_name}-#{facet_name}-facet")
+                  )
+      end
+
+      def self.from_remote(cluster_name,
+                           facet_name,
+                           instance,
+                           node,
+                           machine,
+                           cloud,
+                           cluster_role,
+                           facet_role)
+        machine = NilCheckDelegate.new(machine)
+        cloud = NilCheckDelegate.new(cloud)
+        cluster_role = NilCheckDelegate.new(cluster_role)
+        facet_role = NilCheckDelegate.new(facet_role)
+
+        result = Ironfan::Dsl::MachineManifest.
+          receive(
+
+                  # base server fields
+
+                  environment: node.chef_environment,
+                  name: instance,
+                  cluster_name: cluster_name,
+                  facet_name: facet_name,
+                  components: remote_components(node),
+                  run_list: remote_run_list(node),
+                  cluster_default_attributes: (cluster_role.default_attributes || {}),
+                  cluster_override_attributes: (cluster_role.override_attributes || {}),
+                  facet_default_attributes: (facet_role.default_attributes || {}),
+                  facet_override_attributes: (facet_role.override_attributes || {}),
+
+                  # cloud fields
+
+                  backing: machine.root_device_type,
+                  cloud_name: cloud.name,
+                  availability_zones: [*machine.availability_zone],
+                  ebs_optimized: machine.ebs_optimized,
+                  flavor: machine.flavor_id,
+                  image_id: machine.image_id,
+                  keypair: machine.nilcheck_depth(1).key_pair.name,
+                  monitoring: machine.monitoring,
+                  placement_group: machine.placement_group,
+                  region: machine.availability_zone.to_s[/.*-.*-\d+/],
+                  security_groups: machine.nilcheck_depth(1).groups.map{|x| {name: x}},
+                  subnet: machine.subnet_id,
+                  vpc: machine.vpc_id
+
+                  #-----------------------------------------------------------------------------------
+                  # # FIXME: I haven't determined how to pull some of these fields
+                  # #        in from the remote machines. In fact, some of these
+                  # #        will have to be omitted when comparing. Since
+                  # #        they'll only be necessary when we refactor the
+                  # #        backend to accept manifests for launch, I'm going
+                  # #        to leave these commented out for now. --josh
+                  #
+                  # # base server fields
+                  #
+                  # volume: local_manifest.volume
+                  #
+                  # # cloud fields 
+                  #
+                  # bits: local_manifest.bits,
+                  # bootstrap_distro: local_manifest.bootstrap_distro,
+                  # chef_client_script: local_manifest.chef_client_script,                  
+                  # default_availability_zone: local_manifest.default_availability_zone,
+                  # iam_server_certificates: launch_description.fetch(:iam_server_certificates),
+                  # image_name: local_manifest.image_name,
+                  # elastic_load_balancers: launch_description.fetch(:elastic_load_balancers),
+                  # mount_ephemerals: local_manifest.mount_ephemerals,
+                  # permanent: local_manifest.permanent,
+                  # provider: local_manifest.provider,
+                  # elastic_ip: local_manifest.elastic_ip,
+                  # auto_elastic_ip: local_manifest.auto_elastic_ip,
+                  # allocation_id: local_manifest.allocation_id,
+                  # ssh_user: local_manifest.ssh_user,
+                  # ssh_identity_dir: local_manifest.ssh_identity_dir,
+                  # validation_key: local_manifest.validation_key,
+                  #-----------------------------------------------------------------------------------
+                  )
+      end
+
+      def to_comparable
+        deep_stringify(to_wire.tap do |hsh|
+                         hsh.delete(:_type)
+                         hsh.delete(:ssh_user)
+                         #hsh[:security_groups] = Hash[hsh[:security_groups].map{|x| [x.fetch(:name), x]}]
+                         hsh[:components] = Hash[hsh.fetch(:components).map do |component|
+                                                   [component.fetch(:name), component]
+                                                 end]
+                         hsh[:run_list] = hsh.fetch(:run_list).map do |x|
+                           x.end_with?(']') ? x : "recipe[#{x}]"
+                         end
+                       end)
+      end
+
+      private
+
+      def deep_stringify obj
+        case obj
+        when Hash then Hash[obj.map{|k,v| [k.to_s, deep_stringify(v)]}]
+        when Array then obj.map{|x| deep_stringify(x)}
+        when Symbol then obj.to_s
+        else obj
+        end
+      end
+
+      def self.get_node(node_name)
+        Chef::Node.load(node_name)
+      rescue Net::HTTPServerException => ex
+        Chef::Node.new
+      end
+
+      def self.get_role(role_name)
+        Chef::Role.load(role_name)
+      rescue Net::HTTPServerException => ex
+        Chef::Role.new
+      end
+
+      def self.remote_components(node)
+        announcements = node['components'] || {}
+        node['components'].to_a.map do |_, announce|
+          name = announce['name'].to_sym
+          plugin = Ironfan::Dsl::Compute.plugin_for(name)
+          plugin.from_node(node).tap{|x| x.name = name} if plugin
+        end.compact
+      end
+
+      def self.remote_run_list(node)
+        node.run_list.to_a.map(&:to_s)
+      end
+    end
+
     class Server < Ironfan::Dsl::Compute
       field      :cluster_name, String
       field      :facet_name,   String
@@ -38,6 +250,81 @@ module Ironfan
         errors['missing cluster/facet/server'] = [cluster_name, facet_name, name] unless (cluster_name && facet_name && name)
         errors
       end
+
+      def to_machine_manifest
+        cloud = clouds.each.to_a.first
+        MachineManifest.receive(
+
+                                # base server fields
+
+                                environment: environment,
+                                name: name,
+                                cluster_name: cluster_name,
+                                facet_name: facet_name,
+                                run_list: run_list,
+                                components: components,
+                                cluster_default_attributes: cluster_role.default_attributes,
+                                cluster_override_attributes: cluster_role.override_attributes,
+                                facet_default_attributes: facet_role.default_attributes,
+                                facet_override_attributes: facet_role.override_attributes,
+                                volumes: volumes,
+
+                                # cloud fields
+
+                                cloud_name: cloud.name,
+
+                                availability_zones: cloud.availability_zones,
+                                backing: cloud.backing,
+                                bits: cloud.bits,
+                                bootstrap_distro: cloud.bootstrap_distro,
+                                chef_client_script: cloud.chef_client_script,
+                                default_availability_zone: cloud.default_availability_zone,
+                                elastic_load_balancers: cloud.elastic_load_balancers,
+                                ebs_optimized: cloud.ebs_optimized,
+                                flavor: cloud.flavor,
+                                iam_server_certificates: cloud.iam_server_certificates,
+                                image_id: cloud.image_id,
+                                image_name: cloud.image_name,
+                                keypair: cloud.keypair,
+                                monitoring: cloud.monitoring,
+                                mount_ephemerals: cloud.mount_ephemerals,
+                                permanent: cloud.permanent,
+                                placement_group: cloud.placement_group,
+                                provider: cloud.provider,
+                                elastic_ip: cloud.elastic_ip,
+                                auto_elastic_ip: cloud.auto_elastic_ip,
+                                allocation_id: cloud.allocation_id,
+                                region: cloud.region,
+                                security_groups: cloud.security_groups,
+                                ssh_user: cloud.ssh_user,
+                                ssh_identity_dir: cloud.ssh_identity_dir,
+                                subnet: cloud.subnet,
+                                validation_key: cloud.validation_key,
+                                vpc: cloud.vpc
+
+                                )
+      end
+
+      def canonical_machine_manifest_hash
+        self.class.canonicalize(to_machine_manifest)
+      end
+
+      private
+
+      def self.canonicalize(item)
+        case item
+        when Array, Gorillib::ModelCollection
+          item.each.map{|i| canonicalize(i)}
+        when Ironfan::Dsl::Component
+          canonicalize(item.to_manifest)
+        when Gorillib::Builder, Gorillib::Model
+          canonicalize(item.to_wire.tap{|x| x.delete(:_type)})
+        when Hash then
+          Hash[item.sort.map{|k,v| [k, canonicalize(v)]}]
+        else
+          item
+        end
+      end
     end
 
   end

data/lib/ironfan/dsl/volume.rb

@@ -28,6 +28,7 @@ module Ironfan
       VOLUME_IDS.merge!({
                             :blank_xfs             => 'snap-d9c1edb1',
                             :blank_xfs_tokyo       => 'snap-049d1921',
+                            :blank_xfs_california  => 'snap-514b5c5a', # us-west-1
                         })
 
       def snapshot_id(*)

data/lib/ironfan/dsl.rb

@@ -2,6 +2,138 @@ module Ironfan
 
   class Dsl < Builder
     include Gorillib::Resolution
-  end
 
+    def self.default_cookbook_reqs
+      @default_cookbook_reqs ||= []
+    end
+
+    def self.cookbook_req name, constraint
+      default_cookbook_reqs << new_req(name, constraint)
+        
+    end
+
+    def join_req req1, req2
+      # order requirements by operation: =, >=, ~>
+      req1, req2 = (req1.constraint < req2.constraint) ? [req1, req2] : [req2, req1]
+      cn1, cn2 = [req1.constraint, req2.constraint]
+      vers1, vers2 = [cn1.split.last, cn2.split.last]
+      vers1_c, vers2_c = [vers1.split('.'), vers2.split('.')]
+      op1, op2 = [req1, req2].map{|x| x.constraint.split.first}
+
+      if op1 == '=' and op2 == '='
+        join_eq_eq(req1, req2)
+      elsif op1 == '>=' and op2 == '>='
+        join_geq_geq(req1, req2)
+      elsif op1 == '~>' and op2 == '~>'
+        join_agt_agt(req1, req2)
+      elsif op1 == '=' and op2 == '>='
+        join_eq_gte(req1, req2)
+      elsif op1 == '=' and op2 == '~>'
+        join_eq_agt(req1, req2)
+      elsif op1 == '>=' and op2 == '~>'
+        join_gte_agt(req1, req2)
+      end
+    end
+
+    def cookbook_req name, constraint
+      (@cookbook_reqs ||= []) << self.class.new_req(name, constraint)
+    end
+
+    def children() [] end
+
+    def cookbook_reqs
+      Hash[_cookbook_reqs.map{|x| [x.name, x.constraint]}]
+    end
+
+    def _cookbook_reqs
+      [
+       *shallow_cookbook_reqs,
+         *child_cookbook_reqs
+      ].group_by(&:name).values.map do |group|
+        group.inject{|result, req| join_req(result, req)}
+      end
+    end
+
+    private
+
+    #-----------------------------------------------------------------------------------------------
+
+    def join_eq_eq(req1, req2)
+      (vers(req1) == vers(req2)) ? req1 : bad_reqs(req1, req2)
+    end
+
+    def join_geq_geq(req1, req2)
+      (vers(req1) >= vers(req2)) ? req1 : req2
+    end
+
+    def join_agt_agt(req1, req2)
+      if vers_a(req1).size == vers_a(req2).size and
+          vers_a_head(req1).zip(vers_a_head(req2)).all?{|v1,v2| v1 == v2}
+        (req1.constraint > req2.constraint) ? req1 : req2
+      else
+        bad_reqs(req1, req2)
+      end
+    end
+
+    def join_eq_gte(req1, req2)
+      vers(req1) >= vers(req2) ? req1 : bad_reqs(req1, req2)
+    end
+
+    def join_eq_agt(req1, req2)
+      if match_v_head(req1, req2) and
+          vers_a(req1)[vers_a(req2).size - 1] >= vers_a(req2).last
+        req1
+      else
+        bad_reqs(req1, req2)
+      end
+    end
+
+    def join_gte_agt(req1, req2)
+      if match_v_head(req1, req2) and vers(req1) <= vers(req2)
+        req2
+      else
+        bad_reqs(req1, req2)
+      end
+    end
+
+    def match_v_head(req1, req2)
+      vers_a_head(req1, vers_a(req2).size).zip(vers_a_head(req2)).all?{|v1,v2| v1 == v2}      
+    end
+
+    def op(req)
+      req.constraint.split.first
+    end
+
+    def vers(req)
+      req.constraint.split.last
+    end
+
+    def vers_a(req)
+      vers(req).split('.')
+    end
+
+    def vers_a_head(req, last = 0)
+      vers_a(req)[0...last-1]
+    end
+
+    #-----------------------------------------------------------------------------------------------
+
+    def bad_reqs(req1, req2)
+      raise ArgumentError.new("#{req1.name}: cannot reconcile #{req1.constraint} with #{req2.constraint}")
+    end
+
+    def child_cookbook_reqs
+      children.map(&:_cookbook_reqs).flatten(1)
+    end
+
+    def self.new_req(name, constraint)
+      raise StandardError.new("Please don't use >= constraints. They're too vague!") if
+        constraint.start_with?('>=') and not (@@testing ||= false)
+      Ironfan::Plugin::CookbookRequirement.new(name: name, constraint: constraint)
+    end
+
+    def shallow_cookbook_reqs
+      @cookbook_reqs || self.class.default_cookbook_reqs
+    end
+  end
 end

data/lib/ironfan/headers.rb

@@ -15,9 +15,11 @@ module Ironfan
   end
 
   class Dsl < Builder
+    class Component < Ironfan::Dsl; end
     class Compute < Ironfan::Dsl; end
     class Cluster < Ironfan::Dsl::Compute; end
     class Facet < Ironfan::Dsl::Compute; end
+    class Realm < Ironfan::Dsl::Compute; end
     class Server < Ironfan::Dsl::Compute; end
 
     class Role < Ironfan::Dsl; end
@@ -36,6 +38,11 @@ module Ironfan
     end
   end
 
+  module Plugin
+    class CookbookRequirement; end
+    module Base; end
+  end
+
   class Provider < Builder
     class Resource < Builder; end
   end

data/lib/ironfan/plugin/base.rb

@@ -0,0 +1,89 @@
+require 'gorillib/model'
+require 'gorillib/builder'
+require 'gorillib/string/inflections'
+require 'gorillib/metaprogramming/concern'
+
+Gorillib::Model::Field.class_eval do
+  field :node_attr, String, default: nil
+end
+
+module Ironfan
+
+  module Pluggable
+    def add_plugin name, cls
+      registry[name] = cls
+    end
+    def plugin_for name
+      registry[name]
+    end
+    def registry() @registry ||= {}; end
+  end
+
+  module Plugin
+    class CookbookRequirement
+      include Gorillib::Builder
+
+      magic :name, String
+      magic :constraint, String
+
+      def <=>(other)
+        self.name <=> other.name
+      end
+    end
+
+    module Base
+      extend Gorillib::Concern
+
+      def to_node
+        Chef::Node.new.tap do |node|
+          self.class.fields.select{|_,x| x.node_attr}.each do |_,x|
+            val = send(x.name)
+            (keys = x.node_attr.split('.'))[0...-1].inject(node.set) do |hsh,key|
+              hsh[key]
+            end[keys.last] = val unless val.nil?
+          end
+        end
+      end
+
+      module ClassMethods
+        attr_reader :cookbook_reqs
+        attr_reader :plugin_name
+
+        def from_node(node = NilCheckDelegate.new(nil))
+          new(Hash[
+                   fields.select{|_,x| x.node_attr}.map do |_,x|
+                     [x.name, (x.node_attr.split('.').inject(node) do |hsh,attr|
+                                 if hsh
+                                   (val = hsh[attr]).is_a?(Mash) ? val.to_hash : val
+                                 end
+                               end)]
+                   end.reject{|_,v| v.nil?}
+                  ])
+        end
+
+        def register_with cls, &blk
+          (@dest_class = cls).class_eval{ extend Ironfan::Pluggable }
+        end
+
+        def template plugin_name_parts, base_class=self, &blk
+          plugin_name_parts = [*plugin_name_parts]
+          full_name = plugin_name_parts.map(&:to_s).join('_').to_sym
+          plugin_name = plugin_name_parts.first.to_sym
+
+          Class.new(base_class, &blk).tap do |plugin_class|
+            plugin_class.class_eval{ @plugin_name = plugin_name }
+
+            self.const_set(full_name.to_s.camelize.to_sym, plugin_class)
+
+            @dest_class.class_eval do
+              add_plugin(full_name, plugin_class)
+              define_method(full_name) do |*args, &blk|
+                plugin_class.plugin_hook self, (args.first || {}), plugin_name, full_name, &blk
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ironfan/provider/ec2/security_group.rb

@@ -69,6 +69,8 @@ module Ironfan
           groups_to_create         = [ ]
           authorizations_to_ensure = [ ]
 
+          computers.each{|comp| ensure_groups(comp) if Ec2.applicable(comp) } # Add facet and cluster security groups for the computer
+
           # First, deduce the list of all groups to which at least one instance belongs
           # We'll use this later to decide whether to create groups, or authorize access,
           # using a VPC security group or an EC2 security group.
@@ -76,7 +78,6 @@ module Ironfan
           groups_to_create << groups_that_should_exist
 
           computers.select { |computer| Ec2.applicable computer }.each do |computer|
-            ensure_groups(computer) # Add facet and cluster security groups for the computer
             cloud           = computer.server.cloud(:ec2)
             cluster_name    = computer.server.cluster_name
 

data/lib/ironfan/requirements.rb

@@ -2,13 +2,19 @@
 require 'gorillib/builder'
 require 'gorillib/resolution'
 
+require 'gorillib/nil_check_delegate'
+
 # Pre-declaration of class hierarchy
 require 'ironfan/headers'
 
+# Ironfan plugin mixin
+require 'ironfan/plugin/base'
+
 # DSL for cluster descriptions
 require 'ironfan/dsl'
 require 'ironfan/builder'
 
+require 'ironfan/dsl/component'
 require 'ironfan/dsl/compute'
 require 'ironfan/dsl/server'
 require 'ironfan/dsl/facet'

data/lib/ironfan.rb

@@ -91,6 +91,18 @@ module Ironfan
     end
   end
 
+  def self.load_realm(name)
+    name = name.to_sym
+    raise ArgumentError, "Please supply a realm name" if name.to_s.empty?
+    return @@realms[name] if @@realms[name]
+
+    load_cluster_files
+
+    unless @@realms[name] then die("Couldn't find a realm definition for #{name} in #{cluster_path}") end
+
+    @@realms[name]
+  end
+
   #
   # Return cluster if it's defined. Otherwise, search Ironfan.cluster_path
   # for an eponymous file, load it, and return the cluster it defines.
@@ -104,16 +116,21 @@ module Ironfan
     raise ArgumentError, "Please supply a cluster name" if name.to_s.empty?
     return @@clusters[name] if @@clusters[name]
 
+    load_cluster_files
+
+    unless @@clusters[name] then die("Couldn't find a cluster definition for #{name} in #{cluster_path}") end
+
+    @@clusters[name]
+  end
+
+  def self.load_cluster_files
     cluster_path.each do |cp_dir|
       Dir[ File.join(cp_dir, '*.rb') ].each do |filename|
         Chef::Log.info("Loading cluster file #{filename}")
         require filename
+        clusters.values.each{|cluster| cluster.source_file ||= filename}
       end
     end
-
-    unless @@clusters[name] then  die("Couldn't find a cluster definition for #{name} in #{cluster_path}") end
-
-    @@clusters[name]
   end
 
   #