ironfan 4.12.3 → 5.0.0

data/lib/gorillib/diff.rb

@@ -0,0 +1,266 @@
+require 'diff-lcs'
+
+module Gorillib
+  module AnsiColors
+    def cyan str
+      "\e[36m#{str}\e[0m"
+    end
+  end
+
+  module DiffDisplayTools
+    def display_key_header key
+      indent; @stream.puts("#{key}:")
+    end
+
+    #-------------------------------------------------------------------------------------------------
+
+    def only_left_key key
+      indent; @stream.puts("only on #{b_left}")
+    end
+
+    def only_right_key key
+      indent; @stream.puts("only on #{b_right}")
+    end
+
+    #-------------------------------------------------------------------------------------------------
+
+    def display_hetero this, other
+      indent; @stream.puts("#{b_left} class is #{this.class} but #{b_right} class is #{other.class}")
+    end
+
+    #-------------------------------------------------------------------------------------------------
+
+    def display_noteql_atoms this, other
+      indent; @stream.puts("#{b_left}: #{this.inspect}")
+      indent; @stream.puts("#{b_right}: #{other.inspect}")
+    end
+
+    #-------------------------------------------------------------------------------------------------
+
+    def display_eql(it)
+      indent; @stream.puts("#{'='} #{it}")
+    end
+
+    def display_add(itl, itr)
+      indent
+      if itl.nil?
+        @stream.puts("#{b_right}: #{itr}")
+      else
+        @stream.puts("#{b_left}: #{itl}")
+      end
+    end
+
+    #-------------------------------------------------------------------------------------------------
+
+    def indent
+      @stream.write(" " * @indentation)
+    end
+
+    def display_indices(ixl, ixr)
+      indent; @stream.puts("#{b_left} ix: #{ixl}, #{b_right} ix: #{ixr}")
+    end
+
+    #------------------------------------------------------------------------------------------------
+
+    def b_left
+      "[#{header(@left)}]"
+    end
+
+    def b_right
+      "[#{header(@right)}]"
+    end
+
+    def decrease_indentation
+      @indentation -= @tab_width
+    end
+
+    def header str
+      cyan(str)
+    end
+
+    def increase_indentation
+      @indentation += @tab_width
+    end
+  end
+
+  module DiffDrawerMethods
+    def for_subhash(key)
+      display_key_header(key)
+      increase_indentation
+      yield
+      decrease_indentation
+    end
+
+    #-------------------------------------------------------------------------------------------------
+
+    def display_diff_indices diff_subset
+      return if diff_subset.nil? or diff_subset.empty?
+      diff_subset.first.tap do |cchange|
+        display_indices(cchange.old_position, cchange.new_position)
+      end
+    end
+
+    def display_eql_items diff_subset
+      return if diff_subset.nil?
+      diff_subset.each do |cchange|
+        display_eql(cchange.old_element)
+      end
+    end
+
+    def display_noteql_items diff_subset
+      return if diff_subset.nil?
+      diff_subset.each do |cchange|
+        display_noteql_atoms(cchange.old_element, cchange.new_element)
+      end
+    end
+
+    def display_add_items diff_subset
+      return if diff_subset.nil?
+      diff_subset.each do |cchange|
+        display_add(cchange.old_element, cchange.new_element)
+      end
+    end
+  end
+
+  class DiffDrawer
+    include AnsiColors
+    include DiffDisplayTools
+    include DiffDrawerMethods
+
+    def initialize options = {}
+      @stream = options[:stream] || $stdout
+      @left = options[:left] || 'left'
+      @right = options[:right] || 'right'
+      @indentation = options[:indentation] || 0
+      @tab_width = options[:tab_width] || 4
+    end
+  end
+
+  class DiffFormatter
+    def initialize(options = {})
+      @drawer = (options[:drawer] ||
+                 DiffDrawer.new(stream: options.delete(:stream),
+                                left: options.delete(:left),
+                                right: options.delete(:right),
+                                indentation: options.delete(:indentation),
+                                tab_width: options.delete(:tab_width)))
+
+      @context_atoms = options[:context_atoms] || 0
+    end
+
+    def display_diff(this, other)
+      setup
+      if this.is_a?(Hash) && other.is_a?(Hash)
+        display_diff_hash(this, other)
+      elsif this.is_a?(Array) && other.is_a?(Array)
+        display_diff_arr(this, other)
+      elsif ((this.is_a?(Hash) != other.is_a?(Hash)) ||
+             (this.is_a?(Array) != other.is_a?(Array)))
+        display_diff_hetero(this, other)
+      else
+        display_diff_atom(this, other)
+      end
+    end
+
+    private
+
+    #-------------------------------------------------------------------------------------------------
+
+    def setup
+      @display_last_suffix = display_last_suffix_p
+      @display_this_prefix = nop_p
+    end
+
+    #-------------------------------------------------------------------------------------------------
+
+    def display_diff_hash(this, other)
+      (this.keys & other.keys).each do |key|
+        if this[key] != other[key]
+          @drawer.for_subhash(key){ display_diff(this[key], other[key]) }
+        end
+      end
+      (this.keys - other.keys).each do |key|
+        @drawer.for_subhash(key){ @drawer.only_left_key(key) }
+      end
+      (other.keys - this.keys).each do |key|
+        @drawer.for_subhash(key){ @drawer.only_right_key(key) }
+      end
+    end
+
+    def display_diff_hetero(this, other)
+      @drawer.display_hetero(this, other)
+    end
+
+    def display_diff_atom(this, other)
+      if this != other
+        @drawer.display_noteql_atoms(this, other)
+      end
+    end
+
+    def display_diff_arr(this, other)
+      Diff::LCS.sdiff(this, other).chunk{|x| type_of(x)}.each do |type, diff_atoms|
+        case type
+        when '=' then next_step(diff_atoms, type)
+        when '+' then next_step(diff_atoms, type){ @drawer.display_add_items(diff_atoms) }
+        when '!' then next_step(diff_atoms, type){ @drawer.display_noteql_items(diff_atoms) }
+        end
+      end
+    end
+
+    #-------------------------------------------------------------------------------------------------
+    
+    def next_step(diff_atoms, type)
+      if type == '='
+        @display_this_prefix.call diff_atoms
+        yield(diff_atoms) if block_given?
+        @display_last_suffix = display_last_suffix_p diff_atoms
+        @display_this_prefix = nop_p
+      else
+        @display_last_suffix.call
+        yield(diff_atoms) if block_given?
+        @display_last_suffix = nop_p
+        @display_this_prefix = display_this_prefix_p
+      end
+    end
+
+    #-------------------------------------------------------------------------------------------------
+
+    def display_this_prefix_p()
+      ->(diff_subset) do
+        diff_subset = diff_subset[0...@context_atoms]
+        @drawer.display_eql_items(diff_subset)
+      end
+    end
+
+    def display_last_suffix_p(diff_subset = [])
+      ->() do
+        to_display = diff_subset[[0, (diff_subset.size - @context_atoms)].max..-1]
+        if diff_subset.empty?
+          @drawer.display_indices(0,0)
+        elsif to_display.empty?
+          @drawer.
+            display_indices(diff_subset.first.old_position + diff_subset.map(&:old_element).compact.size,
+                            diff_subset.first.new_position + diff_subset.map(&:new_element).compact.size)
+        else
+          @drawer.display_indices(to_display.first.old_position, to_display.first.new_position)
+        end
+
+        @drawer.display_eql_items(to_display)
+      end
+    end
+
+    def nop_p() ->(*_) {} end
+
+    #-------------------------------------------------------------------------------------------------
+
+    def type_of(diff_atom)
+      if diff_atom.old_element == diff_atom.new_element
+        '='
+      elsif diff_atom.old_element.nil? || diff_atom.new_element.nil?
+        '+'
+      else
+        '!'
+      end
+    end
+  end
+end

data/lib/gorillib/nil_check_delegate.rb

@@ -0,0 +1,30 @@
+require 'delegate'
+
+class NilCheckDelegate < Delegator
+  def initialize(obj)
+    super
+    @delegate_sd_obj = obj
+    @depth = 0
+  end
+
+  def nilcheck_depth(depth = 0)
+    @depth = depth
+    self
+  end
+
+  def _wrap(obj)
+    ((@depth -= 1) >= 0) ? self.class.new(obj) : obj
+  end
+
+  def method_missing m, *args, &blk
+    _wrap(__getobj__.nil? ? nil : super(m, *args, &blk))
+  end
+
+  def __getobj__
+    @delegate_sd_obj
+  end
+
+  def __setobj__(obj)
+    @delegate_sd_obj = obj
+  end
+end

data/lib/ironfan/broker/computer.rb

@@ -76,6 +76,9 @@ module Ironfan
         ensure_dependencies
         iaas_provider.machine_class.create! self
         node.announce_state :started
+        server.to_machine_manifest.components.each do |component|
+          component.announce_to(node)
+        end
         save
         self
       end

data/lib/ironfan/dsl/cluster.rb

@@ -3,6 +3,11 @@ module Ironfan
 
     class Cluster < Ironfan::Dsl::Compute
       collection :facets,       Ironfan::Dsl::Facet,   :resolver => :deep_resolve
+      include Ironfan::Plugin::Base; register_with Ironfan::Dsl::Realm
+
+      def children
+        facets.to_a + components.to_a
+      end
 
       def initialize(attrs={},&block)
         super
@@ -21,6 +26,11 @@ module Ironfan
       def cluster_name
         name
       end
+
+      def self.plugin_hook owner, attrs, plugin_name, full_name, &blk
+        owner.cluster(plugin_name, new(attrs.merge(name: full_name, owner: owner)))
+        _project cluster, &blk
+      end
     end
   end
 end

data/lib/ironfan/dsl/component.rb

@@ -0,0 +1,157 @@
+module Ironfan
+  class Dsl
+    class Component < Ironfan::Dsl
+      include Gorillib::Builder
+      include Gorillib::Concern
+      include Ironfan::Plugin::Base; register_with Ironfan::Dsl::Compute
+
+      field :cluster_name, Symbol
+      field :facet_name, Symbol
+      field :realm_name, Symbol
+      field :name, Symbol
+
+      def initialize(attrs, &blk)
+        attrs.merge!(facet_name: (attrs[:owner].name unless attrs[:owner].nil? or not attrs[:owner].is_a?(Facet)),
+                     cluster_name: (attrs[:owner].cluster_name unless attrs[:owner].nil?),
+                     realm_name: (attrs[:owner].realm_name unless attrs[:owner].nil?))
+        super attrs, &blk
+      end
+
+      def self.plugin_hook owner, attrs, plugin_name, full_name, &blk
+        (this = new(attrs.merge(owner: owner, name: full_name), &blk))._project(owner)
+        this
+      end
+
+      def announce_to node
+        node.set['components']["#{cluster_name}-#{name}"]['name'] = name
+      end
+
+      def self.to_node
+        super.tap do |node|
+          node.set['cluster_name'] = cluster_name
+        end
+      end
+
+      def self.from_node(node = NilCheckDelegate.new(nil))
+        cluster_name = node['cluster_name'].to_s
+        super(node).tap{|x| x.receive!(cluster_name: cluster_name,
+                                       realm_name: cluster_name.split('_').first)}
+      end
+
+      def self.announce_name
+        plugin_name
+      end
+
+      def announce_name
+        self.class.announce_name
+      end
+
+      def _project(compute)
+        compute.component name, self
+        project(compute)
+      end
+
+      def realm_announcements
+        (@@realm_announcements ||= {})
+      end
+      
+      def realm_subscriptions component_name
+        (@@realm_subscriptions ||= {})[component_name] ||= []
+      end
+
+      def announce(component_name)
+        Chef::Log.debug("announced #{announce_name} for #{cluster_name}")
+        realm_announcements[[realm_name, component_name]] = [cluster_name, facet_name]
+        realm_subscriptions(component_name).each{|blk| blk.call(cluster_name, facet_name)}
+      end
+
+      def discover(component_name, &blk)
+        if already_announced = realm_announcements[[realm_name, component_name]]
+          yield *already_announced
+        else
+          Chef::Log.debug("#{cluster_name}: no one announced #{announce_name}. subscribing")
+          realm_subscriptions(component_name) << blk
+        end
+      end
+    end
+
+    module Discovery
+      include Gorillib::Builder
+      extend Gorillib::Concern
+
+      magic :server_cluster, Symbol
+      magic :bidirectional, :boolean, default: false
+
+      (@_dependencies ||= []) << Gorillib::Builder
+
+      module ClassMethods
+        def default_to_bidirectional default=true
+          fields[:bidirectional].default = default
+        end
+      end
+
+      def set_discovery compute, keys
+        if server_cluster
+          wire_to(compute, full_server_cluster, facet_name, keys)
+        else
+          discover(announce_name) do |cluster_name, facet_name|
+            wire_to(compute, cluster_name, facet_name, keys)
+          end
+        end
+      end
+
+      def wire_to(compute, cluster_name, facet_name, keys)
+        discovery = {discovers: keys.reverse.inject(cluster_name){|hsh,key| {key => hsh}}}
+        (compute.facet_role || compute.cluster_role).override_attributes(discovery)
+
+        # FIXME: This is Ec2-specific and probably doesn't belong here.
+        client_group_v = client_group(compute)
+        server_group_v = server_group(cluster_name, facet_name)
+        
+        group_edge(compute.cloud(:ec2), client_group_v, server_group_v)
+        group_edge(compute.cloud(:ec2), server_group_v, client_group_v) if bidirectional
+
+        Chef::Log.debug("discovered #{announce_name} for #{cluster_name}: #{discovery}")
+      end
+
+      protected
+
+      def client_group(compute)
+        security_group(compute.cluster_name, (compute.name if compute.is_a?(Facet)))
+      end
+      
+      def full_server_cluster
+        "#{realm_name}_#{server_cluster}"
+      end
+
+      def group_edge(cloud, group_1, group_2)
+        cloud.security_group(group_1).authorize_group(group_2)
+        Chef::Log.debug("component.rb: allowing access from security group #{group_1} to #{group_2}")
+      end
+
+      def security_group(cluster_name, facet_name)
+        facet_name ? [cluster_name, facet_name].join('-') : cluster_name
+      end
+
+      def server_group(cluster_name, facet_name)
+        security_group(cluster_name, facet_name)
+      end
+    end
+
+    module Announcement
+      include Gorillib::Builder
+
+      def _project(compute)
+        announce announce_name
+        super compute
+      end
+    end
+    def to_manifest
+      to_wire.reject{|k,_| _skip_fields.include? k}
+    end
+
+    def _skip_fields() skip_fields << :_type; end
+
+    def skip_fields() [] end
+  end
+end

data/lib/ironfan/dsl/compute.rb

@@ -1,3 +1,5 @@
+require 'gorillib/metaprogramming/concern'
+
 module Ironfan
   class Dsl
 
@@ -16,6 +18,7 @@ module Ironfan
       field      :name,         String
 
       # Resolve each of the following as a merge of their container's attributes and theirs
+      collection :components,   Ironfan::Dsl::Component,   :resolver => :merge_resolve, :key_method => :name
       collection :run_list_items, RunListItem,             :resolver => :merge_resolve, :key_method => :name
       collection :clouds,       Ironfan::Dsl::Cloud,       :resolver => :merge_resolve, :key_method => :name
       collection :volumes,      Ironfan::Dsl::Volume,      :resolver => :merge_resolve, :key_method => :name
@@ -30,6 +33,14 @@ module Ironfan
       magic      :cluster_names, Whatever
       magic      :realm_name,    Symbol
 
+      field      :source_file,   String
+
+      extend Gorillib::Concern
+
+      def self._project compute, &blk
+        compute.around &blk
+      end
+
       def initialize(attrs={},&block)
         self.underlay   = attrs[:owner] if attrs[:owner]
         super
@@ -89,8 +100,18 @@ module Ironfan
         raise "run_list placement must be one of :first, :normal, :last or nil (also means :normal)" unless [:first, :last, :own, nil].include?(placement)
         placement = :normal if placement.nil?
         @@run_list_rank += 1
+        # Rank is a global order that tells what order this was encountered in. 
         run_list_items[item] = { :name => item, :rank => @@run_list_rank, :placement => placement }
       end
+
+      def around &blk
+        before
+        instance_eval(&blk)
+        after
+      end
+
+      def after() end
+      def before() end
     end
 
   end

data/lib/ironfan/dsl/facet.rb

@@ -2,6 +2,8 @@ module Ironfan
   class Dsl
 
     class Facet < Ironfan::Dsl::Compute
+      include Ironfan::Plugin::Base; register_with Ironfan::Dsl::Cluster
+
       magic      :instances,    Integer,                :default => 1
       collection :servers,      Ironfan::Dsl::Server,   :resolver => :deep_resolve
       field      :cluster_name, String
@@ -16,7 +18,16 @@ module Ironfan
         for i in 0 .. instances-1; server(i); end
       end
 
+      def children
+        servers.to_a + components.to_a
+      end
+
       def full_name()           "#{cluster_name}-#{name}";      end
+
+      def self.plugin_hook owner, attrs, plugin_name, full_name, &blk
+        facet = owner.facet(plugin_name, new(attrs.merge(name: plugin_name, owner: owner)))
+        _project facet, &blk
+      end
     end
   end
 end

data/lib/ironfan/dsl/realm.rb

@@ -4,30 +4,33 @@ module Ironfan
     class Realm < Ironfan::Dsl::Compute
       collection :clusters,       Ironfan::Dsl::Cluster,   :resolver => :deep_resolve
 
+      def children
+        clusters.to_a
+      end
+
       magic :cluster_suffixes,    Whatever
 
       def initialize(attrs={},&block)
         cluster_names({})
         realm_name attrs[:name] if attrs[:name]
-        super
+        attrs[:environment] = realm_name unless attrs.has_key?(:environment)
+        super(attrs, &block)
       end
 
       def cluster(label, attrs={},&blk)
         new_name = [realm_name, label].join('_').to_sym
-        cluster = Ironfan::Dsl::Cluster.new(name: new_name, owner: self, cluster_names: cluster_names)
-        cluster_names[label] = new_name
-        cluster.receive!(attrs, &blk)
-        super(new_name, cluster)
-      end
 
-      def cluster_name suffix
-        clusters.fetch([realm_name, suffix.to_s].join('_').to_sym).name.to_sym
-      end
-      
-      def cluster_suffix suffix
-        clusters.
-          fetch([realm_name, suffix.to_s].join('_').to_sym).name.to_s.
-          gsub(/^#{realm_name}_/, '').to_sym
+        if clusters.keys.include? new_name
+          clusters[new_name].tap do |cl|
+            cl.receive!(attrs)
+            cl.instance_eval(&blk) if block_given?
+          end
+        else
+          cluster = Ironfan::Dsl::Cluster.new(name: new_name, owner: self, cluster_names: cluster_names)
+          cluster_names[label] = new_name
+          cluster.receive!(attrs, &blk)
+          super(new_name, cluster)
+        end
       end
     end
   end