ironfan 3.1.0.rc1

data/lib/chef/knife/cluster_sync.rb

@@ -0,0 +1,77 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/generic_command.rb")
+
+class Chef
+  class Knife
+    class ClusterSync < Ironfan::Script
+      import_banner_and_options(Ironfan::Script)
+
+      option :cloud,
+        :long        => "--[no-]cloud",
+        :description => "Sync to the cloud (default is yes, sync cloud; use --no-cloud to skip)",
+        :default     => true,
+        :boolean     => true
+      option :chef,
+        :long        => "--[no-]chef",
+        :description => "Sync to the chef server (default is yes, sync chef; use --no-chef to skip)",
+        :default     => true,
+        :boolean     => true
+      option :sync_all,
+        :long        => "--[no-]sync-all",
+        :description => "Sync, as best as possible, any defined node (even if it is missing from cloud or chef)",
+        :default     => false,
+        :boolean     => true
+
+
+      def relevant?(server)
+        if config[:sync_all]
+          not server.bogus?
+        else
+          server.created? && server.in_chef?
+        end
+      end
+
+      def perform_execution(target)
+        if config[:chef]
+          sync_to_chef target
+        else Chef::Log.debug("Skipping sync to chef") ; end
+
+        if config[:cloud] && target.any?(&:in_cloud?)
+          sync_to_cloud target
+        else Chef::Log.debug("Skipping sync to cloud") ; end
+      end
+
+      def sync_to_chef(target)
+        if config[:dry_run]
+          ui.info "(can't do a dry-run when syncing to chef -- skipping)"
+          return
+        end
+        ui.info "Syncing to Chef:"
+        target.sync_to_chef
+      end
+
+      def sync_to_cloud(target)
+        ui.info "Syncing to cloud:"
+        target.sync_to_cloud
+      end
+
+    end
+  end
+end

data/lib/chef/knife/generic_command.rb

@@ -0,0 +1,66 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/knife_common.rb")
+
+module Ironfan
+  class Script < Chef::Knife
+    include Ironfan::KnifeCommon
+
+    deps do
+      Ironfan::KnifeCommon.load_deps
+    end
+
+    option :dry_run,
+      :long        => "--dry-run",
+      :description => "Don't really run, just use mock calls",
+      :boolean     => true,
+      :default     => false
+    option :yes,
+      :long        => "--yes",
+      :description => "Skip confirmation prompts on risky actions.",
+      :boolean     => true
+
+    def run
+      load_ironfan
+      die(banner) if @name_args.empty?
+      configure_dry_run
+
+      target = get_relevant_slice(* @name_args)
+
+      die("No nodes to #{sub_command}, exiting", 1) if target.empty?
+
+      ui.info(["\n",
+          ui.color("Running #{sub_command}", :cyan),
+          " on #{target.joined_names}..."].join())
+      unless config[:yes]
+        ui.info("")
+        confirm_execution(target)
+      end
+      #
+      perform_execution(target)
+      ui.info("")
+      ui.info "Finished! Current state:"
+      display(target)
+    end
+
+    def perform_execution(target)
+      target.send(sub_command)
+    end
+  end
+end

data/lib/chef/knife/knife_common.rb

@@ -0,0 +1,195 @@
+require 'chef/knife'
+
+module Ironfan
+  module KnifeCommon
+
+    def self.load_deps
+      require 'formatador'
+      require 'chef/node'
+      require 'chef/api_client'
+      require 'fog'
+    end
+
+    def load_ironfan
+      $LOAD_PATH << File.join(Chef::Config[:ironfan_path], '/lib') if Chef::Config[:ironfan_path]
+      require 'ironfan'
+      $stdout.sync = true
+      Ironfan.ui          = self.ui
+      Ironfan.chef_config = self.config
+    end
+
+    #
+    # A slice of a cluster:
+    #
+    # @param [String] cluster_name  -- cluster to slice
+    # @param [String] facet_name    -- facet to slice (or nil for all in cluster)
+    # @param [Array, String] slice_indexes -- servers in that facet (or nil for all in facet).
+    #   You must specify a facet if you use slice_indexes.
+    #
+    # @return [Ironfan::ServerSlice] the requested slice
+    def get_slice(cluster_name, facet_name=nil, slice_indexes=nil)
+      if facet_name.nil? && slice_indexes.nil?
+        cluster_name, facet_name, slice_indexes = cluster_name.split(/[\s\-]/, 3)
+      end
+      ui.info("Inventorying servers in #{predicate_str(cluster_name, facet_name, slice_indexes)}")
+      cluster = Ironfan.load_cluster(cluster_name)
+      cluster.resolve!
+      cluster.discover!
+      cluster.slice(facet_name, slice_indexes)
+    end
+
+    def predicate_str(cluster_name, facet_name, slice_indexes)
+      [ "#{ui.color(cluster_name, :bold)} cluster",
+        (facet_name    ? "#{ui.color(facet_name, :bold)} facet"      : "#{ui.color("all", :bold)} facets"),
+        (slice_indexes ? "servers #{ui.color(slice_indexes, :bold)}" : "#{ui.color("all", :bold)} servers")
+      ].join(', ')
+    end
+
+    # method to nodes should be filtered on
+    def relevant?(server)
+      server.exists?
+    end
+
+    # override in subclass to confirm risky actions
+    def confirm_execution(*args)
+      # pass
+    end
+
+    #
+    # Get a slice of nodes matching the given filter
+    #
+    # @example
+    #    target = get_relevant_slice(* @name_args)
+    #
+    def get_relevant_slice( *predicate )
+      full_target = get_slice( *predicate )
+      display(full_target) do |svr|
+        rel = relevant?(svr)
+        { :relevant? => (rel ? "[blue]#{rel}[reset]" : '-' ) }
+      end
+      full_target.select{|svr| relevant?(svr) }
+    end
+
+    # passes target to ClusterSlice#display, will show headings in server slice
+    # tables based on the --verbose flag
+    def display(target, display_style=nil, &block)
+      display_style ||= (config[:verbosity] == 0 ? :default : :expanded)
+      target.display(display_style, &block)
+    end
+
+    #
+    # Put Fog into mock mode if --dry_run
+    #
+    def configure_dry_run
+      if config[:dry_run]
+        Fog.mock!
+        Fog::Mock.delay = 0
+      end
+    end
+
+    # Show a pretty progress bar while we wait for a set of threads to finish.
+    def progressbar_for_threads(threads)
+      section "Waiting for servers:"
+      total      = threads.length
+      remaining  = threads.select(&:alive?)
+      start_time = Time.now
+      until remaining.empty?
+        remaining = remaining.select(&:alive?)
+        if config[:verbose]
+          ui.info "waiting: #{total - remaining.length} / #{total}, #{(Time.now - start_time).to_i}s"
+          sleep 5
+        else
+          Formatador.redisplay_progressbar(total - remaining.length, total, {:started_at => start_time })
+          sleep 1
+        end
+      end
+      # Collapse the threads
+      threads.each(&:join)
+      ui.info ''
+    end
+
+    def bootstrapper(server, hostname)
+      bootstrap = Chef::Knife::Bootstrap.new
+      bootstrap.config.merge!(config)
+
+      bootstrap.name_args               = [ hostname ]
+      bootstrap.config[:node]           = server
+      bootstrap.config[:run_list]       = server.combined_run_list
+      bootstrap.config[:ssh_user]       = config[:ssh_user]       || server.cloud.ssh_user
+      bootstrap.config[:attribute]      = config[:attribute]
+      bootstrap.config[:identity_file]  = config[:identity_file]  || server.cloud.ssh_identity_file
+      bootstrap.config[:distro]         = config[:distro]         || server.cloud.bootstrap_distro
+      bootstrap.config[:use_sudo]       = true unless config[:use_sudo] == false
+      bootstrap.config[:chef_node_name] = server.fullname
+      bootstrap.config[:client_key]     = server.client_key.body  if server.client_key.body
+
+      bootstrap
+    end
+
+    def run_bootstrap(node, hostname)
+      bs = bootstrapper(node, hostname)
+      if config[:skip].to_s == 'true'
+        ui.info "Skipping: bootstrapp #{hostname} with #{JSON.pretty_generate(bs.config)}"
+        return
+      end
+      begin
+        bs.run
+      rescue StandardError => e
+        ui.warn e
+        ui.warn e.backtrace
+        ui.warn ""
+        ui.warn node.inspect
+        ui.warn ""
+      end
+    end
+
+    #
+    # Utilities
+    #
+
+    def sub_command
+      self.class.sub_command
+    end
+
+    def confirm_or_exit question, correct_answer
+      response = ui.ask_question(question)
+      unless response.chomp == correct_answer
+        die "I didn't think so.", "Aborting!", 1
+      end
+      ui.info("")
+    end
+
+    #
+    # Announce a new section of tasks
+    #
+    def section(desc, *style)
+      style = [:blue] if style.empty?
+      ui.info(ui.color(desc, *style))
+    end
+
+    def die *args
+      Ironfan.die(*args)
+    end
+
+    module ClassMethods
+      def sub_command
+        self.to_s.gsub(/^.*::/, '').gsub(/^Cluster/, '').downcase
+      end
+
+      def import_banner_and_options(klass, options={})
+        options[:except] ||= []
+        deps{ klass.load_deps }
+        klass.options.sort.each do |name, info|
+          next if options.include?(name) || options[:except].include?(name)
+          option name, info
+        end
+        banner "knife cluster #{sub_command} CLUSTER_NAME [FACET_NAME [INDEXES]] (options)"
+      end
+    end
+    def self.included(base)
+      base.class_eval do
+        extend ClassMethods
+      end
+    end
+  end
+end

data/lib/ironfan.rb

@@ -0,0 +1,143 @@
+require 'chef/mash'
+require 'chef/config'
+#
+require 'gorillib/metaprogramming/class_attribute'
+require 'gorillib/hash/reverse_merge'
+require 'gorillib/object/blank'
+require 'gorillib/hash/compact'
+require 'set'
+
+require 'ironfan/dsl_object'
+require 'ironfan/cloud'
+require 'ironfan/security_group'
+require 'ironfan/compute'           # base class for machine attributes
+require 'ironfan/facet'             # similar machines within a cluster
+require 'ironfan/cluster'           # group of machines with a common mission
+require 'ironfan/server'            # realization of a specific facet
+require 'ironfan/discovery'         # pair servers with Fog and Chef objects
+require 'ironfan/server_slice'      # collection of server objects
+require 'ironfan/volume'            # configure external and internal volumes
+require 'ironfan/private_key'       # coordinate chef keys, cloud keypairs, etc
+require 'ironfan/role_implications' # make roles trigger other actions (security groups, etc)
+#
+require 'ironfan/chef_layer'        # interface to chef for server actions
+require 'ironfan/fog_layer'         # interface to fog  for server actions
+#
+require 'ironfan/deprecated'        # stuff slated to go away
+
+module Ironfan
+
+  # path to search for cluster definition files
+  def self.cluster_path
+    return Chef::Config[:cluster_path] if Chef::Config[:cluster_path]
+    raise "Holy smokes, you have no cookbook_path or cluster_path set up. Follow chef's directions for creating a knife.rb." if Chef::Config[:cookbook_path].blank?
+    cl_path = Chef::Config[:cookbook_path].map{|dir| File.expand_path('../clusters', dir) }.uniq
+    ui.warn "No cluster path set. Taking a wild guess that #{cl_path.inspect} is \nreasonable based on your cookbook_path -- but please set cluster_path in your knife.rb"
+    Chef::Config[:cluster_path] = cl_path
+  end
+
+  #
+  # Delegates
+  def self.clusters
+    Chef::Config[:clusters] ||= Mash.new
+  end
+
+  def self.ui=(ui) @ui = ui ; end
+  def self.ui()    @ui      ; end
+
+  def self.chef_config=(cc) @chef_config = cc ; end
+  def self.chef_config()    @chef_config      ; end
+
+  #
+  # Defines a cluster with the given name.
+  #
+  # @example
+  #   Ironfan.cluster 'demosimple' do
+  #     cloud :ec2 do
+  #       availability_zones  ['us-east-1d']
+  #       flavor              "t1.micro"
+  #       image_name          "ubuntu-natty"
+  #     end
+  #     role                  :base_role
+  #     role                  :chef_client
+  #
+  #     facet :sandbox do
+  #       instances 2
+  #       role                :nfs_client
+  #     end
+  #   end
+  #
+  #
+  def self.cluster(name, attrs={}, &block)
+    name = name.to_sym
+    cl = ( self.clusters[name] ||= Ironfan::Cluster.new(name, attrs) )
+    cl.configure(&block)
+    cl
+  end
+
+  #
+  # Return cluster if it's defined. Otherwise, search Ironfan.cluster_path
+  # for an eponymous file, load it, and return the cluster it defines.
+  #
+  # Raises an error if a matching file isn't found, or if loading that file
+  # doesn't define the requested cluster.
+  #
+  # @return [Ironfan::Cluster] the requested cluster
+  def self.load_cluster(cluster_name)
+    raise ArgumentError, "Please supply a cluster name" if cluster_name.to_s.empty?
+    return clusters[cluster_name] if clusters[cluster_name]
+
+    cluster_file = cluster_filenames[cluster_name] or die("Couldn't find a definition for #{cluster_name} in cluster_path: #{cluster_path.inspect}")
+
+    Chef::Log.info("Loading cluster #{cluster_file}")
+
+    require cluster_file
+    unless clusters[cluster_name] then  die("#{cluster_file} was supposed to have the definition for the #{cluster_name} cluster, but didn't") end
+
+    clusters[cluster_name]
+  end
+
+  #
+  # Map from cluster name to file name
+  #
+  # @return [Hash] map from cluster name to file name
+  def self.cluster_filenames
+    return @cluster_filenames if @cluster_filenames
+    @cluster_filenames = {}
+    cluster_path.each do |cp_dir|
+      Dir[ File.join(cp_dir, '*.rb') ].each do |filename|
+        cluster_name = File.basename(filename).gsub(/\.rb$/, '')
+        @cluster_filenames[cluster_name] ||= filename
+      end
+    end
+    @cluster_filenames
+  end
+
+  #
+  # Utility to die with an error message.
+  # If the last arg is an integer, use it as the exit code.
+  #
+  def self.die *strings
+    exit_code = strings.last.is_a?(Integer) ? strings.pop : -1
+    strings.each{|str| ui.warn str }
+    exit exit_code
+  end
+
+  #
+  # Utility to turn an error into a warning
+  #
+  # @example
+  #   Ironfan.safely do
+  #     Ironfan.fog_connection.associate_address(self.fog_server.id, address)
+  #   end
+  #
+  def self.safely
+    begin
+      yield
+    rescue StandardError => boom
+      ui.info( boom )
+      Chef::Log.error( boom )
+      Chef::Log.error( boom.backtrace.join("\n") )
+    end
+  end
+end