cluster_chef-knife 3.0.5

data/lib/chef/knife/cluster_show.rb

@@ -0,0 +1,56 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/knife_common.rb")
+
+class Chef
+  class Knife
+    class ClusterShow < Knife
+      include ClusterChef::KnifeCommon
+      deps do
+        ClusterChef::KnifeCommon.load_deps
+      end
+
+      banner "knife cluster show CLUSTER_NAME [FACET_NAME [INDEXES]] (options)"
+
+      def run
+        load_cluster_chef
+        die(banner) if @name_args.empty?
+        configure_dry_run
+
+        # Load the cluster/facet/slice/whatever
+        target = get_slice(* @name_args)
+
+        #
+        # Dump entire contents of objects if -VV flag given
+        #
+        if config[:verbosity] >= 2
+          target.each do |svr|
+            Chef::Log.debug( "Server #{svr.name}: #{JSON.pretty_generate(svr.to_hash)}" )
+            Chef::Log.debug( "- cloud: #{JSON.pretty_generate(svr.cloud.to_hash)}" )
+            Chef::Log.debug( "- fog:   #{JSON.pretty_generate(svr.fog_description_for_launch)}" )
+          end
+        end
+
+        # Display same
+        display(target)
+
+      end
+    end
+  end
+end

data/lib/chef/knife/cluster_ssh.rb

@@ -0,0 +1,94 @@
+#
+# Author:: Chris Howe (<chris@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/knife_common.rb")
+require 'chef/knife/ssh'
+
+class Chef
+  class Knife
+    class ClusterSsh < Chef::Knife::Ssh
+      include ClusterChef::KnifeCommon
+
+      deps do
+        Chef::Knife::Ssh.load_deps
+        ClusterChef::KnifeCommon.load_deps
+      end
+
+      banner 'knife cluster ssh "CLUSTER [FACET [INDEXES]]" COMMAND (options)'
+      Chef::Knife::Ssh.options.each do |name, hsh|
+        next if name == :attribute
+        option name, hsh
+      end
+
+      option :attribute,
+        :short => "-a ATTR",
+        :long => "--attribute ATTR",
+        :description => "The attribute to use for opening the connection - default is fqdn (ec2 users may prefer cloud.public_hostname)"
+
+      def configure_session
+        predicate = @name_args[0].split(/[\s\-]+/,3).map(&:strip)
+
+        target = get_slice(*predicate).select(&:sshable?)
+
+        display(target) if config[:verbose]
+
+        config[:attribute]     ||= Chef::Config[:knife][:ssh_address_attribute] || "fqdn"
+        config[:ssh_user]      ||= Chef::Config[:knife][:ssh_user]
+        config[:identity_file] ||= target.ssh_identity_file
+
+        @action_nodes = target.chef_nodes
+        addresses = target.servers.map do |svr|
+          if (svr.cloud.public_ip)            then address = svr.cloud.public_ip ; end
+          if (not address) && (svr.fog_server) then address = svr.fog_server.public_ip_address ; end
+          if (not address) && (svr.chef_node)  then address = format_for_display( svr.chef_node )[config[:attribute]] ; end
+          address
+        end.compact
+
+        (ui.fatal("No nodes returned from search!"); exit 10) if addresses.nil? || addresses.length == 0
+
+        session_from_list(addresses)
+      end
+
+      def cssh
+        exec "cssh "+session.servers_for.map {|server| server.user ? "#{server.user}@#{server.host}" : server.host}.join(" ")
+      end
+
+      def run
+        load_cluster_chef
+        die(banner) if @name_args.empty?
+        extend Chef::Mixin::Command
+
+        @longest = 0
+        configure_session
+
+        case @name_args[1]
+        when "interactive"  then interactive
+        when "screen"       then screen
+        when "tmux"         then tmux
+        when "macterm"      then macterm
+        when "cssh"         then cssh
+        else
+          ssh_command(@name_args[1..-1].join(" "))
+        end
+
+        session.close
+      end
+
+    end
+  end
+end

data/lib/chef/knife/cluster_start.rb

@@ -0,0 +1,32 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/knife_common.rb")
+require File.expand_path(File.dirname(__FILE__)+"/generic_command.rb")
+
+class Chef
+  class Knife
+    class ClusterStart < ClusterChef::Script
+      import_banner_and_options(ClusterChef::Script)
+
+      def relevant?(server)
+        server.startable?
+      end
+    end
+  end
+end

data/lib/chef/knife/cluster_stop.rb

@@ -0,0 +1,37 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/generic_command.rb")
+
+class Chef
+  class Knife
+    class ClusterStop < ClusterChef::Script
+      import_banner_and_options(ClusterChef::Script)
+
+      def relevant?(server)
+        server.running?
+      end
+
+      def confirm_execution(target)
+        ui.info "  Unless these nodes are backed by EBS volumes, this will result in loss of all data"
+        ui.info "  not saved elsewhere. Even if they are EBS backed, there may still be some data loss."
+        confirm_or_exit("Are you absolutely certain that you want to perform this action? (Type 'Yes' to confirm) ", 'Yes')
+      end
+    end
+  end
+end

data/lib/chef/knife/cluster_sync.rb

@@ -0,0 +1,76 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/generic_command.rb")
+
+class Chef
+  class Knife
+    class ClusterSync < ClusterChef::Script
+      import_banner_and_options(ClusterChef::Script)
+
+      option :cloud,
+        :long        => "--[no-]cloud",
+        :description => "Sync to the cloud (default syncs cloud; use --no-cloud to skip)",
+        :default     => true,
+        :boolean     => true
+      option :chef,
+        :long        => "--[no-]chef",
+        :description => "Sync to the chef server (default syncs chef; use --no-chef to skip)",
+        :default     => true,
+        :boolean     => true
+      option :sync_all,
+        :long        => "--[no-]sync-all",
+        :description => "Sync, as best as possible, any defined node (even if it is missing from cloud or chef)",
+        :default     => false,
+        :boolean     => true
+
+
+      def relevant?(server)
+        if config[:sync_all]
+          not server.bogus?
+        else
+          server.created? && server.in_chef?
+        end
+      end
+
+      def perform_execution(target)
+        if config[:chef]
+          sync_to_chef target
+        else Chef::Log.debug("Skipping sync to chef") ; end
+        if config[:cloud] && target.any?(&:in_cloud?)
+          sync_to_cloud target
+        else Chef::Log.debug("Skipping sync to cloud") ; end
+      end
+
+      def sync_to_chef(target)
+        if config[:dry_run]
+          ui.info "(can't do a dry-run when syncing to chef -- skipping)"
+          return
+        end
+        ui.info "Syncing to Chef:"
+        target.sync_to_chef
+      end
+
+      def sync_to_cloud(target)
+        ui.info "Syncing to cloud:"
+        target.sync_to_cloud
+      end
+
+    end
+  end
+end

data/lib/chef/knife/generic_command.rb

@@ -0,0 +1,66 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/knife_common.rb")
+
+module ClusterChef
+  class Script < Chef::Knife
+    include ClusterChef::KnifeCommon
+
+    deps do
+      ClusterChef::KnifeCommon.load_deps
+    end
+
+    option :dry_run,
+      :long        => "--dry-run",
+      :description => "Don't really run, just use mock calls",
+      :boolean     => true,
+      :default     => false
+    option :yes,
+      :long        => "--yes",
+      :description => "Skip confirmation prompts on risky actions.",
+      :boolean     => true
+
+    def run
+      load_cluster_chef
+      die(banner) if @name_args.empty?
+      configure_dry_run
+
+      target = get_relevant_slice(* @name_args)
+
+      die("No nodes to #{sub_command}, exiting", 1) if target.empty?
+
+      ui.info(["\n",
+          ui.color("Running #{sub_command}", :cyan),
+          " on #{target.joined_names}..."].join())
+      unless config[:yes]
+        ui.info("")
+        confirm_execution(target)
+      end
+      #
+      perform_execution(target)
+      ui.info("")
+      ui.info "Finished! Current state:"
+      display(target)
+    end
+
+    def perform_execution(target)
+      target.send(sub_command)
+    end
+  end
+end

data/lib/chef/knife/knife_common.rb

@@ -0,0 +1,199 @@
+require 'chef/knife'
+
+module ClusterChef
+  module KnifeCommon
+
+    def self.load_deps
+      require 'readline'
+      require 'formatador'
+      require 'chef/node'
+      require 'chef/api_client'
+      require 'fog'
+    end
+
+    def load_cluster_chef
+      $LOAD_PATH << File.join(Chef::Config[:cluster_chef_path], '/lib') if Chef::Config[:cluster_chef_path]
+      require 'cluster_chef'
+      $stdout.sync = true
+      ClusterChef.ui          = self.ui
+      ClusterChef.chef_config = self.config
+    end
+
+    #
+    # A slice of a cluster:
+    #
+    # @param [String] cluster_name  -- cluster to slice
+    # @param [String] facet_name    -- facet to slice (or nil for all in cluster)
+    # @param [Array, String] slice_indexes -- servers in that facet (or nil for all in facet).
+    #   You must specify a facet if you use slice_indexes.
+    #
+    # @return [ClusterChef::ServerSlice] the requested slice
+    def get_slice(cluster_name, facet_name=nil, slice_indexes=nil)
+      if facet_name.nil? && slice_indexes.nil?
+        cluster_name, facet_name, slice_indexes = cluster_name.split(/[\s\-]/, 3)
+      end
+      ui.info("Inventorying servers in #{predicate_str(cluster_name, facet_name, slice_indexes)}")
+      cluster = ClusterChef.load_cluster(cluster_name)
+      cluster.resolve!
+      cluster.discover!
+      cluster.slice(facet_name, slice_indexes)
+    end
+
+    def predicate_str(cluster_name, facet_name, slice_indexes)
+      [ "#{ui.color(cluster_name, :bold)} cluster",
+        (facet_name    ? "#{ui.color(facet_name, :bold)} facet"      : "#{ui.color("all", :bold)} facets"),
+        (slice_indexes ? "servers #{ui.color(slice_indexes, :bold)}" : "#{ui.color("all", :bold)} servers")
+      ].join(', ')
+    end
+
+    # method to nodes should be filtered on
+    def relevant?(server)
+      server.exists?
+    end
+
+    # override in subclass to confirm risky actions
+    def confirm_execution(*args)
+      # pass
+    end
+
+    #
+    # Get a slice of nodes matching the given filter
+    #
+    # @example
+    #    target = get_relevant_slice(* @name_args)
+    #
+    def get_relevant_slice( *predicate )
+      full_target = get_slice( *predicate )
+      display(full_target) do |svr|
+        rel = relevant?(svr)
+        { :relevant? => (rel ? "[blue]#{rel}[reset]" : '-' ) }
+      end
+      full_target.select{|svr| relevant?(svr) }
+    end
+
+    # passes target to ClusterSlice#display, will show headings in server slice
+    # tables based on the --verbose flag
+    def display(target, display_style=nil, &block)
+      display_style ||= (config[:verbosity] == 0 ? :default : :expanded)
+      target.display(display_style, &block)
+    end
+
+    #
+    # Put Fog into mock mode if --dry_run
+    #
+    def configure_dry_run
+      if config[:dry_run]
+        Fog.mock!
+        Fog::Mock.delay = 0
+      end
+    end
+
+    # Show a pretty progress bar while we wait for a set of threads to finish.
+    def progressbar_for_threads(threads)
+      section "Waiting for servers:"
+      total      = threads.length
+      remaining  = threads.select(&:alive?)
+      start_time = Time.now
+      until remaining.empty?
+        remaining = remaining.select(&:alive?)
+        if config[:verbose]
+          ui.info "waiting: #{total - remaining.length} / #{total}, #{(Time.now - start_time).to_i}s"
+          sleep 5
+        else
+          Formatador.redisplay_progressbar(total - remaining.length, total, {:started_at => start_time })
+          sleep 1
+        end
+      end
+      # Collapse the threads
+      threads.each(&:join)
+      ui.info ''
+    end
+
+    def bootstrapper(server, hostname)
+      bootstrap = Chef::Knife::Bootstrap.new
+      bootstrap.config.merge!(config)
+
+      bootstrap.name_args               = [ hostname ]
+      bootstrap.config[:node]           = server
+      bootstrap.config[:run_list]       = server.combined_run_list
+      bootstrap.config[:ssh_user]       = config[:ssh_user]       || server.cloud.ssh_user
+      bootstrap.config[:attribute]      = config[:attribute]
+      bootstrap.config[:identity_file]  = config[:identity_file]  || server.cloud.ssh_identity_file
+      bootstrap.config[:distro]         = config[:distro]         || server.cloud.bootstrap_distro
+      bootstrap.config[:use_sudo]       = true unless config[:use_sudo] == false
+      bootstrap.config[:chef_node_name] = server.fullname
+      bootstrap.config[:client_key]     = server.client_key.body            if server.client_key.body
+
+      bootstrap
+    end
+
+    def run_bootstrap(node, hostname)
+      bs = bootstrapper(node, hostname)
+      if config[:skip].to_s == 'true'
+        ui.info "Skipping: bootstrapp #{hostname} with #{JSON.pretty_generate(bs.config)}"
+        return
+      end
+      begin
+        bs.run
+      rescue StandardError => e
+        ui.warn e
+        ui.warn e.backtrace
+        ui.warn ""
+        ui.warn node.inspect
+        ui.warn ""
+      end
+    end
+
+    #
+    # Utilities
+    #
+
+    def sub_command
+      self.class.sub_command
+    end
+
+    def confirm_or_exit question, correct_answer
+      response = ui.ask_question(question)
+      unless response.chomp == correct_answer
+        die "I didn't think so.", "Aborting!", 1
+      end
+      ui.info("")
+    end
+
+    #
+    # Announce a new section of tasks
+    #
+    def section(desc, *style)
+      style = [:blue] if style.empty?
+      ui.info(ui.color(desc, *style))
+    end
+
+    def die *args
+      ClusterChef.die(*args)
+    end
+
+    module ClassMethods
+      def sub_command
+        self.to_s.gsub(/^.*::/, '').gsub!(/^Cluster/, '').downcase
+      end
+
+      def import_banner_and_options(klass, options={})
+        options[:except] ||= []
+        klass.options.each do |name, info|
+          next if options.include?(name) || options[:except].include?(name)
+          option name, info
+        end
+        banner "knife cluster #{sub_command} CLUSTER_NAME [FACET_NAME [INDEXES]] (options)"
+
+        deps do
+          klass.load_deps
+        end
+      end
+    end
+    def self.included(base)
+      base.class_eval do
+        extend ClassMethods
+      end
+    end
+  end
+end

data/rspec.watchr

@@ -0,0 +1,29 @@
+# -*- ruby -*-
+
+def run_spec(file)
+  unless File.exist?(file)
+    puts "#{file} does not exist"
+    return
+  end
+
+  puts   "Running #{file}"
+  system "rspec #{file}"
+  puts
+end
+
+watch("spec/.*/*_spec\.rb") do |match|
+  run_spec match[0]
+end
+
+watch("lib/(.*)\.rb") do |match|
+  file = %{spec/#{match[1]}_spec.rb}
+  run_spec file if File.exists?(file)
+end
+
+# watch('lib/cluster_chef/cookbook_munger\.rb') do |match|
+#   system match[0]
+# end
+#
+# watch('lib/cluster_chef/cookbook_munger/.*\.erb') do |match|
+#   system 'lib/cluster_chef/cookbook_munger.rb'
+# end

data/spec/cluster_chef/cluster_spec.rb

@@ -0,0 +1,13 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require CLUSTER_CHEF_DIR("lib/cluster_chef")
+
+describe ClusterChef::Cluster do
+  describe 'discover!' do
+    let(:cluster){ get_example_cluster(:monkeyballs) }
+
+    it 'enumerates chef nodes' do
+      cluster.discover!
+    end
+  end
+end

data/spec/cluster_chef/facet_spec.rb

@@ -0,0 +1,70 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+require CLUSTER_CHEF_DIR("lib/cluster_chef")
+
+describe ClusterChef::Facet do
+  let(:cluster){ ClusterChef.cluster(:gibbon) }
+  let(:facet){
+    cluster.facet(:namenode) do
+      instances 5
+    end
+  }
+
+  describe 'slicing' do
+    it 'has servers' do
+      facet.indexes.should == [0, 1, 2, 3, 4]
+      facet.valid_indexes.should == [0, 1, 2, 3, 4]
+      facet.server(3){ name(:bob) }
+      svrs = facet.servers
+      svrs.length.should == 5
+      svrs.map{|svr| svr.name }.should == ["gibbon-namenode-0", "gibbon-namenode-1", "gibbon-namenode-2", :bob, "gibbon-namenode-4"]
+    end
+
+    it 'servers have bogosity if out of range' do
+      facet.server(69).should be_bogus
+      facet.servers.select(&:bogus?).map(&:facet_index).should == [69]
+      facet.indexes.should       == [0, 1, 2, 3, 4, 69]
+      facet.valid_indexes.should == [0, 1, 2, 3, 4]
+    end
+
+    it 'returns all on nil or "", but [] means none' do
+      facet.server(69)
+      facet.slice('' ).map(&:facet_index).should == [0, 1, 2, 3, 4, 69]
+      facet.slice(nil).map(&:facet_index).should == [0, 1, 2, 3, 4, 69]
+      facet.slice([] ).map(&:facet_index).should == []
+    end
+
+    it 'slice returns all by default' do
+      facet.server(69)
+      facet.slice().map(&:facet_index).should == [0, 1, 2, 3, 4, 69]
+    end
+
+    it 'with an array returns specified indexes (bogus or not) in sorted order' do
+      facet.server(69)
+      facet.slice( [3, 1, 0]     ).map(&:facet_index).should == [0, 1, 3]
+      facet.slice( [3, 1, 69, 0] ).map(&:facet_index).should == [0, 1, 3, 69]
+    end
+
+    it 'with an array does not create new dummy servers' do
+      facet.server(69)
+      facet.slice( [3, 1, 69, 0, 75, 123] ).map(&:facet_index).should == [0, 1, 3, 69]
+      facet.has_server?(75).should be_false
+      facet.has_server?(69).should be_true
+    end
+
+    it 'with a string, converts to intervals' do
+      facet.slice('1'      ).map(&:facet_index).should == [1]
+      facet.slice('5'      ).map(&:facet_index).should == []
+      facet.slice('1-1'    ).map(&:facet_index).should == [1]
+      facet.slice('0-1'    ).map(&:facet_index).should == [0,1]
+      facet.slice('0-1,3-4').map(&:facet_index).should == [0,1,3,4]
+      facet.slice('0-1,69' ).map(&:facet_index).should == [0,1,69]
+      facet.slice('0-2,1-3').map(&:facet_index).should == [0,1,2,3]
+      facet.slice('3-1'    ).map(&:facet_index).should == []
+      facet.slice('2-5'    ).map(&:facet_index).should == [2,3,4]
+      facet.slice(1).map(&:facet_index).should == [1]
+    end
+
+  end
+end
+