ironfan 4.5.1 → 4.5.2

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# v4.5.2:
+* 'knife cluster launch --bootstrap' should ensure that Chef::Config[:environment] is set, just as 'knife cluster bootstrap' does
+* Cleaning up knife commands to skip bogus servers (fixes #213)
+* EXPERIMENTAL: Adding (cross-platform) ironfaned chef omnibus bootstrap
+
 # v4.5.1:
 * Clean up on 12.04 template - do dist-upgrade, include omnibus bin in $PATH, nicer first-boot.json
 * Removing superfluous raise on duplicate machines during load

data/VERSION

@@ -1 +1 @@
-4.5.1
+4.5.2

data/ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.5.1"
+  s.version = "4.5.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-11-14"
+  s.date = "2012-11-27"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [
@@ -38,6 +38,7 @@ Gem::Specification.new do |s|
     "config/ubuntu12.04-ironfan.erb",
     "ironfan.gemspec",
     "lib/chef/knife/bootstrap/centos6.2-ironfan.erb",
+    "lib/chef/knife/bootstrap/chef-full-ironfan.erb",
     "lib/chef/knife/bootstrap/ubuntu10.04-ironfan.erb",
     "lib/chef/knife/bootstrap/ubuntu12.04-ironfan.erb",
     "lib/chef/knife/cluster_bootstrap.rb",
@@ -120,6 +121,7 @@ Gem::Specification.new do |s|
     "notes/walkthrough-hadoop.md",
     "notes/walkthrough-web.md",
     "spec/chef/cluster_bootstrap_spec.rb",
+    "spec/chef/cluster_launch_spec.rb",
     "spec/fixtures/ec2/elb/snakeoil.crt",
     "spec/fixtures/ec2/elb/snakeoil.key",
     "spec/fixtures/gunbai.rb",
@@ -127,7 +129,6 @@ Gem::Specification.new do |s|
     "spec/integration/minimal-chef-repo/chefignore",
     "spec/integration/minimal-chef-repo/environments/_default.json",
     "spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb",
-    "spec/integration/minimal-chef-repo/knife/credentials/knife-user-ironfantester.rb",
     "spec/integration/minimal-chef-repo/knife/knife.rb",
     "spec/integration/minimal-chef-repo/roles/systemwide.rb",
     "spec/integration/spec/elb_build_spec.rb",
@@ -148,7 +149,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.rubygems_version = "1.8.24"
   s.summary = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
-  s.test_files = ["spec/spec_helper/dummy_chef.rb", "spec/integration/spec_helper/launch_cluster.rb", "spec/integration/minimal-chef-repo/roles/systemwide.rb", "spec/integration/minimal-chef-repo/environments/_default.json", "spec/integration/minimal-chef-repo/chefignore", "spec/integration/minimal-chef-repo/knife/knife.rb", "spec/integration/minimal-chef-repo/knife/credentials/knife-user-ironfantester.rb", "spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb", "spec/integration/spec/simple_cluster_spec.rb", "spec/integration/spec/elb_build_spec.rb", "spec/integration/spec_helper.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/elb_spec.rb", "spec/ironfan/ec2/security_group_spec.rb", "spec/chef/cluster_bootstrap_spec.rb", "spec/fixtures/gunbai_slice.json", "spec/fixtures/ec2/elb/snakeoil.key", "spec/fixtures/ec2/elb/snakeoil.crt", "spec/fixtures/gunbai.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
+  s.test_files = ["spec/spec_helper/dummy_chef.rb", "spec/integration/spec_helper/launch_cluster.rb", "spec/integration/minimal-chef-repo/roles/systemwide.rb", "spec/integration/minimal-chef-repo/environments/_default.json", "spec/integration/minimal-chef-repo/chefignore", "spec/integration/minimal-chef-repo/knife/knife.rb", "spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb", "spec/integration/spec/simple_cluster_spec.rb", "spec/integration/spec/elb_build_spec.rb", "spec/integration/spec_helper.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/elb_spec.rb", "spec/ironfan/ec2/security_group_spec.rb", "spec/chef/cluster_launch_spec.rb", "spec/chef/cluster_bootstrap_spec.rb", "spec/fixtures/gunbai_slice.json", "spec/fixtures/ec2/elb/snakeoil.key", "spec/fixtures/ec2/elb/snakeoil.crt", "spec/fixtures/gunbai.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/lib/chef/knife/bootstrap/chef-full-ironfan.erb

@@ -0,0 +1,97 @@
+bash -c '
+
+##
+# EXPERIMENTAL
+##
+# This code was submitted as an alternative to the per-distribution scripts, just after a new bootstrap
+# (Ubuntu 12.04) was put into production at Infochimps; therefore, we have had no time to validate it
+# internally. Please exercise caution in applying it.
+#
+# Submitter: Ryan Schlesinger (https://github.com/ryansch)
+# Source: https://github.com/infochimps-labs/ironfan/pull/212
+# 
+
+<%= "export http_proxy=\"#{knife_config[:bootstrap_proxy]}\"" if knife_config[:bootstrap_proxy] -%>
+
+exists() {
+  if command -v $1 &>/dev/null
+  then
+    return 0
+  else
+    return 1
+  fi
+}
+
+install_sh="http://opscode.com/chef/install.sh"
+version_string="-v <%= chef_version %>"
+
+if ! exists /usr/bin/chef-client; then
+  if exists wget; then
+    bash <(wget <%= "--proxy=on " if knife_config[:bootstrap_proxy] %> ${install_sh} -O -) ${version_string}
+  else
+    if exists curl; then
+      bash <(curl -L <%= "--proxy=on " if knife_config[:bootstrap_proxy] %> ${install_sh}) ${version_string}
+    fi
+  fi
+fi
+
+mkdir -p /etc/chef
+
+<%- if @config[:client_key] %>
+(
+cat <<'EOP'
+<%= @config[:client_key] %>
+EOP
+) > /tmp/client.pem
+awk NF /tmp/client.pem > /etc/chef/client.pem
+rm /tmp/client.pem
+chmod 0600 /etc/chef/client.pem
+<%- else %>
+(
+cat <<'EOP'
+<%= validation_key %>
+EOP
+) > /tmp/validation.pem
+awk NF /tmp/validation.pem > /etc/chef/validation.pem
+rm /tmp/validation.pem
+chmod 0600 /etc/chef/validation.pem
+<% end -%>
+
+<% if @chef_config[:encrypted_data_bag_secret] -%>
+(
+cat <<'EOP'
+<%= encrypted_data_bag_secret %>
+EOP
+) > /tmp/encrypted_data_bag_secret
+awk NF /tmp/encrypted_data_bag_secret > /etc/chef/encrypted_data_bag_secret
+rm /tmp/encrypted_data_bag_secret
+chmod 0600 /etc/chef/encrypted_data_bag_secret
+<% end -%>
+
+<%# Generate Ohai Hints -%>
+<% unless @chef_config[:knife][:hints].nil? || @chef_config[:knife][:hints].empty? -%>
+mkdir -p /etc/chef/ohai/hints
+
+<% @chef_config[:knife][:hints].each do |name, hash| -%>
+(
+cat <<'EOP'
+<%= hash.to_json %>
+EOP
+) > /etc/chef/ohai/hints/<%= name %>.json
+<% end -%>
+<% end -%>
+
+(
+cat <<'EOP'
+<%= config_content %>
+<%= @config[:computer].chef_client_script_content %>
+EOP
+) > /etc/chef/client.rb
+
+(
+cat <<'EOP'
+<%= { "run_list" => @run_list, "cluster_name" => @config[:server].cluster_name, "facet_name" => @config[:server].facet_name, "facet_index" => @config[:server].index }.to_json %>
+EOP
+) > /etc/chef/first-boot.json
+
+<%= start_chef %>'

data/lib/chef/knife/cluster_bootstrap.rb

@@ -59,7 +59,7 @@ class Chef
       end
 
       def perform_execution(target)
-        reconcile_chef_config(target)
+        ensure_common_environment(target)
         # Execute across all servers in parallel
         Ironfan.parallel(target.values) {|computer| run_bootstrap(computer)}
 #         threads = target.servers.map{ |server| Thread.new(server) { |svr| run_bootstrap(svr, svr.public_hostname) } }
@@ -70,18 +70,6 @@ class Chef
         confirm_or_exit("Are you absolutely certain that you want to perform this action? (Type 'Yes' to confirm) ", 'Yes')
       end
 
-    protected
-
-      def reconcile_chef_config(target)
-        environments = target.environments
-        if environments.length > 1
-          ui.error "You cannot bootstrap machines in multiple chef environments: got #{environments.inspect} from #{target.map(&:name)}"
-          ui.error "Re-run this command on each subgroup of machines that share an environment"
-          raise StandardError, "Cannot bootstrap multiple chef environments"
-        end
-        Chef::Config[:environment] = environments.first
-      end
-
     end
   end
 end

data/lib/chef/knife/cluster_launch.rb

@@ -71,6 +71,13 @@ class Chef
 
         die("", "#{ui.color("All computers are running -- not launching any.",:blue)}", "", 1) if target.empty?
 
+        # If a bootstrap was requested, ensure that we will be able to perform the
+        # bootstrap *before* trying to launch all of the servers in target. This
+        # will save the user a lot of time if they've made a configuration mistake
+        if config[:bootstrap]
+          ensure_common_environment(target)
+        end
+
         # Pre-populate information in chef
         section("Syncing to chef")
         target.save :providers => :chef

data/lib/chef/knife/cluster_ssh.rb

@@ -50,6 +50,7 @@ class Chef
 #        config[:identity_file] ||= target.ssh_identity_file
 
 #         @action_nodes = target.chef_nodes
+        target = target.select {|t| not t.bogus? }
         addresses = target.map {|c| c.machine.public_hostname }.compact
 
         (ui.fatal("No nodes returned from search!"); exit 10) if addresses.nil? || addresses.length == 0

data/lib/chef/knife/cluster_sync.rb

@@ -41,11 +41,9 @@ class Chef
 
 
       def relevant?(computer)
-        if config[:sync_all]
-          not computer.bogus?
-        else
-          computer.created? or computer.node?
-        end
+        return false    if computer.bogus?
+        return true     if config[:sync_all]
+        computer.created? or computer.node?
       end
 
       def perform_execution(target)

data/lib/chef/knife/ironfan_knife_common.rb

@@ -236,5 +236,18 @@ module Ironfan
         extend ClassMethods
       end
     end
+
+    protected
+
+    def ensure_common_environment(target)
+      environments = target.environments
+      if environments.length > 1
+        ui.error "You cannot bootstrap machines in multiple chef environments: got #{environments.inspect} from #{target.map(&:name)}"
+        ui.error "Re-run this command on each subgroup of machines that share an environment"
+        raise StandardError, "Cannot bootstrap multiple chef environments"
+      end
+      Chef::Config[:environment] = environments.first
+    end
+
   end
 end

data/lib/ironfan/broker/computer.rb

@@ -301,7 +301,7 @@ module Ironfan
       def aggregate
         computers = self
         provider_keys = values.map {|c| c.chosen_providers({ :providers => :iaas})}.flatten.uniq
-        providers     = provider_keys.map { |pk| values.map { |c| c.providers[pk] } }.flatten.uniq
+        providers     = provider_keys.map { |pk| values.map { |c| c.providers[pk] } }.flatten.compact.uniq
         providers.each { |p| p.aggregate! computers }
       end
 

data/spec/chef/cluster_launch_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+ironfan_go!
+require 'chef/knife/cluster_launch'
+
+describe Chef::Knife::ClusterLaunch do
+  let(:cluster) do
+    Ironfan.load_cluster(:gunbai)
+  end
+
+  let(:target) do
+    Ironfan.broker.discover!(cluster)
+  end
+
+  let(:computers) do
+    Ironfan::Broker::Computers.receive(
+      MultiJson.load(
+        File.open(Pathname.path_to(:fixtures, 'gunbai_slice.json'))))
+  end
+
+  subject do
+    described_class.new(slice)
+  end
+
+  context 'getting slice' do
+    before do
+      subject.stub(:relevant?){ true }
+      subject.config[:yes] = true
+      subject.config[:bootstrap] = true
+    end
+    context 'full slice' do
+      let(:slice){ ['gunbai'] }
+      it 'fails if there are multiple environments' do
+        expect{ subject.run }.to raise_error("Cannot bootstrap multiple chef environments")
+      end
+    end
+
+  end
+end

metadata

@@ -2,7 +2,7 @@
 name: ironfan
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 4.5.1
+  version: 4.5.2
 platform: ruby
 authors: 
 - Infochimps
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-11-14 00:00:00 Z
+date: 2012-11-27 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: chef
@@ -164,6 +164,7 @@ files:
 - config/ubuntu12.04-ironfan.erb
 - ironfan.gemspec
 - lib/chef/knife/bootstrap/centos6.2-ironfan.erb
+- lib/chef/knife/bootstrap/chef-full-ironfan.erb
 - lib/chef/knife/bootstrap/ubuntu10.04-ironfan.erb
 - lib/chef/knife/bootstrap/ubuntu12.04-ironfan.erb
 - lib/chef/knife/cluster_bootstrap.rb
@@ -246,6 +247,7 @@ files:
 - notes/walkthrough-hadoop.md
 - notes/walkthrough-web.md
 - spec/chef/cluster_bootstrap_spec.rb
+- spec/chef/cluster_launch_spec.rb
 - spec/fixtures/ec2/elb/snakeoil.crt
 - spec/fixtures/ec2/elb/snakeoil.key
 - spec/fixtures/gunbai.rb
@@ -253,7 +255,6 @@ files:
 - spec/integration/minimal-chef-repo/chefignore
 - spec/integration/minimal-chef-repo/environments/_default.json
 - spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb
-- spec/integration/minimal-chef-repo/knife/credentials/knife-user-ironfantester.rb
 - spec/integration/minimal-chef-repo/knife/knife.rb
 - spec/integration/minimal-chef-repo/roles/systemwide.rb
 - spec/integration/spec/elb_build_spec.rb
@@ -281,7 +282,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -4226427612594670863
+      hash: -2019769305702491484
       segments: 
       - 0
       version: "0"
@@ -305,7 +306,6 @@ test_files:
 - spec/integration/minimal-chef-repo/environments/_default.json
 - spec/integration/minimal-chef-repo/chefignore
 - spec/integration/minimal-chef-repo/knife/knife.rb
-- spec/integration/minimal-chef-repo/knife/credentials/knife-user-ironfantester.rb
 - spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb
 - spec/integration/spec/simple_cluster_spec.rb
 - spec/integration/spec/elb_build_spec.rb
@@ -314,6 +314,7 @@ test_files:
 - spec/ironfan/ec2/cloud_provider_spec.rb
 - spec/ironfan/ec2/elb_spec.rb
 - spec/ironfan/ec2/security_group_spec.rb
+- spec/chef/cluster_launch_spec.rb
 - spec/chef/cluster_bootstrap_spec.rb
 - spec/fixtures/gunbai_slice.json
 - spec/fixtures/ec2/elb/snakeoil.key

data/spec/integration/minimal-chef-repo/knife/credentials/knife-user-ironfantester.rb

@@ -1,9 +0,0 @@
-Chef::Config.instance_eval do
-  organization            "infochimps_test"
-  chef_server_url         "https://api.opscode.com/organizations/#{organization}"
-  knife[:aws_account_id]         = "346633265723"
-
-  knife[:aws_access_key_id]      = "AKIAIFPVHWHGTQDXOF6Q"
-  knife[:aws_secret_access_key]  = "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"
-end
-