ironfan 3.1.7 → 3.2.2

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+# v3.2.0: First refactoring pass in preparation for multi-cloud support
+* Rebuilt the internal models of Ironfan to use gorillib's Field/Model/Builder architecture.
+* [#145](https://github.com/infochimps-labs/ironfan/pull/145): node attribues should be 'normal', not 'override' -- they don't show up as printed on the node otherwise
+* [#144](https://github.com/infochimps-labs/ironfan/pull/144): knife cluster ssh sets exit status based on commands' exit status
+* [#139](https://github.com/infochimps-labs/ironfan/pull/139): Fix detection of hostname in cluster_launch
+
+# v3.1.6
+* [#136](https://github.com/infochimps-labs/ironfan/pull/136): Basic support for VPC
+* cleanup of gemspec version constraints, including move to chef 0.10.10
+* CentOS compatibility tweaks to several cookbooks
+
 # v3.1.5: A bug with quoting in bootstrap scripts
 
 * Bootstrap scripts with env vars were being interpolated twice, so the bootstrap script quoting was changed to use `'` (single-quote) semantics. I am a bit troubled, because I can't tell when this regression happened. I believe I introduced it while merging @gpaco's bootstrap scripts and didn't notice because the bug strikes most forcefully within the fenced-off 'Is ruby installed?' part. But there's a chance I just screwed over chef 0.10.04 users.

data/Gemfile

@@ -3,27 +3,30 @@ source "http://rubygems.org"
 gem 'chef',          ">= 0.10.4"
 gem 'fog',           "~> 1.2"
 gem 'formatador',    "~> 0.2"
-gem 'gorillib',      "~> 0.1"
+gem 'gorillib',      "~> 0.4.2"
 
 group :development do
-  gem 'bundler',     ">= 1.1"
+  gem 'bundler',     "~> 1.1"
+  gem 'rake'
+  gem 'rspec',       "~> 2.8"
+  gem 'yard',        ">= 0.7"
+  #
+  gem 'redcarpet',   ">= 2.1"
+  gem 'oj',          ">= 1.2"
+  gem 'json',                  :platform => :jruby
+end
+
+group :support do
   gem 'jeweler',     ">= 1.6"
-  gem 'rspec',       "~> 2.5"
-  gem 'yard',        ">= 0.6"
-  gem 'redcarpet',   "~> 2"
+  gem 'pry'
+  gem 'grit'
 end
 
 group :test do
-  gem 'rcov',        ">= 0.9.9", :platform => :ruby_18
   gem 'simplecov',   ">= 0.5",   :platform => :ruby_19
   #
-  gem 'ruby_gntp'
   gem 'guard',         "~> 1"
   gem 'guard-rspec'
   gem 'guard-yard'
-end
-
-group :support do
-  gem 'pry'
-  gem 'grit'
+  gem 'ruby_gntp'
 end

data/Rakefile

@@ -21,7 +21,7 @@
 require 'rubygems' unless defined?(Gem)
 require 'bundler'
 begin
-  Bundler.setup(:default, :development)
+  Bundler.setup(:default, :development, :support)
 rescue Bundler::BundlerError => e
   $stderr.puts e.message
   $stderr.puts "Run `bundle install` to install missing gems"

data/VERSION

@@ -1 +1 @@
-3.1.7
+3.2.2

data/config/ubuntu10.04-ironfan.erb

@@ -107,6 +107,16 @@ EOP
 awk NF /tmp/knife-bootstrap/validation.pem > /etc/chef/validation.pem
 <%- end %>
 
+<% if @chef_config[:encrypted_data_bag_secret] -%>
+(
+cat <<'EOP'
+<%= encrypted_data_bag_secret %>
+EOP
+) > /tmp/encrypted_data_bag_secret
+awk NF /tmp/encrypted_data_bag_secret > /etc/chef/encrypted_data_bag_secret
+rm /tmp/encrypted_data_bag_secret
+<% end -%>
+
 echo -e "`date` \n\n**** \n**** Nuking our temp files:\n****\n"
 
 cd /tmp

data/config/ubuntu11.10-ironfan.erb

@@ -95,6 +95,16 @@ EOP
 awk NF /tmp/knife-bootstrap/validation.pem > /etc/chef/validation.pem
 <%- end %>
 
+<% if @chef_config[:encrypted_data_bag_secret] -%>
+(
+cat <<'EOP'
+<%= encrypted_data_bag_secret %>
+EOP
+) > /tmp/encrypted_data_bag_secret
+awk NF /tmp/encrypted_data_bag_secret > /etc/chef/encrypted_data_bag_secret
+rm /tmp/encrypted_data_bag_secret
+<% end -%>
+
 echo -e "`date` \n\n**** \n**** Nuking our temp files:\n****\n"
 
 cd /tmp

data/ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "3.1.7"
+  s.version = "3.2.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-07-24"
+  s.date = "2012-09-10"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [
@@ -62,7 +62,7 @@ Gem::Specification.new do |s|
     "lib/ironfan/compute.rb",
     "lib/ironfan/deprecated.rb",
     "lib/ironfan/discovery.rb",
-    "lib/ironfan/dsl_object.rb",
+    "lib/ironfan/dsl_builder.rb",
     "lib/ironfan/facet.rb",
     "lib/ironfan/fog_layer.rb",
     "lib/ironfan/private_key.rb",
@@ -71,37 +71,6 @@ Gem::Specification.new do |s|
     "lib/ironfan/server.rb",
     "lib/ironfan/server_slice.rb",
     "lib/ironfan/volume.rb",
-    "notes/Backup of ec2-pricing_and_capacity.numbers",
-    "notes/Home.md",
-    "notes/INSTALL-cloud_setup.md",
-    "notes/INSTALL.md",
-    "notes/Ironfan-Roadmap.md",
-    "notes/advanced-superpowers.md",
-    "notes/aws_servers.jpg",
-    "notes/aws_user_key.png",
-    "notes/cookbook-versioning.md",
-    "notes/core_concepts.md",
-    "notes/declaring_volumes.md",
-    "notes/design_notes-aspect_oriented_devops.md",
-    "notes/design_notes-ci_testing.md",
-    "notes/design_notes-cookbook_event_ordering.md",
-    "notes/design_notes-meta_discovery.md",
-    "notes/ec2-pricing_and_capacity.md",
-    "notes/ec2-pricing_and_capacity.numbers",
-    "notes/homebase-layout.txt",
-    "notes/knife-cluster-commands.md",
-    "notes/named-cloud-objects.md",
-    "notes/opscode_org_key.png",
-    "notes/opscode_user_key.png",
-    "notes/philosophy.md",
-    "notes/rake_tasks.md",
-    "notes/renamed-recipes.txt",
-    "notes/silverware.md",
-    "notes/style_guide.md",
-    "notes/tips_and_troubleshooting.md",
-    "notes/version-3_2.md",
-    "notes/walkthrough-hadoop.md",
-    "notes/walkthrough-web.md",
     "spec/ironfan/cluster_spec.rb",
     "spec/ironfan/facet_spec.rb",
     "spec/ironfan/server_slice_spec.rb",
@@ -115,9 +84,9 @@ Gem::Specification.new do |s|
   s.homepage = "http://infochimps.com/labs"
   s.licenses = ["apachev2"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.11"
+  s.rubygems_version = "1.8.24"
   s.summary = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
-  s.test_files = ["spec/ironfan/cluster_spec.rb", "spec/ironfan/facet_spec.rb", "spec/ironfan/server_slice_spec.rb", "spec/ironfan/server_spec.rb", "spec/ironfan_spec.rb", "spec/spec_helper/dummy_chef.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
+  s.test_files = ["spec/ironfan_spec.rb", "spec/spec_helper/dummy_chef.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/server_slice_spec.rb", "spec/ironfan/server_spec.rb", "spec/ironfan/facet_spec.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
@@ -126,33 +95,39 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<chef>, [">= 0.10.4"])
       s.add_runtime_dependency(%q<fog>, ["~> 1.2"])
       s.add_runtime_dependency(%q<formatador>, ["~> 0.2"])
-      s.add_runtime_dependency(%q<gorillib>, ["~> 0.1"])
-      s.add_development_dependency(%q<bundler>, [">= 1.1"])
-      s.add_development_dependency(%q<jeweler>, [">= 1.6"])
-      s.add_development_dependency(%q<rspec>, ["~> 2.5"])
-      s.add_development_dependency(%q<yard>, [">= 0.6"])
-      s.add_development_dependency(%q<redcarpet>, ["~> 2"])
+      s.add_runtime_dependency(%q<gorillib>, ["~> 0.4.2"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.1"])
+      s.add_development_dependency(%q<rake>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.8"])
+      s.add_development_dependency(%q<yard>, [">= 0.7"])
+      s.add_development_dependency(%q<redcarpet>, [">= 2.1"])
+      s.add_development_dependency(%q<oj>, [">= 1.2"])
+      s.add_development_dependency(%q<json>, [">= 0"])
     else
       s.add_dependency(%q<chef>, [">= 0.10.4"])
       s.add_dependency(%q<fog>, ["~> 1.2"])
       s.add_dependency(%q<formatador>, ["~> 0.2"])
-      s.add_dependency(%q<gorillib>, ["~> 0.1"])
-      s.add_dependency(%q<bundler>, [">= 1.1"])
-      s.add_dependency(%q<jeweler>, [">= 1.6"])
-      s.add_dependency(%q<rspec>, ["~> 2.5"])
-      s.add_dependency(%q<yard>, [">= 0.6"])
-      s.add_dependency(%q<redcarpet>, ["~> 2"])
+      s.add_dependency(%q<gorillib>, ["~> 0.4.2"])
+      s.add_dependency(%q<bundler>, ["~> 1.1"])
+      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["~> 2.8"])
+      s.add_dependency(%q<yard>, [">= 0.7"])
+      s.add_dependency(%q<redcarpet>, [">= 2.1"])
+      s.add_dependency(%q<oj>, [">= 1.2"])
+      s.add_dependency(%q<json>, [">= 0"])
     end
   else
     s.add_dependency(%q<chef>, [">= 0.10.4"])
     s.add_dependency(%q<fog>, ["~> 1.2"])
     s.add_dependency(%q<formatador>, ["~> 0.2"])
-    s.add_dependency(%q<gorillib>, ["~> 0.1"])
-    s.add_dependency(%q<bundler>, [">= 1.1"])
-    s.add_dependency(%q<jeweler>, [">= 1.6"])
-    s.add_dependency(%q<rspec>, ["~> 2.5"])
-    s.add_dependency(%q<yard>, [">= 0.6"])
-    s.add_dependency(%q<redcarpet>, ["~> 2"])
+    s.add_dependency(%q<gorillib>, ["~> 0.4.2"])
+    s.add_dependency(%q<bundler>, ["~> 1.1"])
+    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["~> 2.8"])
+    s.add_dependency(%q<yard>, [">= 0.7"])
+    s.add_dependency(%q<redcarpet>, [">= 2.1"])
+    s.add_dependency(%q<oj>, [">= 1.2"])
+    s.add_dependency(%q<json>, [">= 0"])
   end
 end
 

data/lib/chef/knife/bootstrap/centos6.2-ironfan.erb

@@ -89,6 +89,16 @@ EOP
 awk NF /tmp/knife-bootstrap/validation.pem > /etc/chef/validation.pem
 <%- end %>
 
+<% if @chef_config[:encrypted_data_bag_secret] -%>
+(
+cat <<'EOP'
+<%= encrypted_data_bag_secret %>
+EOP
+) > /tmp/encrypted_data_bag_secret
+awk NF /tmp/encrypted_data_bag_secret > /etc/chef/encrypted_data_bag_secret
+rm /tmp/encrypted_data_bag_secret
+<% end -%>
+
 echo -e "`date` \n\n**** \n**** Nuking our temp files:\n****\n"
 
 cd /tmp

data/lib/chef/knife/bootstrap/ubuntu10.04-ironfan.erb

@@ -107,6 +107,16 @@ EOP
 awk NF /tmp/knife-bootstrap/validation.pem > /etc/chef/validation.pem
 <%- end %>
 
+<% if @chef_config[:encrypted_data_bag_secret] -%>
+(
+cat <<'EOP'
+<%= encrypted_data_bag_secret %>
+EOP
+) > /tmp/encrypted_data_bag_secret
+awk NF /tmp/encrypted_data_bag_secret > /etc/chef/encrypted_data_bag_secret
+rm /tmp/encrypted_data_bag_secret
+<% end -%>
+
 echo -e "`date` \n\n**** \n**** Nuking our temp files:\n****\n"
 
 cd /tmp

data/lib/chef/knife/bootstrap/ubuntu11.10-ironfan.erb

@@ -95,6 +95,16 @@ EOP
 awk NF /tmp/knife-bootstrap/validation.pem > /etc/chef/validation.pem
 <%- end %>
 
+<% if @chef_config[:encrypted_data_bag_secret] -%>
+(
+cat <<'EOP'
+<%= encrypted_data_bag_secret %>
+EOP
+) > /tmp/encrypted_data_bag_secret
+awk NF /tmp/encrypted_data_bag_secret > /etc/chef/encrypted_data_bag_secret
+rm /tmp/encrypted_data_bag_secret
+<% end -%>
+
 echo -e "`date` \n\n**** \n**** Nuking our temp files:\n****\n"
 
 cd /tmp

data/lib/chef/knife/cluster_kick.rb

@@ -42,6 +42,11 @@ class Chef
         :description => "Where to find the pid file. Typically /var/run/chef/client.pid (init.d) or /etc/sv/chef-client/supervise/pid (runit)",
         :default     => "/etc/sv/chef-client/supervise/pid"
 
+      option :log_file,
+        :long        => "--log_file",
+        :description => "Where to find the log file. Typically /var/log/chef/client.log",
+        :default     => "/var/log/chef/client.log"
+
       def run
         @name_args = [ @name_args.join('-') ]
         config[:display_target] = true
@@ -62,7 +67,7 @@ if sudo  -p 'knife sudo password: ' service chef-client status ; then
 
 # running
 pid_file="<%= config[:pid_file] %>"
-log_file=/var/log/chef/client.log
+log_file=<%= config[:log_file] %>
 
 declare tail_pid
 
@@ -76,7 +81,7 @@ trap "on_exit" EXIT ERR
 pipe=/tmp/pipe-$$
 mkfifo $pipe
 
-tail -fn0 "$log_file" > $pipe &
+tail -Fn0 "$log_file" > $pipe &
 
 tail_pid=$!
 

data/lib/chef/knife/cluster_launch.rb

@@ -136,6 +136,9 @@ class Chef
       rescue Errno::ECONNREFUSED
         sleep 2
         false
+      rescue Errno::EHOSTUNREACH
+        sleep 2
+        false
       ensure
         tcp_socket && tcp_socket.close
       end

data/lib/chef/knife/cluster_ssh.rb

@@ -84,7 +84,7 @@ class Chef
       def ssh_command(command, subsession=nil)
         subsession ||= session
         command = fixup_sudo(command)
-        notifications = []
+        #
         subsession.open_channel do |ch|
           ch.request_pty
           ch.exec command do |ch, success|
@@ -101,13 +101,12 @@ class Chef
               exit_status = data.read_long
               if exit_status != 0
                 command_snippet = (command.length < 70) ? command : (command[0..45] + ' ... ' + command[-19..-1])
-                notifications << [ichannel[:host], "'#{command_snippet.gsub(/[\r\n]+/, "; ")}' terminated with error status #{exit_status}", :err]
+                has_problem ->{ print_data(ichannel[:host], "'#{command_snippet.gsub(/[\r\n]+/, "; ")}' terminated with error status #{exit_status}", :err) }
               end
             end
           end
         end
         session.loop
-        notifications.each{|args| print_data(*args) }
       end
 
       def cssh
@@ -133,6 +132,7 @@ class Chef
         end
 
         session.close
+        exit_if_unhealthy!
       end
 
     end

data/lib/chef/knife/ironfan_knife_common.rb

@@ -163,6 +163,27 @@ module Ironfan
       ui.info("")
     end
 
+    # list of problems encountered
+    attr_accessor :problems
+    # register that a problem was encountered
+    def has_problem(desc)
+      (@problems||=[]) << desc
+    end
+    # healthy if no problems
+    def healthy?() problems.blank? ; end
+
+    def exit_if_unhealthy!
+      return if healthy?
+      problems.each do |problem|
+        if problem.respond_to?(:call)
+          problem.call
+        else
+          ui.warn(problem)
+        end
+      end
+      exit(2) if not healthy?
+    end
+
     #
     # Announce a new section of tasks
     #

data/lib/chef/knife/ironfan_script.rb

@@ -57,6 +57,8 @@ module Ironfan
       ui.info("")
       ui.info "Finished! Current state:"
       display(target)
+      #
+      exit_if_unhealthy!
     end
 
     def perform_execution(target)

data/lib/ironfan/chef_layer.rb

@@ -73,12 +73,12 @@ module Ironfan
       @chef_client = cluster.find_client(fullname) || false
     end
 
-    # The chef node, if it already exists in the server.
-    # Use the 'ensure' method to create/update it.
-    def chef_node
-      return @chef_node unless @chef_node.nil?
-      @chef_node   = cluster.find_node(fullname) || false
-    end
+#     # The chef node, if it already exists in the server.
+#     # Use the 'ensure' method to create/update it.
+#     def chef_node
+#       return @chef_node unless @chef_node.nil?
+#       @chef_node   = cluster.find_node(fullname) || false
+#     end
 
     # true if chef client is created and discovered
     def chef_client?
@@ -193,15 +193,15 @@ module Ironfan
     end
 
     def sync_volume_attributes
-      composite_volumes.each do |vol_name, vol|
+      volumes.each_pair do |vol_name, vol|
         chef_node.normal[:volumes] ||= Mash.new
-        chef_node.normal[:volumes][vol_name] = vol.to_mash.compact
+        chef_node.normal[:volumes][vol_name] = vol.attributes.compact
       end
     end
 
     def set_chef_node_attributes
       step("  setting node runlist and essential attributes")
-      @chef_node.run_list = Chef::RunList.new(*@settings[:run_list])
+      @chef_node.run_list = Chef::RunList.new(*combined_run_list)
       @chef_node.normal[:organization] = organization if organization
       @chef_node.normal[:permanent]    = cloud.permanent if cloud.permanent
       @chef_node.normal[:cluster_name] = cluster_name