ironfan 4.12.3 → 5.0.0

data/Gemfile

@@ -6,6 +6,7 @@ gem 'excon',         "~> 0.21.0"        # v0.22 breaks EC2 calls
 gem 'formatador',    "~> 0.2"
 gem 'gorillib',      "~> 0.5.0"
 gem 'rbvmomi'
+gem 'diff-lcs',      "~> 1.2.5"
 
 # Everything in the world is being a stupid dick about JSON versions. Pin it
 #   to the one that doesn't seem to angrify everyone.

data/Gemfile.lock

@@ -33,7 +33,7 @@ GEM
     configliere (0.4.18)
       highline (>= 1.5.2)
       multi_json (>= 1.1)
-    diff-lcs (1.2.4)
+    diff-lcs (1.2.5)
     erubis (2.7.0)
     excon (0.21.0)
     ffi (1.9.0)
@@ -132,14 +132,14 @@ GEM
     redcarpet (2.3.0)
     rest-client (1.6.7)
       mime-types (>= 1.16)
-    rspec (2.13.0)
-      rspec-core (~> 2.13.0)
-      rspec-expectations (~> 2.13.0)
-      rspec-mocks (~> 2.13.0)
-    rspec-core (2.13.1)
-    rspec-expectations (2.13.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
       diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.13.1)
+    rspec-mocks (2.14.4)
     ruby-debug-base19 (0.11.25)
       columnize (>= 0.3.1)
       linecache19 (>= 0.5.11)
@@ -174,6 +174,7 @@ DEPENDENCIES
   bundler (~> 1.0)
   chef (~> 10.16)
   chef-zero
+  diff-lcs (~> 1.2.5)
   excon (~> 0.21.0)
   fog (~> 1.2)
   formatador (~> 0.2)

data/VERSION

@@ -1 +1 @@
-4.12.3
+5.0.0

data/ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "4.12.3"
+  s.version = "5.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2013-10-10"
+  s.date = "2013-12-11"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [
@@ -44,6 +44,7 @@ Gem::Specification.new do |s|
     "lib/chef/knife/bootstrap/ubuntu10.04-ironfan.erb",
     "lib/chef/knife/bootstrap/ubuntu12.04-ironfan.erb",
     "lib/chef/knife/cluster_bootstrap.rb",
+    "lib/chef/knife/cluster_diff.rb",
     "lib/chef/knife/cluster_kick.rb",
     "lib/chef/knife/cluster_kill.rb",
     "lib/chef/knife/cluster_launch.rb",
@@ -55,8 +56,11 @@ Gem::Specification.new do |s|
     "lib/chef/knife/cluster_start.rb",
     "lib/chef/knife/cluster_stop.rb",
     "lib/chef/knife/cluster_sync.rb",
+    "lib/chef/knife/environment_from_realm.rb",
     "lib/chef/knife/ironfan_knife_common.rb",
     "lib/chef/knife/ironfan_script.rb",
+    "lib/gorillib/diff.rb",
+    "lib/gorillib/nil_check_delegate.rb",
     "lib/gorillib/resolution.rb",
     "lib/ironfan.rb",
     "lib/ironfan/broker.rb",
@@ -67,6 +71,7 @@ Gem::Specification.new do |s|
     "lib/ironfan/dsl.rb",
     "lib/ironfan/dsl/cloud.rb",
     "lib/ironfan/dsl/cluster.rb",
+    "lib/ironfan/dsl/component.rb",
     "lib/ironfan/dsl/compute.rb",
     "lib/ironfan/dsl/ec2.rb",
     "lib/ironfan/dsl/facet.rb",
@@ -78,6 +83,7 @@ Gem::Specification.new do |s|
     "lib/ironfan/dsl/volume.rb",
     "lib/ironfan/dsl/vsphere.rb",
     "lib/ironfan/headers.rb",
+    "lib/ironfan/plugin/base.rb",
     "lib/ironfan/provider.rb",
     "lib/ironfan/provider/chef.rb",
     "lib/ironfan/provider/chef/client.rb",
@@ -149,20 +155,26 @@ Gem::Specification.new do |s|
     "spec/integration/spec_helper.rb",
     "spec/integration/spec_helper/launch_cluster.rb",
     "spec/ironfan/cluster_spec.rb",
+    "spec/ironfan/diff_spec.rb",
+    "spec/ironfan/dsl_spec.rb",
     "spec/ironfan/ec2/cloud_provider_spec.rb",
     "spec/ironfan/ec2/elb_spec.rb",
     "spec/ironfan/ec2/security_group_spec.rb",
+    "spec/ironfan/manifest_spec.rb",
+    "spec/ironfan/plugin_spec.rb",
+    "spec/ironfan/realm_spec.rb",
     "spec/spec_helper.rb",
     "spec/spec_helper/dummy_chef.rb",
+    "spec/spec_helper/dummy_diff_drawer.rb",
     "spec/test_config.rb",
     "tasks/chef_config.rake"
   ]
   s.homepage = "http://infochimps.com/labs"
   s.licenses = ["apachev2"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.23"
+  s.rubygems_version = "1.8.25"
   s.summary = "Infochimps' lightweight cloud orchestration toolkit, built on top of Chef."
-  s.test_files = ["spec/fixtures/gunbai.rb", "spec/fixtures/ec2/elb/snakeoil.key", "spec/fixtures/ec2/elb/snakeoil.crt", "spec/fixtures/gunbai_slice.json", "spec/fixtures/knife/knife.rb", "spec/test_config.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/elb_spec.rb", "spec/ironfan/ec2/security_group_spec.rb", "spec/ironfan/cluster_spec.rb", "spec/spec_helper.rb", "spec/spec_helper/dummy_chef.rb", "spec/chef/cluster_bootstrap_spec.rb", "spec/chef/cluster_launch_spec.rb", "spec/integration/spec/elb_build_spec.rb", "spec/integration/spec/simple_cluster_spec.rb", "spec/integration/spec_helper.rb", "spec/integration/minimal-chef-repo/chefignore", "spec/integration/minimal-chef-repo/roles/systemwide.rb", "spec/integration/minimal-chef-repo/environments/_default.json", "spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb", "spec/integration/minimal-chef-repo/knife/knife.rb", "spec/integration/spec_helper/launch_cluster.rb"]
+  s.test_files = ["spec/chef/cluster_bootstrap_spec.rb", "spec/chef/cluster_launch_spec.rb", "spec/fixtures/ec2/elb/snakeoil.crt", "spec/fixtures/ec2/elb/snakeoil.key", "spec/fixtures/gunbai.rb", "spec/fixtures/gunbai_slice.json", "spec/fixtures/knife/knife.rb", "spec/integration/minimal-chef-repo/chefignore", "spec/integration/minimal-chef-repo/environments/_default.json", "spec/integration/minimal-chef-repo/knife/credentials/knife-org.rb", "spec/integration/minimal-chef-repo/knife/knife.rb", "spec/integration/minimal-chef-repo/roles/systemwide.rb", "spec/integration/spec/elb_build_spec.rb", "spec/integration/spec/simple_cluster_spec.rb", "spec/integration/spec_helper/launch_cluster.rb", "spec/integration/spec_helper.rb", "spec/ironfan/cluster_spec.rb", "spec/ironfan/diff_spec.rb", "spec/ironfan/dsl_spec.rb", "spec/ironfan/ec2/cloud_provider_spec.rb", "spec/ironfan/ec2/elb_spec.rb", "spec/ironfan/ec2/security_group_spec.rb", "spec/ironfan/manifest_spec.rb", "spec/ironfan/plugin_spec.rb", "spec/ironfan/realm_spec.rb", "spec/spec_helper/dummy_chef.rb", "spec/spec_helper/dummy_diff_drawer.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
@@ -174,6 +186,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<formatador>, ["~> 0.2"])
       s.add_runtime_dependency(%q<gorillib>, ["~> 0.5.0"])
       s.add_runtime_dependency(%q<rbvmomi>, [">= 0"])
+      s.add_runtime_dependency(%q<diff-lcs>, ["~> 1.2.5"])
       s.add_runtime_dependency(%q<json>, ["= 1.5.4"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0"])
       s.add_development_dependency(%q<rake>, [">= 0"])
@@ -188,6 +201,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<formatador>, ["~> 0.2"])
       s.add_dependency(%q<gorillib>, ["~> 0.5.0"])
       s.add_dependency(%q<rbvmomi>, [">= 0"])
+      s.add_dependency(%q<diff-lcs>, ["~> 1.2.5"])
       s.add_dependency(%q<json>, ["= 1.5.4"])
       s.add_dependency(%q<bundler>, ["~> 1.0"])
       s.add_dependency(%q<rake>, [">= 0"])
@@ -203,6 +217,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<formatador>, ["~> 0.2"])
     s.add_dependency(%q<gorillib>, ["~> 0.5.0"])
     s.add_dependency(%q<rbvmomi>, [">= 0"])
+    s.add_dependency(%q<diff-lcs>, ["~> 1.2.5"])
     s.add_dependency(%q<json>, ["= 1.5.4"])
     s.add_dependency(%q<bundler>, ["~> 1.0"])
     s.add_dependency(%q<rake>, [">= 0"])

data/lib/chef/knife/cluster_diff.rb

@@ -0,0 +1,95 @@
+require 'gorillib/model/serialization'
+require 'gorillib/nil_check_delegate'
+require 'yaml'
+require_relative '../../gorillib/diff'
+
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path('ironfan_knife_common', File.dirname(File.realdirpath(__FILE__)))
+require 'yaml'
+
+class Chef
+  class Knife
+    class ClusterDiff < Knife
+      include Ironfan::KnifeCommon
+      deps do
+        Ironfan::KnifeCommon.load_deps
+      end
+
+      banner "knife cluster diff        CLUSTER[-FACET[-INDEXES]] (options) - differences between a cluster and its realization"
+
+      option :cache_file,
+        :long        => "--cache_file FILE",
+        :description => "file to load chef information from, for testing purposes"
+
+      def _run
+        load_ironfan
+        die(banner) if @name_args.empty?
+        configure_dry_run
+
+        @test_chef_data = 
+          if config.has_key? :cache_file
+            Hash[open(config.fetch(:cache_file)).readlines.map{|line| datum = MultiJson.load(line); [datum['name'], datum]}]
+          else
+            {}
+          end
+        
+        # Load the cluster/facet/slice/whatever
+        target = get_slice(* @name_args)
+
+        exit(1) if self.class.mismatches?(target)
+      end
+
+      def self.mismatches?(target)
+        target.map do |computer|
+          local_manifest = computer.server.to_machine_manifest
+          remote_manifest = Ironfan::Dsl::MachineManifest.from_computer(computer)
+          display_diff(local_manifest, remote_manifest)
+          local_manifest != remote_manifest
+        end.to_a.any?
+      end
+      
+    private
+
+      def self.node_name(manifest)
+        "#{manifest.cluster_name}-#{manifest.facet_name}-#{manifest.name}"
+      end
+
+      def self.display_diff(local_manifest, remote_manifest)
+        header("diffing manifests: local #{node_name(local_manifest)} <-> remote #{node_name(remote_manifest)}")
+        differ.display_diff(local_manifest.to_comparable, remote_manifest.to_comparable)
+      end
+
+      #---------------------------------------------------------------------------------------------
+
+      def self.differ
+        Gorillib::DiffFormatter.new(left: :local,
+                                    right: :remote,
+                                    stream: $stdout,
+                                    indentation: 4)
+      end
+
+      def self.header str
+        $stdout.puts("  #{'-' * 80}")
+        $stdout.puts("  #{str}")
+        $stdout.puts("  #{'-' * 80}")
+      end
+    end
+  end
+end

data/lib/chef/knife/cluster_launch.rb

@@ -55,7 +55,7 @@ class Chef
         :boolean     => true,
         :default     => false
 
-      def run
+      def _run
         load_ironfan
         die(banner) if @name_args.empty?
         configure_dry_run

data/lib/chef/knife/cluster_list.rb

@@ -36,11 +36,13 @@ class Chef
         :default     => false,
         :boolean     => true
       
-      def run
+      def _run
         load_ironfan
         configure_dry_run
+        Ironfan.load_cluster_files
 
-        data = Ironfan.cluster_filenames.map do |name, path|
+        data = Ironfan.clusters.values.map do |cluster|
+          name, path = [cluster.name, cluster.source_file]
           as_table = { :cluster => name, :path => path }
           if config[:facets]
             facets = Ironfan.load_cluster(name).facets.to_a.map(&:name).join(', ')

data/lib/chef/knife/cluster_pry.rb

@@ -35,7 +35,7 @@ class Chef
         :default     => true,
         :boolean     => true
 
-      def run
+      def _run
         load_ironfan
         die(banner) if @name_args.empty?
         configure_dry_run

data/lib/chef/knife/cluster_show.rb

@@ -17,6 +17,7 @@
 #
 
 require File.expand_path('ironfan_knife_common', File.dirname(File.realdirpath(__FILE__)))
+require 'yaml'
 
 class Chef
   class Knife
@@ -34,7 +35,7 @@ class Chef
         :default     => true,
         :boolean     => true
 
-      def run
+      def _run
         with_verbosity(1){ config[:include_terminated] = true }
         load_ironfan
         die(banner) if @name_args.empty?
@@ -46,6 +47,7 @@ class Chef
         dump_command_config
         dump_chef_config
         #
+
         target.each do |computer|
           dump_computer(computer)
         end
@@ -60,6 +62,11 @@ class Chef
         header = "Computer #{computer.name} (#{computer.class})"
         with_verbosity 1 do
           Chef::Log.info(header)
+
+          # Terminated instances don't have servers.
+          unless computer.server.nil?
+            Chef::Log.info(computer.server.canonical_machine_manifest_hash.to_yaml)
+          end
         end
         with_verbosity 2 do
           dump(header, computer.to_wire)
@@ -85,10 +92,6 @@ class Chef
         Chef::Log.info( MultiJson.dump(hsh, pretty: true ) )
       end
 
-      def with_verbosity(num)
-        yield if config[:verbosity] >= num
-      end
-
     end
   end
 end

data/lib/chef/knife/cluster_ssh.rb

@@ -114,7 +114,7 @@ class Chef
         exec "cssh "+session.servers_for.map{|server| server.user ? "#{server.user}@#{server.host}" : server.host}.join(" ")
       end
 
-      def run
+      def _run
         load_ironfan
         die(banner) if @name_args.empty?
         extend Chef::Mixin::Command

data/lib/chef/knife/environment_from_realm.rb

@@ -0,0 +1,60 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path('ironfan_knife_common', File.dirname(File.realdirpath(__FILE__)))
+require 'yaml'
+
+class Chef
+  class Knife
+    class EnvironmentFromRealm < Knife
+      include Ironfan::KnifeCommon
+      deps do
+        Ironfan::KnifeCommon.load_deps
+      end
+
+      banner "knife environment from realm        realm (options) - syncs a realm's environment"
+
+      option :dry_run,
+        :long        => "--dry-run",
+        :description => "Don't really run, just use mock calls",
+        :boolean     => true,
+        :default     => false
+
+      def _run
+        load_ironfan
+        die(banner) unless @name_args.size == 1
+        configure_dry_run
+
+        # Load the cluster/facet/slice/whatever
+        target = Ironfan.load_realm(* @name_args)
+
+        env = Chef::Environment.new.tap do |env|
+          env.name target.environment
+          env.description "Ironfan-created environment for #{target.name} realm"
+          Chef::Log.info "pinning cookbooks in #{target.name} realm"
+          target.cookbook_reqs.each do |cookbook, version|
+            Chef::Log.info "  pinning cookbook #{cookbook} #{version}"
+            env.cookbook cookbook, version
+          end
+        end
+
+        env.save unless config[:dry_run]
+      end
+    end
+  end
+end

data/lib/chef/knife/ironfan_knife_common.rb

@@ -22,6 +22,25 @@ module Ironfan
       self.broker         = Ironfan.broker
     end
 
+    def run()
+      gemfile_v = gemfile(@name_args.first.to_s.split(/[_-]/).first)
+
+      if ENV['BUNDLE_GEMFILE'] == gemfile_v
+        _run
+      elsif not File.exist?(gemfile_v)
+        ui.info("no realm-specific Gemfile found. using default Gemfile.")
+        _run
+      else
+        cmd = "bundle exec knife #{ARGV.join(' ')}"
+        ui.info("re-running `#{cmd}` with BUNDLE_GEMFILE=#{gemfile_v}")
+        return Bundler.clean_exec({'BUNDLE_GEMFILE' => gemfile_v}, cmd)
+      end
+    end
+
+    def gemfile(realm_or_cluster_name)
+      "Gemfile.#{realm_or_cluster_name}"
+    end
+
     #
     # A slice of a cluster:
     #
@@ -249,5 +268,8 @@ module Ironfan
       Chef::Config[:environment] = environments.first
     end
 
+    def with_verbosity(num)
+      yield if config[:verbosity] >= num
+    end
   end
 end

data/lib/chef/knife/ironfan_script.rb

@@ -36,7 +36,7 @@ module Ironfan
       :description => "Skip confirmation prompts on risky actions.",
       :boolean     => true
 
-    def run
+    def _run
       load_ironfan
 
       die(banner) if @name_args.empty?