ironfan 3.2.2 → 4.0.0

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# v4.0.0:  Major refactoring to allow multicloud support
+* First pass at a provider plugin API, with EC2 as the working example.
+* Removed role_implications: these can be handled by explicit ec2.security_group calls for now. See https://github.com/infochimps-labs/ironfan/wiki/Upgrading-to-v4 for more details.
+* Added default_cloud flag to cloud statement (sets that cloud as the default one, if there are multiple clouds available), added use_cloud statement to compute components (cluster/facet/server) which overrides those defaults. There are plans for a command-line override, as well.
+
 # v3.2.0: First refactoring pass in preparation for multi-cloud support
 * Rebuilt the internal models of Ironfan to use gorillib's Field/Model/Builder architecture.
 * [#145](https://github.com/infochimps-labs/ironfan/pull/145): node attribues should be 'normal', not 'override' -- they don't show up as printed on the node otherwise

data/VERSION

@@ -1 +1 @@
-3.2.2
+4.0.0

data/ironfan.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ironfan"
-  s.version = "3.2.2"
+  s.version = "4.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Infochimps"]
-  s.date = "2012-09-10"
+  s.date = "2012-09-06"
   s.description = "Ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
   s.email = "coders@infochimps.com"
   s.extra_rdoc_files = [
@@ -49,28 +49,41 @@ Gem::Specification.new do |s|
     "lib/chef/knife/cluster_start.rb",
     "lib/chef/knife/cluster_stop.rb",
     "lib/chef/knife/cluster_sync.rb",
-    "lib/chef/knife/cluster_vagrant.rb",
     "lib/chef/knife/ironfan_knife_common.rb",
     "lib/chef/knife/ironfan_script.rb",
-    "lib/chef/knife/vagrant/ironfan_environment.rb",
-    "lib/chef/knife/vagrant/ironfan_provisioners.rb",
-    "lib/chef/knife/vagrant/skeleton_vagrantfile.rb",
+    "lib/gorillib/resolution.rb",
     "lib/ironfan.rb",
-    "lib/ironfan/chef_layer.rb",
-    "lib/ironfan/cloud.rb",
-    "lib/ironfan/cluster.rb",
-    "lib/ironfan/compute.rb",
+    "lib/ironfan/broker.rb",
+    "lib/ironfan/broker/computer.rb",
+    "lib/ironfan/broker/drive.rb",
+    "lib/ironfan/builder.rb",
     "lib/ironfan/deprecated.rb",
-    "lib/ironfan/discovery.rb",
-    "lib/ironfan/dsl_builder.rb",
-    "lib/ironfan/facet.rb",
-    "lib/ironfan/fog_layer.rb",
-    "lib/ironfan/private_key.rb",
-    "lib/ironfan/role_implications.rb",
-    "lib/ironfan/security_group.rb",
-    "lib/ironfan/server.rb",
-    "lib/ironfan/server_slice.rb",
-    "lib/ironfan/volume.rb",
+    "lib/ironfan/dsl.rb",
+    "lib/ironfan/dsl/cloud.rb",
+    "lib/ironfan/dsl/cluster.rb",
+    "lib/ironfan/dsl/compute.rb",
+    "lib/ironfan/dsl/ec2.rb",
+    "lib/ironfan/dsl/facet.rb",
+    "lib/ironfan/dsl/role.rb",
+    "lib/ironfan/dsl/server.rb",
+    "lib/ironfan/dsl/virtualbox.rb",
+    "lib/ironfan/dsl/volume.rb",
+    "lib/ironfan/headers.rb",
+    "lib/ironfan/provider.rb",
+    "lib/ironfan/provider/chef.rb",
+    "lib/ironfan/provider/chef/client.rb",
+    "lib/ironfan/provider/chef/node.rb",
+    "lib/ironfan/provider/chef/role.rb",
+    "lib/ironfan/provider/ec2.rb",
+    "lib/ironfan/provider/ec2/ebs_volume.rb",
+    "lib/ironfan/provider/ec2/elastic_ip.rb",
+    "lib/ironfan/provider/ec2/key_pair.rb",
+    "lib/ironfan/provider/ec2/machine.rb",
+    "lib/ironfan/provider/ec2/placement_group.rb",
+    "lib/ironfan/provider/ec2/security_group.rb",
+    "lib/ironfan/provider/virtualbox.rb",
+    "lib/ironfan/provider/virtualbox/machine.rb",
+    "lib/ironfan/requirements.rb",
     "spec/ironfan/cluster_spec.rb",
     "spec/ironfan/facet_spec.rb",
     "spec/ironfan/server_slice_spec.rb",

data/lib/chef/knife/cluster_kick.rb

@@ -65,30 +65,30 @@ sudo -p 'knife sudo password: ' true
 
 if sudo  -p 'knife sudo password: ' service chef-client status ; then
 
-# running
-pid_file="<%= config[:pid_file] %>"
-log_file=<%= config[:log_file] %>
+  # running
+  pid_file="<%= config[:pid_file] %>"
+  log_file=<%= config[:log_file] %>
 
-declare tail_pid
+  declare tail_pid
 
-on_exit() {
-  rm -f $pipe
-  [ -n "$tail_pid" ] && kill $tail_pid
-}
+  on_exit() {
+    rm -f $pipe
+  }
 
-trap "on_exit" EXIT ERR
+  trap "on_exit" EXIT ERR
 
-pipe=/tmp/pipe-$$
-mkfifo $pipe
+  pipe=/tmp/pipe-$$
+  mkfifo $pipe
 
-tail -Fn0 "$log_file" > $pipe &
+  tail -Fn0 "$log_file" > $pipe &
 
-tail_pid=$!
-
-pid="$(sudo -p 'knife sudo password: ' cat $pid_file)"
-sudo -p 'knife sudo password: ' kill -USR1 "$pid"
-sed -r "/(ERROR: Sleeping for [0-9]+ seconds before trying again|INFO: Report handlers complete)\$/{q}" $pipe
+  tail_pid=$!
 
+  pid="$(sudo -p 'knife sudo password: ' cat $pid_file)"
+  sudo -p 'knife sudo password: ' kill -USR1 "$pid"
+  GOOD_RESULT='INFO: Report handlers complete\$'
+  BAD_RESULT='ERROR: Sleeping for [0-9]+ seconds before trying again\$'
+  sed -r -e "/$GOOD_RESULT/{q 0}" -e"/$BAD_RESULT/{q 1}" $pipe
 else
   echo -e "****\n\nchef-client daemon not running, invoking chef-client directly\n\n****\n"
   sudo -p 'knife sudo password: ' chef-client

data/lib/chef/knife/cluster_kill.rb

@@ -46,26 +46,32 @@ class Chef
       # Execute every last mf'ing one of em
       def perform_execution(target)
         if config[:cloud]
-          section("Killing Cloud Machines")
-          target.select(&:in_cloud?).destroy
+          section("Killing Cloud Computers")
+          target.kill :providers => :iaas
         end
 
         if config[:chef]
           section("Killing Chef")
-          target.select(&:in_chef? ).delete_chef
+          target.kill :providers => :chef
         end
       end
 
       def display(target, *args, &block)
         super
-        ui.info Formatador.display_line("servers with [red]'permanent=true'[reset] ignored: [blue]#{target.select(&:permanent?).map(&:fullname).inspect}[reset]. (To kill, change 'permanent' to false, run knife cluster sync, and re-try)") unless target.none?(&:permanent?)
-        ui.info Formatador.display_line("[red]Bogus servers detected[reset]: [blue]#{target.bogus_servers.map(&:fullname).inspect}[reset]") unless target.bogus_servers.empty?
+
+        permanent = target.select(&:permanent?)
+        ui.info Formatador.display_line("servers with [red]'permanent=true'[reset] ignored: [blue]#{permanent.map(&:name).inspect}[reset]. (To kill, change 'permanent' to false, run knife cluster sync, and re-try)") unless permanent.empty?
+
+        bogus = target.select(&:bogus?)
+        ui.info Formatador.display_line("[red]Bogus servers detected[reset]: [blue]#{bogus.map(&:name).inspect}[reset]") unless bogus.empty?
       end
 
       def confirm_execution(target)
+        nodes           = target.map(&:node).compact
+        machines       = target.map(&:machine).compact
         delete_message = [
-          (((!config[:chef])   || target.chef_nodes.empty?)  ? nil : "#{target.chef_nodes.length} chef nodes"),
-          (((!config[:cloud])  || target.fog_servers.empty?) ? nil : "#{target.fog_servers.length} fog servers") ].compact.join(" and ")
+          (((!config[:chef])   || nodes.empty?)  ? nil : "#{nodes.length} chef nodes"),
+          (((!config[:cloud])  || machines.empty?) ? nil : "#{machines.length} fog servers") ].compact.join(" and ")
         confirm_or_exit("Are you absolutely certain that you want to delete #{delete_message}? (Type 'Yes' to confirm) ", 'Yes')
       end
 

data/lib/chef/knife/cluster_launch.rb

@@ -30,7 +30,7 @@ class Chef
         Chef::Knife::ClusterBootstrap.load_deps
       end
 
-      banner "knife cluster launch      CLUSTER[-FACET[-INDEXES]] (options) - Creates chef node and chef apiclient, pre-populates chef node, and instantiates in parallel their cloud machines. With --bootstrap flag, will ssh in to machines as they become ready and launch the bootstrap process"
+      banner "knife cluster launch      CLUSTER[-FACET[-INDEXES]] (options) - Creates chef node and chef apiclient, pre-populates chef node, and instantiates in parallel their cloud computers. With --bootstrap flag, will ssh in to computers as they become ready and launch the bootstrap process"
       [ :ssh_port, :ssh_user, :ssh_password, :identity_file, :use_sudo,
         :prerelease, :bootstrap_version, :template_file, :distro,
         :bootstrap_runs_chef_client, :host_key_verify
@@ -67,81 +67,75 @@ class Chef
         display(full_target)
         target = full_target.select(&:launchable?)
 
-        warn_or_die_on_bogus_servers(full_target) unless full_target.bogus_servers.empty?
+        warn_or_die_on_bogus_servers(full_target) unless full_target.select(&:bogus?).empty?
 
-        die("", "#{ui.color("All servers are running -- not launching any.",:blue)}", "", 1) if target.empty?
+        die("", "#{ui.color("All computers are running -- not launching any.",:blue)}", "", 1) if target.empty?
 
         # Pre-populate information in chef
-        section("Sync'ing to chef and cloud")
-        target.sync_to_cloud
-        target.sync_to_chef
-
-        # Launch servers
-        section("Launching machines", :green)
-        target.create_servers
+        section("Syncing to chef")
+        target.save :providers => :chef
 
+        # Launch computers
         ui.info("")
+        section("Launching computers", :green)
         display(target)
-
-        # As each server finishes, configure it
-        watcher_threads = target.parallelize do |svr|
-          perform_after_launch_tasks(svr)
-        end
-
-        progressbar_for_threads(watcher_threads)
+        target.launch
+
+        Chef::Log.warn("Need to set up target.parallelize")
+        # TODO:
+        # # As each server finishes, configure it
+        # watcher_threads = target.parallelize do |svr|
+        #   perform_after_launch_tasks(svr)
+        # end
+        #
+        # progressbar_for_threads(watcher_threads)
 
         display(target)
       end
 
-      def display(target)
-        super(target, ["Name", "InstanceID", "State", "Flavor", "Image", "AZ", "Public IP", "Private IP", "Created At", 'Volumes', 'Elastic IP']) do |svr|
-          { 'launchable?' => (svr.launchable? ? "[blue]#{svr.launchable?}[reset]" : '-' ), }
-        end
-      end
-
-      def perform_after_launch_tasks(server)
-        # Wait for node creation on amazon side
-        server.fog_server.wait_for{ ready? }
-
-        # Try SSH
-        unless config[:dry_run]
-          nil until tcp_test_ssh(server.fog_server.dns_name){ sleep @initial_sleep_delay ||= 10  }
-        end
-
-        # Make sure our list of volumes is accurate
-        Ironfan.fetch_fog_volumes
-        server.discover_volumes!
-
-        # Attach volumes, etc
-        server.sync_to_cloud
-
-        # Run Bootstrap
-        if config[:bootstrap]
-          run_bootstrap(server, server.fog_server.dns_name)
-        end
-      end
-
-      def tcp_test_ssh(hostname)
-        tcp_socket = TCPSocket.new(hostname, 22)
-        readable = IO.select([tcp_socket], nil, nil, 5)
-        if readable
-          Chef::Log.debug("sshd accepting connections on #{hostname}, banner is #{tcp_socket.gets}")
-          yield
-          true
-        else
-          false
-        end
-      rescue Errno::ETIMEDOUT
-        false
-      rescue Errno::ECONNREFUSED
-        sleep 2
-        false
-      rescue Errno::EHOSTUNREACH
-        sleep 2
-        false
-      ensure
-        tcp_socket && tcp_socket.close
-      end
+#       def perform_after_launch_tasks(server)
+#         # Wait for node creation on amazon side
+#         server.fog_server.wait_for{ ready? }
+# 
+#         # Try SSH
+#         unless config[:dry_run]
+#           nil until tcp_test_ssh(server.fog_server.dns_name){ sleep @initial_sleep_delay ||= 10  }
+#         end
+# 
+#         # Make sure our list of volumes is accurate
+#         Ironfan.fetch_fog_volumes
+#         server.discover_volumes!
+# 
+#         # Attach volumes, etc
+#         server.sync_to_cloud
+# 
+#         # Run Bootstrap
+#         if config[:bootstrap]
+#           run_bootstrap(server, server.fog_server.dns_name)
+#         end
+#       end
+# 
+#       def tcp_test_ssh(hostname)
+#         tcp_socket = TCPSocket.new(hostname, 22)
+#         readable = IO.select([tcp_socket], nil, nil, 5)
+#         if readable
+#           Chef::Log.debug("sshd accepting connections on #{hostname}, banner is #{tcp_socket.gets}")
+#           yield
+#           true
+#         else
+#           false
+#         end
+#       rescue Errno::ETIMEDOUT
+#         false
+#       rescue Errno::ECONNREFUSED
+#         sleep 2
+#         false
+#       rescue Errno::EHOSTUNREACH
+#         sleep 2
+#         false
+#       ensure
+#         tcp_socket && tcp_socket.close
+#       end
 
       def warn_or_die_on_bogus_servers(target)
         ui.info("")

data/lib/chef/knife/cluster_pry.rb

@@ -31,7 +31,7 @@ class Chef
 
       option :cloud,
         :long        => "--[no-]cloud",
-        :description => "Look up machines on AWS cloud (default is yes, look up machines; use --no-cloud to skip)",
+        :description => "Look up computers on AWS cloud (default is yes, look up computers; use --no-cloud to skip)",
         :default     => true,
         :boolean     => true
 
@@ -52,7 +52,7 @@ class Chef
         # Commands to try:
         #   nn = Chef::Node.load('node-name')
         #   cluster_nodes = cluster.servers.map(&:chef_node)
-        #   fog_machines  = cluster.servers.map(&:fog_server)
+        #   fog_computers  = cluster.servers.map(&:fog_server)
         #
         binding.pry
       end

data/lib/chef/knife/cluster_show.rb

@@ -30,7 +30,7 @@ class Chef
 
       option :cloud,
         :long        => "--[no-]cloud",
-        :description => "Look up machines on AWS cloud (default is yes, look up machines; use --no-cloud to skip)",
+        :description => "Look up computers on AWS cloud (default is yes, look up computers; use --no-cloud to skip)",
         :default     => true,
         :boolean     => true
 
@@ -46,11 +46,8 @@ class Chef
         # Dump entire contents of objects if -VV flag given
         #
         if config[:verbosity] >= 2
-          target.each do |svr|
-            Chef::Log.debug( "Server #{svr.name}: #{JSON.pretty_generate(svr.to_hash)}" )
-            Chef::Log.debug( "- volumes: #{JSON.pretty_generate(svr.composite_volumes.values.map(&:to_hash))}" )
-            Chef::Log.debug( "- cloud:   #{JSON.pretty_generate(svr.cloud.to_hash)}" )
-            Chef::Log.debug( "- fog:     #{JSON.pretty_generate(svr.fog_launch_description)}" )
+          target.each do |computer|
+            Chef::Log.debug( "Computer #{computer.name}: #{JSON.pretty_generate(computer.to_wire)}" )
           end
         end
 

data/lib/chef/knife/cluster_ssh.rb

@@ -41,22 +41,16 @@ class Chef
         :description => "The attribute to use for opening the connection - default is fqdn (ec2 users may prefer cloud.public_hostname)"
 
       def configure_session
-        target = get_slice(@name_args[0]).select(&:sshable?)
+        target = get_slice(@name_args[0]).select(&:running?)
 
         display(target) if config[:verbose] || config[:display_target]
 
         config[:attribute]     ||= Chef::Config[:knife][:ssh_address_attribute] || "fqdn"
         config[:ssh_user]      ||= Chef::Config[:knife][:ssh_user]
-        config[:identity_file] ||= target.ssh_identity_file
+#        config[:identity_file] ||= target.ssh_identity_file
 
-        @action_nodes = target.chef_nodes
-        addresses = target.servers.map do |svr|
-          address = svr.public_hostname
-          if address.blank? && (svr.chef_node)
-            address = format_for_display( svr.chef_node )[config[:attribute]]
-          end
-          address
-        end.compact
+#         @action_nodes = target.chef_nodes
+        addresses = target.map {|c| c.machine.public_hostname }.compact
 
         (ui.fatal("No nodes returned from search!"); exit 10) if addresses.nil? || addresses.length == 0
 
@@ -122,8 +116,8 @@ class Chef
         configure_session
 
         case @name_args[1]
+        when "screen",nil   then screen
         when "interactive"  then interactive
-        when "screen"       then screen
         when "tmux"         then tmux
         when "macterm"      then macterm
         when "cssh"         then cssh

data/lib/chef/knife/cluster_start.rb

@@ -24,14 +24,12 @@ class Chef
       import_banner_and_options(Ironfan::Script)
 
       def relevant?(server)
-        server.startable?
+        server.stopped?
       end
 
       def perform_execution(target)
-        section("Starting machines")
+        section("Starting computers")
         super(target)
-        section("Announcing Chef nodes as started")
-        target.send(:delegate_to_servers, :announce_as_started)
       end
 
     end

data/lib/chef/knife/cluster_stop.rb

@@ -28,10 +28,8 @@ class Chef
       end
 
       def perform_execution(target)
-        section("Stopping machines")
+        section("Stopping computers")
         super(target)
-        section("Announcing Chef nodes as stopped")
-        target.send(:delegate_to_servers, :announce_as_stopped)
       end
 
       def confirm_execution(target)

data/lib/chef/knife/cluster_sync.rb

@@ -21,7 +21,7 @@ require File.expand_path('ironfan_script', File.dirname(File.realdirpath(__FILE_
 class Chef
   class Knife
     class ClusterSync < Ironfan::Script
-      import_banner_and_options(Ironfan::Script, :description => "Update chef server and cloud machines with current cluster definition")
+      import_banner_and_options(Ironfan::Script, :description => "Update chef server and cloud computers with current cluster definition")
 
       option :cloud,
         :long        => "--[no-]cloud",
@@ -40,38 +40,30 @@ class Chef
         :boolean     => true
 
 
-      def relevant?(server)
+      def relevant?(computer)
         if config[:sync_all]
-          not server.bogus?
+          not computer.bogus?
         else
-          server.created? && server.in_chef?
+          computer.created? or computer.node?
         end
       end
 
       def perform_execution(target)
         if config[:chef]
-          sync_to_chef target
+          if config[:dry_run]
+            ui.info "(can't do a dry-run when syncing to chef -- skipping)"
+          else 
+            ui.info "Syncing to Chef:"
+            target.save :providers => :chef
+          end
         else Chef::Log.debug("Skipping sync to chef") ; end
 
-        if config[:cloud] && target.any?(&:in_cloud?)
-          sync_to_cloud target
+        if config[:cloud] && target.any?(&:machine?)
+          ui.info "Syncing to cloud:"
+          target.save :providers => :iaas
         else Chef::Log.debug("Skipping sync to cloud") ; end
       end
 
-      def sync_to_chef(target)
-        if config[:dry_run]
-          ui.info "(can't do a dry-run when syncing to chef -- skipping)"
-          return
-        end
-        ui.info "Syncing to Chef:"
-        target.sync_to_chef
-      end
-
-      def sync_to_cloud(target)
-        ui.info "Syncing to cloud:"
-        target.sync_to_cloud
-      end
-
     end
   end
 end

data/lib/chef/knife/ironfan_knife_common.rb

@@ -2,6 +2,7 @@ require 'chef/knife'
 
 module Ironfan
   module KnifeCommon
+    attr_accessor :broker
 
     def self.load_deps
       require 'formatador'
@@ -17,6 +18,7 @@ module Ironfan
       Ironfan.ui          = self.ui
       self.config[:cloud] = Chef::Config[:cloud] if Chef::Config.has_key?(:cloud)
       Ironfan.chef_config = self.config
+      self.broker         = Ironfan.broker
     end
 
     #
@@ -37,9 +39,8 @@ module Ironfan
       cluster_name, facet_name, slice_indexes = slice_string.split(/[\s\-]/, 3)
       ui.info("Inventorying servers in #{predicate_str(cluster_name, facet_name, slice_indexes)}")
       cluster = Ironfan.load_cluster(cluster_name)
-      cluster.resolve!
-      cluster.discover!
-      cluster.slice(facet_name, slice_indexes)
+      computers =  broker.discover! cluster
+      computers.slice(facet_name, slice_indexes)
     end
 
     def predicate_str(cluster_name, facet_name, slice_indexes)
@@ -67,18 +68,19 @@ module Ironfan
     #
     def get_relevant_slice( *predicate )
       full_target = get_slice( *predicate )
-      display(full_target) do |svr|
-        rel = relevant?(svr)
+      display(full_target) do |m|
+        rel = relevant?(m)
         { :relevant? => (rel ? "[blue]#{rel}[reset]" : '-' ) }
       end
-      full_target.select{|svr| relevant?(svr) }
+      full_target.select{|m| relevant?(m) }
     end
 
-    # passes target to ClusterSlice#display, will show headings in server slice
+    # passes target to Broker::Conductor#display, will show headings in server slice
     # tables based on the --verbose flag
     def display(target, display_style=nil, &block)
       display_style ||= (config[:verbosity] == 0 ? :default : :expanded)
-      target.display(display_style, &block)
+#       target.display(ui, display_style, &block)
+      broker.display(target,display_style)
     end
 
     #
@@ -133,7 +135,7 @@ module Ironfan
     def run_bootstrap(node, hostname)
       bs = bootstrapper(node, hostname)
       if config[:skip].to_s == 'true'
-        ui.info "Skipping: bootstrapp #{hostname} with #{JSON.pretty_generate(bs.config)}"
+        ui.info "Skipping: bootstrap #{hostname} with #{JSON.pretty_generate(bs.config)}"
         return
       end
       begin

data/lib/chef/knife/ironfan_script.rb

@@ -38,12 +38,13 @@ module Ironfan
 
     def run
       load_ironfan
+
       die(banner) if @name_args.empty?
       configure_dry_run
 
       target = get_relevant_slice(* @name_args)
 
-      die("No nodes to #{sub_command}, exiting", 1) if target.empty?
+      die("No computers to #{sub_command}, exiting", 1) if target.empty?
 
       ui.info(["\n",
           ui.color("Running #{sub_command}", :cyan),