knife-ec2 0.5.14 → 0.6.0

data/knife-ec2.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.add_dependency "fog", "~> 1.3"
+  s.add_dependency "fog", "~> 1.6"
   s.add_dependency "chef", ">= 0.10.10"
   %w(rspec-core rspec-expectations rspec-mocks  rspec_junit_formatter).each { |gem| s.add_development_dependency gem }
 

data/lib/chef/knife/ec2_base.rb

@@ -49,7 +49,6 @@ class Chef
           option :region,
             :long => "--region REGION",
             :description => "Your AWS region",
-            :default => "us-east-1",
             :proc => Proc.new { |key| Chef::Config[:knife][:region] = key }
         end
       end

data/lib/chef/knife/ec2_server_create.rb

@@ -42,8 +42,7 @@ class Chef
         :short => "-f FLAVOR",
         :long => "--flavor FLAVOR",
         :description => "The flavor of server (m1.small, m1.medium, etc)",
-        :proc => Proc.new { |f| Chef::Config[:knife][:flavor] = f },
-        :default => "m1.small"
+        :proc => Proc.new { |f| Chef::Config[:knife][:flavor] = f }
 
       option :image,
         :short => "-I IMAGE",
@@ -73,13 +72,13 @@ class Chef
         :short => "-Z ZONE",
         :long => "--availability-zone ZONE",
         :description => "The Availability Zone",
-        :default => "us-east-1b",
         :proc => Proc.new { |key| Chef::Config[:knife][:availability_zone] = key }
 
       option :chef_node_name,
         :short => "-N NAME",
         :long => "--node-name NAME",
-        :description => "The Chef node name for your new node"
+        :description => "The Chef node name for your new node",
+        :proc => Proc.new { |key| Chef::Config[:knife][:chef_node_name] = key }
 
       option :ssh_key_name,
         :short => "-S KEY",
@@ -105,6 +104,13 @@ class Chef
         :default => "22",
         :proc => Proc.new { |key| Chef::Config[:knife][:ssh_port] = key }
 
+      option :ssh_gateway,
+        :short => "-w GATEWAY",
+        :long => "--ssh-gateway GATEWAY",
+        :description => "The ssh gateway server",
+        :proc => Proc.new { |key| Chef::Config[:knife][:ssh_gateway] = key }
+
+
       option :identity_file,
         :short => "-i IDENTITY_FILE",
         :long => "--identity-file IDENTITY_FILE",
@@ -123,8 +129,7 @@ class Chef
         :short => "-d DISTRO",
         :long => "--distro DISTRO",
         :description => "Bootstrap a distro using a template; default is 'chef-full'",
-        :proc => Proc.new { |d| Chef::Config[:knife][:distro] = d },
-        :default => "chef-full"
+        :proc => Proc.new { |d| Chef::Config[:knife][:distro] = d }
 
       option :template_file,
         :long => "--template-file TEMPLATE",
@@ -136,30 +141,31 @@ class Chef
         :long => "--ebs-size SIZE",
         :description => "The size of the EBS volume in GB, for EBS-backed instances"
 
+      option :ebs_optimized,
+        :long => "--ebs-optimized",
+        :description => "Enabled optimized EBS I/O"
+
       option :ebs_no_delete_on_term,
         :long => "--ebs-no-delete-on-term",
-        :description => "Do not delete EBS volumn on instance termination"
+        :description => "Do not delete EBS volume on instance termination"
 
       option :run_list,
         :short => "-r RUN_LIST",
         :long => "--run-list RUN_LIST",
         :description => "Comma separated list of roles/recipes to apply",
-        :proc => lambda { |o| o.split(/[\s,]+/) },
-        :default => []
+        :proc => lambda { |o| o.split(/[\s,]+/) }
 
       option :json_attributes,
         :short => "-j JSON",
         :long => "--json-attributes JSON",
         :description => "A JSON string to be added to the first run of chef-client",
-        :proc => lambda { |o| JSON.parse(o) },
-        :default => {}
-
+        :proc => lambda { |o| JSON.parse(o) }
 
       option :subnet_id,
         :short => "-s SUBNET-ID",
         :long => "--subnet SUBNET-ID",
         :description => "create node in this Virtual Private Cloud Subnet ID (implies VPC mode)",
-        :default => false
+        :proc => Proc.new { |key| Chef::Config[:knife][:subnet_id] = key }
 
       option :host_key_verify,
         :long => "--[no-]host-key-verify",
@@ -174,8 +180,29 @@ class Chef
         :proc => Proc.new { |m| Chef::Config[:knife][:aws_user_data] = m },
         :default => nil
 
-      def tcp_test_ssh(hostname)
-        tcp_socket = TCPSocket.new(hostname, config[:ssh_port])
+      Chef::Config[:knife][:hints] ||= {"ec2" => {}}
+      option :hint,
+        :long => "--hint HINT_NAME[=HINT_FILE]",
+        :description => "Specify Ohai Hint to be set on the bootstrap target.  Use multiple --hint options to specify multiple hints.",
+        :proc => Proc.new { |h|
+           name, path = h.split("=")
+           Chef::Config[:knife][:hints][name] = path ? JSON.parse(::File.read(path)) : Hash.new
+        }
+
+      option :ephemeral,
+        :long => "--ephemeral EPHEMERAL_DEVICES",
+        :description => "Comma separated list of device locations (eg - /dev/sdb) to map ephemeral devices",
+        :proc => lambda { |o| o.split(/[\s,]+/) },
+        :default => []
+
+      option :server_connect_attribute,
+        :long => "--server-connect-attribute ATTRIBUTE",
+        :short => "-a ATTRIBUTE",
+        :description => "The EC2 server attribute to use for SSH connection",
+        :default => nil
+
+      def tcp_test_ssh(hostname, ssh_port)
+        tcp_socket = TCPSocket.new(hostname, ssh_port)
         readable = IO.select([tcp_socket], nil, nil, 5)
         if readable
           Chef::Log.debug("sshd accepting connections on #{hostname}, banner is #{tcp_socket.gets}")
@@ -184,23 +211,10 @@ class Chef
         else
           false
         end
-      rescue SocketError
+      rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, IOError
         sleep 2
         false
-      rescue Errno::ETIMEDOUT
-        false
-      rescue Errno::EPERM
-        false
-      rescue Errno::ECONNREFUSED
-        sleep 2
-        false
-      # This happens on EC2 quite often
-      rescue Errno::EHOSTUNREACH
-        sleep 2
-        false
-      # This happens on EC2 sometimes
-      rescue Errno::ENETUNREACH
-        sleep 2
+      rescue Errno::EPERM, Errno::ETIMEDOUT
         false
       ensure
         tcp_socket && tcp_socket.close
@@ -236,7 +250,7 @@ class Chef
         # default security group id at this point unless we look it up, hence
         # 'default' is printed if no id was specified.
         printed_security_groups = "default"
-        printed_security_groups = @server.groups.join(", ") if @server.groups 
+        printed_security_groups = @server.groups.join(", ") if @server.groups
         msg_pair("Security Groups", printed_security_groups) unless vpc_mode? or (@server.groups.nil? and @server.security_group_ids)
 
         printed_security_group_ids = "default"
@@ -264,14 +278,9 @@ class Chef
 
         print "\n#{ui.color("Waiting for sshd", :magenta)}"
 
-        fqdn = vpc_mode? ? @server.private_ip_address : @server.dns_name
+        wait_for_sshd(ssh_connect_host)
 
-        print(".") until tcp_test_ssh(fqdn) {
-          sleep @initial_sleep_delay ||= (vpc_mode? ? 40 : 10)
-          puts("done")
-        }
-
-        bootstrap_for_node(@server,fqdn).run
+        bootstrap_for_node(@server,ssh_connect_host).run
 
         puts "\n"
         msg_pair("Instance ID", @server.id)
@@ -300,6 +309,9 @@ class Chef
             end
           end
         end
+        if config[:ebs_optimized]
+          msg_pair("EBS is Optimized", @server.ebs_optimized.to_s)
+        end
         if vpc_mode?
           msg_pair("Subnet ID", @server.subnet_id)
         else
@@ -309,22 +321,23 @@ class Chef
         end
         msg_pair("Private IP Address", @server.private_ip_address)
         msg_pair("Environment", config[:environment] || '_default')
-        msg_pair("Run List", config[:run_list].join(', '))
-        msg_pair("JSON Attributes",config[:json_attributes]) unless config[:json_attributes].empty?
+        msg_pair("Run List", (config[:run_list] || []).join(', '))
+        msg_pair("JSON Attributes",config[:json_attributes]) unless !config[:json_attributes] || config[:json_attributes].empty?
       end
 
-      def bootstrap_for_node(server,fqdn)
+      def bootstrap_for_node(server,ssh_host)
         bootstrap = Chef::Knife::Bootstrap.new
-        bootstrap.name_args = [fqdn]
-        bootstrap.config[:run_list] = config[:run_list]
+        bootstrap.name_args = [ssh_host]
+        bootstrap.config[:run_list] = locate_config_value(:run_list) || []
         bootstrap.config[:ssh_user] = config[:ssh_user]
         bootstrap.config[:ssh_port] = config[:ssh_port]
+        bootstrap.config[:ssh_gateway] = config[:ssh_gateway]
         bootstrap.config[:identity_file] = config[:identity_file]
-        bootstrap.config[:chef_node_name] = config[:chef_node_name] || server.id
+        bootstrap.config[:chef_node_name] = locate_config_value(:chef_node_name) || server.id
         bootstrap.config[:prerelease] = config[:prerelease]
         bootstrap.config[:bootstrap_version] = locate_config_value(:bootstrap_version)
-        bootstrap.config[:first_boot_attributes] = config[:json_attributes]
-        bootstrap.config[:distro] = locate_config_value(:distro)
+        bootstrap.config[:first_boot_attributes] = locate_config_value(:json_attributes) || {}
+        bootstrap.config[:distro] = locate_config_value(:distro) || "chef-full"
         bootstrap.config[:use_sudo] = true unless config[:ssh_user] == 'root'
         bootstrap.config[:template_file] = locate_config_value(:template_file)
         bootstrap.config[:environment] = config[:environment]
@@ -336,7 +349,7 @@ class Chef
       def vpc_mode?
         # Amazon Virtual Private Cloud requires a subnet_id. If
         # present, do a few things differently
-        !!config[:subnet_id]
+        !!locate_config_value(:subnet_id)
       end
 
       def ami
@@ -351,7 +364,7 @@ class Chef
           ui.error("You have not provided a valid image (AMI) value.  Please note the short option for this value recently changed from '-i' to '-I'.")
           exit 1
         end
-        
+
         if vpc_mode? and !!config[:security_groups]
           ui.error("You are using a VPC, security groups specified with '-G' are not allowed, specify one or more security group ids with '-g' instead.")
           exit 1
@@ -372,12 +385,12 @@ class Chef
         server_def = {
           :image_id => locate_config_value(:image),
           :groups => config[:security_groups],
-          :security_group_ids => config[:security_group_ids],
+          :security_group_ids => locate_config_value(:security_group_ids),
           :flavor_id => locate_config_value(:flavor),
           :key_name => Chef::Config[:knife][:aws_ssh_key_id],
           :availability_zone => locate_config_value(:availability_zone)
         }
-        server_def[:subnet_id] = config[:subnet_id] if config[:subnet_id]
+        server_def[:subnet_id] = locate_config_value(:subnet_id) if vpc_mode?
 
         if Chef::Config[:knife][:aws_user_data]
           begin
@@ -387,6 +400,12 @@ class Chef
           end
         end
 
+        if config[:ebs_optimized]
+          server_def[:ebs_optimized] = "true"
+        else
+          server_def[:ebs_optimized] = "false"
+        end
+
         if ami.root_device_type == "ebs"
           ami_map = ami.block_device_mapping.first
           ebs_size = begin
@@ -405,6 +424,7 @@ class Chef
                         else
                           ami_map["deleteOnTermination"]
                         end
+
           server_def[:block_device_mapping] =
             [{
                'DeviceName' => ami_map["deviceName"],
@@ -413,8 +433,55 @@ class Chef
              }]
         end
 
+        (config[:ephemeral] || []).each_with_index do |device_name, i|
+          server_def[:block_device_mapping] = (server_def[:block_device_mapping] || []) << {'VirtualName' => "ephemeral#{i}", 'DeviceName' => device_name}
+        end
+
         server_def
       end
+
+      def wait_for_sshd(hostname)
+        config[:ssh_gateway] ? wait_for_tunnelled_sshd(hostname) : wait_for_direct_sshd(hostname, config[:ssh_port])
+      end
+
+      def wait_for_tunnelled_sshd(hostname)
+        print(".")
+        print(".") until tunnel_test_ssh(ssh_connect_host) {
+          sleep @initial_sleep_delay ||= (vpc_mode? ? 40 : 10)
+          puts("done")
+        }
+      end
+
+      def tunnel_test_ssh(hostname, &block)
+        gw_host, gw_user = config[:ssh_gateway].split('@').reverse
+        gw_host, gw_port = gw_host.split(':')
+        gateway = Net::SSH::Gateway.new(gw_host, gw_user, :port => gw_port || 22)
+        status = false
+        gateway.open(hostname, config[:ssh_port]) do |local_tunnel_port|
+          status = tcp_test_ssh('localhost', local_tunnel_port, &block)
+        end
+        status
+      rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, IOError
+        sleep 2
+        false
+      rescue Errno::EPERM, Errno::ETIMEDOUT
+        false
+      end
+
+      def wait_for_direct_sshd(hostname, ssh_port)
+        print(".") until tcp_test_ssh(ssh_connect_host, ssh_port) {
+          sleep @initial_sleep_delay ||= (vpc_mode? ? 40 : 10)
+          puts("done")
+        }
+      end
+
+      def ssh_connect_host
+        @ssh_connect_host ||= if config[:server_connect_attribute]
+          server.send(config[:server_connect_attribute])
+        else
+          vpc_mode? ? server.private_ip_address : server.dns_name
+        end
+      end
     end
   end
 end

data/lib/chef/knife/ec2_server_list.rb

@@ -27,6 +27,18 @@ class Chef
 
       banner "knife ec2 server list (options)"
 
+      option :name,
+        :short => "-n",
+        :long => "--no-name",
+        :boolean => true,
+        :default => true,
+        :description => "Do not display name tag in output"
+
+      option :tags,
+        :short => "-t TAG1,TAG2",
+        :long => "--tags TAG1,TAG2",
+        :description => "List of tags to output"
+
       def run
         $stdout.sync = true
 
@@ -34,22 +46,49 @@ class Chef
 
         server_list = [
           ui.color('Instance ID', :bold),
+        
+          if config[:name]
+            ui.color("Name", :bold)
+          end,
+
           ui.color('Public IP', :bold),
           ui.color('Private IP', :bold),
           ui.color('Flavor', :bold),
           ui.color('Image', :bold),
           ui.color('SSH Key', :bold),
           ui.color('Security Groups', :bold),
+          
+          if config[:tags]
+            config[:tags].split(",").collect do |tag_name|
+              ui.color("Tag:#{tag_name}", :bold)
+            end
+          end,
+          
           ui.color('State', :bold)
-        ]
+        ].flatten.compact
+        
+        output_column_count = server_list.length
+        
         connection.servers.all.each do |server|
           server_list << server.id.to_s
+          
+          if config[:name]
+            server_list << server.tags["Name"].to_s
+          end
+          
           server_list << server.public_ip_address.to_s
           server_list << server.private_ip_address.to_s
           server_list << server.flavor_id.to_s
           server_list << server.image_id.to_s
           server_list << server.key_name.to_s
           server_list << server.groups.join(", ")
+          
+          if config[:tags]
+            config[:tags].split(",").each do |tag_name|
+              server_list << server.tags[tag_name].to_s
+            end
+          end
+          
           server_list << begin
             state = server.state.to_s.downcase
             case state
@@ -62,11 +101,9 @@ class Chef
             end
           end
         end
-        puts ui.list(server_list, :uneven_columns_across, 8)
+        puts ui.list(server_list, :uneven_columns_across, output_column_count)
 
       end
     end
   end
 end
-
-

data/lib/knife-ec2/version.rb

@@ -1,6 +1,6 @@
 module Knife
   module Ec2
-    VERSION = "0.5.14"
+    VERSION = "0.6.0"
     MAJOR, MINOR, TINY = VERSION.split('.')
   end
 end

data/spec/unit/ec2_server_create_spec.rb

@@ -128,6 +128,7 @@ describe Chef::Knife::Ec2ServerCreate do
       @knife_ec2_create.config[:ssh_user] = "ubuntu"
       @knife_ec2_create.config[:identity_file] = "~/.ssh/aws-key.pem"
       @knife_ec2_create.config[:ssh_port] = 22
+      @knife_ec2_create.config[:ssh_gateway] = 'bastion.host.com'
       @knife_ec2_create.config[:chef_node_name] = "blarf"
       @knife_ec2_create.config[:template_file] = '~/.chef/templates/my-bootstrap.sh.erb'
       @knife_ec2_create.config[:distro] = 'ubuntu-10.04-magic-sparkles'
@@ -153,6 +154,10 @@ describe Chef::Knife::Ec2ServerCreate do
       @bootstrap.config[:ssh_user].should == 'ubuntu'
     end
 
+    it "configures the bootstrap to use the correct ssh_gateway host" do
+      @bootstrap.config[:ssh_gateway].should == 'bastion.host.com'
+    end
+
     it "configures the bootstrap to use the correct ssh identity file" do
       @bootstrap.config[:identity_file].should == "~/.ssh/aws-key.pem"
     end
@@ -192,6 +197,10 @@ describe Chef::Knife::Ec2ServerCreate do
     it "configured the bootstrap to use the desired template" do
       @bootstrap.config[:template_file].should == '~/.chef/templates/my-bootstrap.sh.erb'
     end
+
+    it "configured the bootstrap to set an ec2 hint (via Chef::Config)" do
+      Chef::Config[:knife][:hints]["ec2"].should_not be_nil
+    end
   end
 
   describe "when validating the command-line parameters" do
@@ -251,6 +260,47 @@ describe Chef::Knife::Ec2ServerCreate do
 
       server_def[:availability_zone].should == "dis-one"
     end
+
+    it "adds the specified ephemeral device mappings" do
+      @knife_ec2_create.config[:ephemeral] = [ "/dev/sdb", "/dev/sdc", "/dev/sdd", "/dev/sde" ]
+      server_def = @knife_ec2_create.create_server_def
+
+      server_def[:block_device_mapping].should == [{ "VirtualName" => "ephemeral0", "DeviceName" => "/dev/sdb" },
+                                                   { "VirtualName" => "ephemeral1", "DeviceName" => "/dev/sdc" },
+                                                   { "VirtualName" => "ephemeral2", "DeviceName" => "/dev/sdd" },
+                                                   { "VirtualName" => "ephemeral3", "DeviceName" => "/dev/sde" }]
+    end
   end
 
+  describe "ssh_connect_host" do
+    before(:each) do
+      @new_ec2_server.stub!(
+        :dns_name => 'public_name',
+        :private_ip_address => 'private_ip',
+        :custom => 'custom'
+      )
+      @knife_ec2_create.stub!(:server => @new_ec2_server)
+    end
+
+    describe "by default" do
+      it 'should use public dns name' do
+        @knife_ec2_create.ssh_connect_host.should == 'public_name'
+      end
+    end
+
+    describe "with vpc_mode?" do
+      it 'should use private ip' do
+        @knife_ec2_create.stub!(:vpc_mode? => true)
+        @knife_ec2_create.ssh_connect_host.should == 'private_ip'
+      end
+
+    end
+
+    describe "with custom server attribute" do
+      it 'should use custom server attribute' do
+        @knife_ec2_create.config[:server_connect_attribute] = 'custom'
+        @knife_ec2_create.ssh_connect_host.should == 'custom'
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: knife-ec2
 version: !ruby/object:Gem::Version
-  version: 0.5.14
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-26 00:00:00.000000000 Z
+date: 2012-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -19,7 +19,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,7 +27,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: chef
   requirement: !ruby/object:Gem::Requirement
@@ -162,3 +162,4 @@ summary: EC2 Support for Chef's Knife Command
 test_files:
 - spec/spec_helper.rb
 - spec/unit/ec2_server_create_spec.rb
+has_rdoc: true