openstudio-aws 0.2.5 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31a9071ad83312a4fce7ad046e3db9c5531ec1cd
-  data.tar.gz: 965059858cd562bf5837836bb4e81ed1e08179ab
+  metadata.gz: 7e3f40caa3eed2715d9774ae42a8913b3d605434
+  data.tar.gz: ac233b57d15ba24a027019e94c919da9a0850cf0
 SHA512:
-  metadata.gz: bcd7712ea77b6cf33094f2e1fa9695f35d711418d07d8356f7cee7d344c7da559996253abea9dbc26bb0299a898a133ef35e166b682b6a775ead5a108a208e8c
-  data.tar.gz: 80fa6a7288f9e988fef90eae8ca6924ae756d0392838bc75d929a7bc0d6ec5dd7df3163c79306ec9bb3a65258543df589b79d0e60628c2dba0c0fa4a3e6acbed
+  metadata.gz: 76a729d74b97e3b9fe3e8e737b303c38939443567d2006e9ebfd46ce709995a7d0e1a29305951ceeb362122e36d7a9a55968aba2e6127f03dfefae29ef7c94cf
+  data.tar.gz: 5d46f753111b741a8b51349d2e83c1ef75689f51a42d006a590d5e0795a1d4e7f46acb25c259d43188a3bca411fa66afe469cca186b14a04c849498b8458d092

data/lib/openstudio-aws.rb

@@ -7,7 +7,7 @@ require 'securerandom'
 require 'semantic'
 require 'semantic/core_ext'
 require 'fileutils'
-
+require 'sshkey'
 # AWS SDK CORE
 begin
   require 'aws-sdk-core'

data/lib/openstudio/aws/aws.rb

@@ -80,7 +80,9 @@ module OpenStudio
         @default_amis = OpenStudioAmis.new(options[:ami_lookup_version], ami_options).get_amis
       end
 
-      # def load_data_from_json(json_filename)
+
+
+      # # def load_data_from_json(json_filename)
       #   @os_aws.load_data_from_json
       #
       #
@@ -90,11 +92,13 @@ module OpenStudio
       def create_server(options = {}, instances_json = 'server_data.json')
         defaults = {
           instance_type: 'm2.xlarge',
-          security_group: 'openstudio-server-sg-v1',
+          security_groups: [],
           image_id: @default_amis[:server],
           user_id: 'unknown_user',
 
           # optional -- will default later
+          associate_public_ip_address: nil,
+          subnet_id: nil,
           ebs_volume_id: nil,
           aws_key_pair_name: nil,
           private_key_file_name: nil, # required if using an existing "aws_key_pair_name"
@@ -102,12 +106,24 @@ module OpenStudio
         }
         options = defaults.merge(options)
 
+        # for backwards compatibilty, still allow security_group
+        if options[:security_group]
+          warn "Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0"
+          options[:security_groups] = [ options[:security_group] ]
+        end
+
         if options[:aws_key_pair_name]
           fail 'Must pass in the private_key_file_name' unless options[:private_key_file_name]
           fail "Private key was not found: #{options[:private_key_file_name]}" unless File.exist? options[:private_key_file_name]
         end
 
-        @os_aws.create_or_retrieve_security_group options[:security_group]
+        if options[:security_groups].empty?
+          # if the user has not specified any security groups, then create one called: 'openstudio-server-sg-v1'
+          @os_aws.create_or_retrieve_default_security_group
+        else
+          @os_aws.security_groups = options[:security_groups]
+        end
+
         @os_aws.create_or_retrieve_key_pair options[:aws_key_pair_name]
 
         # If using an already_existing key_pair, then you must pass in the private key file name
@@ -119,7 +135,17 @@ module OpenStudio
           @os_aws.save_private_key('ec2_server_key.pem')
         end
 
-        server_options = { user_id: options[:user_id], tags: options[:tags] }
+        server_options = {
+            user_id: options[:user_id],
+            tags: options[:tags],
+            subnet_id: options[:subnet_id],
+            associate_public_ip_address: options[:associate_public_ip_address],
+        }
+
+        # save the worker pem and public to the directory
+        # presently, this will always overwrite the worker key, is that okay? Is this really needed later?
+        @os_aws.save_worker_keys('.')
+
         # if instance_data[:ebs_volume_id]
         #   server_options[:ebs_volume_id] = instance_data[:ebs_volume_id]
         # end
@@ -127,7 +153,7 @@ module OpenStudio
         @os_aws.launch_server(options[:image_id], options[:instance_type], server_options)
 
         @instances_json = instances_json
-        File.open(@instances_json, 'w') { |f| f << JSON.pretty_generate(@os_aws.server.to_os_hash) }
+        save_cluster_json(@instances_json)
 
         # Print out some debugging commands (probably work on mac/linux only)
         puts ''
@@ -139,11 +165,13 @@ module OpenStudio
       def create_workers(number_of_instances, options = {}, user_id = 'unknown_user')
         defaults = {
           instance_type: 'm2.4xlarge',
-          security_group: 'openstudio-server-sg-v1',
-          image_id: @default_amis[:server],
+          security_groups: [],
+          image_id: nil, # don't prescribe the image id so that it can determine later
           user_id: user_id,
 
           # optional -- will default later
+          associate_public_ip_address: nil,
+          subnet_id: nil,
           ebs_volume_id: nil,
           aws_key_pair_name: nil,
           private_key_file_name: nil, # required if using an existing "aws_key_pair_name",
@@ -151,13 +179,16 @@ module OpenStudio
         }
         options = defaults.merge(options)
 
+        # for backwards compatibilty, still allow security_group
+        if options[:security_group]
+          warn "Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0"
+          options[:security_groups] = [ options[:security_group] ]
+        end
+
+
         # Get the right worker AMI ids based on the type of instance
         if options[:image_id].nil?
-          if options[:instance_type] =~ /cc2|c3/
-            options[:image_id] = @default_amis[:cc2worker]
-          else
-            options[:image_id] = @default_amis[:worker]
-          end
+          options[:image_id] = determine_image_type(options[:instance_type])
         end
 
         fail "Can't create workers without a server instance running" if @os_aws.server.nil?
@@ -166,29 +197,21 @@ module OpenStudio
           puts ''
           puts 'No workers requested'
         else
-          worker_options = { user_id: options[:user_id], tags: options[:tags] }
+          worker_options = {
+              user_id: options[:user_id],
+              tags: options[:tags],
+              subnet_id: options[:subnet_id],
+              associate_public_ip_address: options[:associate_public_ip_address]
+          }
+
           # if options[:ebs_volume_size]
           #   worker_options[:ebs_volume_size] = options[:ebs_volume_size]
           # end
 
           @os_aws.launch_workers(options[:image_id], options[:instance_type], number_of_instances, worker_options)
 
-          ## append the information to the server_data hash that already exists
-          # @server_data[:instance_type] = instance_data[:instance_type]
-          # @server_data[:num] = number_of_instances
-          # server_string = @server_data.to_json.gsub("\"", "\\\\\"")
-          #
-          # start_string = "ruby #{os_aws_file_location} #{@config.access_key} #{@config.secret_key} us-east-1 EC2 launch_workers \"#{server_string}\""
-          # puts "Worker Command: #{start_string}"
-          # worker_data_string = `#{start_string}`
-          # @worker_data = JSON.parse(worker_data_string, :symbolize_names => true)
-          # File.open("worker_data.json", "w") { |f| f << JSON.pretty_generate(worker_data) }
-          #
-          ## Print out some debugging commands (probably work on mac/linux only)
           # Add the worker data to the JSON
-          h = JSON.parse(File.read(@instances_json))
-          h[:workers] = @os_aws.to_os_worker_hash[:workers]
-          File.open(@instances_json, 'w') { |f| f << JSON.pretty_generate(h) }
+          save_cluster_json @instances_json
 
           puts ''
           puts 'Worker SSH Command:'
@@ -203,6 +226,21 @@ module OpenStudio
         @os_aws.configure_server_and_workers
       end
 
+      # Return information on the cluster instances as a hash. This includes IP addresses, host names, number of processors, etc.
+      # @return [Hash] Data about the configured cluster
+      def cluster_info
+        h = @os_aws.server.to_os_hash
+        h[:workers] = @os_aws.to_os_worker_hash[:workers]
+
+        h
+      end
+
+      # Save a JSON with information about the cluster that was configured.
+      # @param filename [String] Path and filename to save the JSON file
+      def save_cluster_json(filename)
+        File.open(@instances_json, 'w') { |f| f << JSON.pretty_generate(cluster_info) }
+      end
+
       # openstudio_instance_type as symbol
       def stop_instances(group_id, openstudio_instance_type)
         instances = @os_aws.describe_running_instances(group_id, openstudio_instance_type.to_sym)
@@ -221,24 +259,41 @@ module OpenStudio
         resp
       end
 
-      # Warning, this appears that it terminates all the instances of the openstudio_instance_type
-      def terminate_instances_by_group_id(group_id, openstudio_instance_type)
-        instances = @os_aws.describe_running_instances(group_id, openstudio_instance_type.to_sym)
+      # Warning, this appears that it terminates all the instances
+      def terminate_instances_by_group_id(group_id)
+        instances = @os_aws.describe_running_instances(group_id)
         ids = instances.map { |k, _| k[:instance_id] }
 
         puts "Terminating the following instances #{ids}"
         resp = []
         resp = @os_aws.terminate_instances(ids).to_hash unless ids.empty?
-        resp
+        resp[:terminating_instances].first[:current_state][:name] == 'shutting-down'
+      end
+
+      # Terminate the entire cluster
+      def terminate
+        puts "Terminating any instance with group ID: #{@os_aws.group_uuid}"
+
+        terminate_instances_by_group_id(@os_aws.group_uuid)
       end
 
       def load_instance_info_from_file(filename)
         fail 'Could not find instance description JSON file' unless File.exist? filename
 
         h = JSON.parse(File.read(filename), symbolize_names: true)
-        @os_aws.find_server(h)
+        if h[:location] == 'AWS'
+          @os_aws.find_server(h)
+        else
+          puts "Instance file '#{filename}' does not have the location of 'AWS'"
+          return false
+        end
 
         # load the worker nodes someday
+        true
+      end
+
+      def server
+        @os_aws.server
       end
 
       # Send a file to the server or worker nodes
@@ -276,6 +331,17 @@ module OpenStudio
         end
       end
 
+      def determine_image_type(instance_type)
+        image = nil
+        if instance_type =~ /cc2/
+          image = @default_amis[:cc2worker]
+        else
+          image = @default_amis[:worker]
+        end
+
+        image
+      end
+
       private
 
       def os_aws_file_location

data/lib/openstudio/aws/version.rb

@@ -1,5 +1,5 @@
 module OpenStudio
   module Aws
-    VERSION = '0.2.5'
+    VERSION = '0.3.0'
   end
 end

data/lib/openstudio/lib/openstudio_aws_instance.rb

@@ -24,20 +24,24 @@ class OpenStudioAwsInstance
   attr_reader :openstudio_instance_type
   attr_reader :data
   attr_reader :private_key_file_name
+  attr_reader :private_ip_address
   attr_reader :group_uuid
 
-  def initialize(aws_session, openstudio_instance_type, key_pair_name, security_group_name, group_uuid, private_key,
-                 private_key_file_name, proxy = nil)
+
+  # param security_groups can be a single instance or an array
+  def initialize(aws_session, openstudio_instance_type, key_pair_name, security_groups, group_uuid, private_key,
+                 private_key_file_name, subnet_id, proxy = nil)
     @data = nil # stored information about the instance
     @aws = aws_session
     @openstudio_instance_type = openstudio_instance_type # :server, :worker
     @key_pair_name = key_pair_name
-    @security_group_name = security_group_name
+    @security_groups = security_groups
     @group_uuid = group_uuid.to_s
     @init_timestamp = Time.now  # This is the timestamp and is typically just tracked for the server
     @private_key = private_key
     @private_key_file_name = private_key_file_name
     @proxy = proxy
+    @subnet_id = subnet_id
 
     # Important System information
     @host = nil
@@ -74,12 +78,10 @@ class OpenStudioAwsInstance
     # need to wait until it is available
 
     @aws.attach_volume(
-
           volume_id: resp[:volume_id],
           instance_id: instance_id,
           # required
           device: '/dev/sdh'
-
     )
 
     # Wait for the volume to attach
@@ -88,21 +90,42 @@ class OpenStudioAwsInstance
   end
 
   def launch_instance(image_id, instance_type, user_data, user_id, options = {})
-    # logger.info("user_data #{user_data.inspect}")
+    # determine the instance type of the server
     instance = {
-      image_id: image_id,
-      key_name: @key_pair_name,
-      security_groups: [@security_group_name],
-      user_data: Base64.encode64(user_data),
-      instance_type: instance_type,
-      placement: {
-        availability_zone: 'us-east-1c'
-      },
-      min_count: 1,
-      max_count: 1
+        image_id: image_id,
+        key_name: @key_pair_name,
+        security_group_ids: @security_groups,
+        subnet_id: options[:subnet_id],
+        user_data: Base64.encode64(user_data),
+        instance_type: instance_type,
+        min_count: 1,
+        max_count: 1
     }
+
+    if options[:availability_zone]
+      # use the availability zone from the server
+      # logger.info("user_data #{user_data.inspect}")
+      instance[:placement] ||= {}
+      instance[:placement][:availability_zone] = options[:availability_zone]
+    end
+
+    if options[:associate_public_ip_address]
+      # You have to delete the security group and subnet_id from the instance hash and put into the network interface
+      # otherwise you will get an error on launch with an InvalidParameterCombination error.
+      instance[:network_interfaces] = [
+          {
+              subnet_id: instance.delete(:subnet_id),
+              groups: instance.delete(:security_group_ids),
+              device_index: 0,
+              associate_public_ip_address: true
+          }
+      ]
+    end
+
+    # Documentation for run_instances is here: http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/EC2/Client.html#run_instances-instance_method
     result = @aws.run_instances(instance)
 
+
     # determine how many processors are suppose to be in this image (lookup for now?)
     processors = find_processors(instance_type)
 
@@ -133,11 +156,18 @@ class OpenStudioAwsInstance
     end
 
     # only asked for 1 instance, so therefore it should be the first
-    aws_instance = result.data.instances.first
-    @aws.create_tags(
-        resources: [aws_instance.instance_id],
-        tags: aws_tags
-    )
+    begin
+      tries ||= 3
+      aws_instance = result.data.instances.first
+      @aws.create_tags(
+          resources: [aws_instance.instance_id],
+          tags: aws_tags
+      )
+    rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
+      sleep 1000
+      retry unless (tries -= 1).zero?
+    end
+
 
     # get the instance information
     test_result = @aws.describe_instance_status(instance_ids: [aws_instance.instance_id]).data.instance_statuses.first
@@ -185,12 +215,14 @@ class OpenStudioAwsInstance
         # private_key_path: "#{File.expand_path(@private_key_path)}"
         #:private_key => @private_key, # need to stop printing this out
         location: 'AWS',
+        availability_zone: @data.availability_zone,
         server: {
           id: @data.id,
-          ip: 'http://' + @data.ip,
+          ip: "http://#{@data.ip}",
           dns: @data.dns,
           procs: @data.procs,
-          private_key_file_name: @private_key_file_name
+          private_key_file_name: @private_key_file_name,
+          private_ip_address: @private_ip_address
         }
       }
     else
@@ -202,6 +234,14 @@ class OpenStudioAwsInstance
     h
   end
 
+  def ip
+    @data.ip
+  end
+
+  def hostname
+    "http://#{@data.ip}"
+  end
+
   def find_processors(instance)
     lookup = {
       'm3.medium' => 1,
@@ -385,11 +425,20 @@ class OpenStudioAwsInstance
   # store some of the data into a custom struct.  The instance is the full description.  The remaining fields are
   # just easier accessors to the data in the raw request except for procs which is a custom request.
   def create_struct(instance, procs)
-    instance_struct = Struct.new(:instance, :id, :ip, :dns, :procs)
-    s = instance_struct.new(instance, instance[:instance_id], instance[:public_ip_address], instance[:public_dns_name], procs)
+    instance_struct = Struct.new(:instance, :id, :ip, :dns, :procs, :availability_zone, :private_ip_address)
+    s = instance_struct.new(
+        instance,
+        instance[:instance_id],
+        instance[:public_ip_address],
+        instance[:public_dns_name],
+        procs,
+        instance[:placement][:availability_zone],
+        instance[:private_ip_address]
+    )
 
     # store some values into the member variables
     @ip_address = instance[:public_ip_address]
+    @private_ip_address = instance[:private_ip_address]
     @instance_id = instance[:instance_id]
 
     s

data/lib/openstudio/lib/openstudio_aws_wrapper.rb

@@ -42,23 +42,26 @@ class OpenStudioAwsWrapper
   include Logging
 
   attr_reader :group_uuid
-  attr_reader :security_group_name
   attr_reader :key_pair_name
   attr_reader :server
   attr_reader :workers
   attr_reader :proxy
+  attr_reader :worker_keys
 
   attr_accessor :private_key_file_name
+  attr_accessor :security_groups
 
   VALID_OPTIONS = [:proxy, :credentials]
 
   def initialize(options = {}, group_uuid = nil)
     @group_uuid = group_uuid || (SecureRandom.uuid).gsub('-', '')
 
-    @security_group_name = nil
+    @security_groups = []
     @key_pair_name = nil
     @private_key_file_name = nil
 
+    @worker_keys = SSHKey.generate
+
     @private_key = nil # Private key data
 
     # List of instances
@@ -72,32 +75,33 @@ class OpenStudioAwsWrapper
     @aws = Aws::EC2::Client.new(options[:credentials])
   end
 
-  def create_or_retrieve_security_group(sg_name = nil)
-    tmp_name = sg_name || 'openstudio-server-sg-v1'
+  def create_or_retrieve_default_security_group(tmp_name = 'openstudio-server-sg-v2.1')
     group = @aws.describe_security_groups(filters: [{ name: 'group-name', values: [tmp_name] }])
     logger.info "Length of the security group is: #{group.data.security_groups.length}"
     if group.data.security_groups.length == 0
-      logger.info 'server group not found --- will create a new one'
+      logger.info 'security group not found --- will create a new one'
       @aws.create_security_group(group_name: tmp_name, description: "group dynamically created by #{__FILE__}")
       @aws.authorize_security_group_ingress(
           group_name: tmp_name,
           ip_permissions: [
-            { ip_protocol: 'tcp', from_port: 1, to_port: 65_535, ip_ranges: [cidr_ip: '0.0.0.0/0'] }
-          ]
-      )
-      @aws.authorize_security_group_ingress(
-          group_name: tmp_name,
-          ip_permissions: [
-            { ip_protocol: 'icmp', from_port: -1, to_port: -1, ip_ranges: [cidr_ip: '0.0.0.0/0']
-            }
+            { ip_protocol: 'tcp', from_port: 22, to_port: 22, ip_ranges: [cidr_ip: '0.0.0.0/0'] }, # Eventually make this only the user's IP address seen by the internet
+            { ip_protocol: 'tcp', from_port: 80, to_port: 80, ip_ranges: [cidr_ip: '0.0.0.0/0'] },
+            { ip_protocol: 'tcp', from_port: 443, to_port: 443, ip_ranges: [cidr_ip: '0.0.0.0/0'] },
+            { ip_protocol: 'tcp', from_port: 0, to_port: 65535, user_id_group_pairs: [{ group_name: tmp_name}]}, # allow all machines in the security group talk to each other openly
+            { ip_protocol: 'icmp', from_port: -1, to_port: -1, ip_ranges: [cidr_ip: '0.0.0.0/0'] }
           ]
       )
 
       # reload group information
       group = @aws.describe_security_groups(filters: [{ name: 'group-name', values: [tmp_name] }])
+    else
+      logger.info "Found existing security group"
     end
-    @security_group_name = group.data.security_groups.first.group_name
-    logger.info("server_group #{group.data.security_groups.first.group_name}")
+
+    @security_groups = [group.data.security_groups.first.group_id]
+    logger.info("server_group #{group.data.security_groups.first.group_name}:#{group.data.security_groups.first.group_id}")
+
+    group.data.security_groups.first
   end
 
   def describe_availability_zones
@@ -260,7 +264,17 @@ class OpenStudioAwsWrapper
   end
 
   def load_private_key(filename)
-    fail "Could not find private key #{filename}" unless File.exist? filename
+    if !File.exist? filename
+      # check if the file basename exists in your user directory
+      filename = File.expand_path("~/.ssh/#{File.basename(filename)}")
+      if File.exist? filename
+        puts "Found key of same name in user's home ssh folder #{filename}"
+        # using the key in your home directory
+      else
+        fail "Could not find private key #{filename}" unless File.exist? filename
+      end
+    end
+
     @private_key_file_name = File.expand_path filename
     @private_key = File.read(filename)
   end
@@ -275,15 +289,30 @@ class OpenStudioAwsWrapper
     end
   end
 
+  # save off the worker public/private keys that were created
+  def save_worker_keys(directory='.')
+    File.open("#{directory}/ec2_worker_key.pem", 'w') { |f| f << @worker_keys.private_key}
+    File.open("#{directory}/ec2_worker_key.pub", 'w') { |f| f << @worker_keys.public_key}
+  end
+
   def launch_server(image_id, instance_type, launch_options = {})
-    defaults = { user_id: 'unknown_user', tags: [], ebs_volume_size: nil }
+    defaults = {
+        user_id: 'unknown_user',
+        tags: [],
+        ebs_volume_size: nil
+    }
     launch_options = defaults.merge(launch_options)
 
-    user_data = File.read(File.expand_path(File.dirname(__FILE__)) + '/server_script.sh')
-    @server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, @security_group_name, @group_uuid, @private_key,
+    # replace the server_script.sh.template with the keys to add
+    user_data = File.read(File.expand_path(File.dirname(__FILE__)) + '/server_script.sh.template')
+    user_data.gsub!(/SERVER_HOSTNAME/, 'openstudio.server')
+    user_data.gsub!(/WORKER_PRIVATE_KEY_TEMPLATE/, worker_keys.private_key.gsub("\n","\\n"))
+    user_data.gsub!(/WORKER_PUBLIC_KEY_TEMPLATE/, worker_keys.ssh_public_key)
+
+    @server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, @security_groups, @group_uuid, @private_key,
                                         @private_key_file_name, @proxy)
 
-    # create the EBS volumes instead of the ephemeral storage - needed especially for the m3 instances (SSD)
+    # TODO: create the EBS volumes instead of the ephemeral storage - needed especially for the m3 instances (SSD)
 
     fail 'image_id is nil' unless image_id
     fail 'instance type is nil' unless instance_type
@@ -291,19 +320,25 @@ class OpenStudioAwsWrapper
   end
 
   def launch_workers(image_id, instance_type, num, launch_options = {})
-    defaults = { user_id: 'unknown_user', tags: [], ebs_volume_size: nil }
+    defaults = {
+        user_id: 'unknown_user',
+        tags: [],
+        ebs_volume_size: nil,
+        availability_zone: @server.data.availability_zone,
+        worker_public_key: ''
+    }
     launch_options = defaults.merge(launch_options)
 
     user_data = File.read(File.expand_path(File.dirname(__FILE__)) + '/worker_script.sh.template')
-    user_data.gsub!(/SERVER_IP/, @server.data.ip)
-    user_data.gsub!(/SERVER_HOSTNAME/, 'master')
-    user_data.gsub!(/SERVER_ALIAS/, '')
+    user_data.gsub!(/SERVER_IP/, @server.data.private_ip_address)
+    user_data.gsub!(/SERVER_HOSTNAME/, 'openstudio.server')
+    user_data.gsub!(/WORKER_PUBLIC_KEY_TEMPLATE/, worker_keys.ssh_public_key)
     logger.info("worker user_data #{user_data.inspect}")
 
     # thread the launching of the workers
 
     num.times do
-      @workers << OpenStudioAwsInstance.new(@aws, :worker, @key_pair_name, @security_group_name, @group_uuid,
+      @workers << OpenStudioAwsInstance.new(@aws, :worker, @key_pair_name, @security_groups, @group_uuid,
                                             @private_key, @private_key_file_name, @proxy)
     end
 
@@ -321,15 +356,17 @@ class OpenStudioAwsWrapper
   end
 
   # blocking method that waits for servers and workers to be fully configured (i.e. execution of user_data has
-  # occured on all nodes)
+  # occured on all nodes). Ideally none of these methods would ever need to exist.
+  #
+  # @return [Boolean] Will return true unless an exception is raised
   def configure_server_and_workers
     logger.info('waiting for server user_data to complete')
     @server.wait_command('[ -e /home/ubuntu/user_data_done ] && echo "true"')
     logger.info('waiting for worker user_data to complete')
     @workers.each { |worker| worker.wait_command('[ -e /home/ubuntu/user_data_done ] && echo "true"') }
 
-    ips = "master|#{@server.data.ip}|#{@server.data.dns}|#{@server.data.procs}|ubuntu|ubuntu|true\n"
-    @workers.each { |worker| ips << "worker|#{worker.data.ip}|#{worker.data.dns}|#{worker.data.procs}|ubuntu|ubuntu|true\n" }
+    ips = "master|#{@server.data.private_ip_address}|#{@server.data.dns}|#{@server.data.procs}|ubuntu|ubuntu|true\n"
+    @workers.each { |worker| ips << "worker|#{worker.data.private_ip_address}|#{worker.data.dns}|#{worker.data.procs}|ubuntu|ubuntu|true\n" }
     file = Tempfile.new('ip_addresses')
     file.write(ips)
     file.close
@@ -337,11 +374,9 @@ class OpenStudioAwsWrapper
     file.unlink
     logger.info("ips #{ips}")
     @server.shell_command('chmod 664 /home/ubuntu/ip_addresses')
-    @server.shell_command('~/setup-ssh-keys.sh')
-    @server.shell_command('~/setup-ssh-worker-nodes.sh ip_addresses')
 
     mongoid = File.read(File.expand_path(File.dirname(__FILE__)) + '/mongoid.yml.template')
-    mongoid.gsub!(/SERVER_IP/, @server.data.ip)
+    mongoid.gsub!(/SERVER_IP/, @server.data.private_ip_address)
     file = Tempfile.new('mongoid.yml')
     file.write(mongoid)
     file.close
@@ -352,8 +387,6 @@ class OpenStudioAwsWrapper
     @server.shell_command('chmod 664 /mnt/openstudio/rails-models/mongoid.yml')
     @workers.each { |worker| worker.shell_command('chmod 664 /mnt/openstudio/rails-models/mongoid.yml') }
 
-    # I'm removing this as of 10/6/14. This should have been resolved by now.
-    # sleep 60 # wait 60 more seconds for everything -- this is cheesy
     true
   end
 
@@ -361,25 +394,29 @@ class OpenStudioAwsWrapper
   # if it is found, then it will set the @server member variable.
   # Note that the information around keys and security groups is pulled from the instance information.
   def find_server(server_data_hash)
-    group_uuid = server_data_hash[:group_id] || @group_uuid
+    @group_uuid = server_data_hash[:group_id] || @group_uuid
     load_private_key(server_data_hash[:server][:private_key_file_name])
 
     logger.info "finding the server for groupid of #{group_uuid}"
     fail 'no group uuid defined either in member variable or method argument' if group_uuid.nil?
 
+ 	@server = nil
     resp = describe_running_instances(group_uuid, :server)
     if resp
       fail "more than one server running with group uuid of #{group_uuid} found, expecting only one" if resp.size > 1
       resp = resp.first
       if !@server
-        logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]"
-        @server = OpenStudioAwsInstance.new(@aws, :server, resp[:key_name], resp[:security_groups].first[:group_name], group_uuid, @private_key, @private_key_file_name)
-        @server.load_instance_data(resp)
+      	if resp
+        	logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]"
+        	@server = OpenStudioAwsInstance.new(@aws, :server, resp[:key_name], resp[:security_groups].first[:group_name], group_uuid, @private_key, @private_key_file_name, @proxy)
+
+        	@server.load_instance_data(resp)
+        end
       else
         logger.info "Server instance is already defined with instance #{resp[:instance_id]}"
       end
     else
-      fail 'could not find a running server instance'
+      puts 'could not find a running server instance'
     end
 
     # Really don't need to return anything because this sets the class instance variable
@@ -437,7 +474,8 @@ class OpenStudioAwsWrapper
           ip: "http://#{worker.data.ip}",
           dns: worker.data.dns,
           procs: worker.data.procs,
-          private_key_file_name: worker.private_key_file_name
+          private_key_file_name: worker.private_key_file_name,
+          private_ip_address: worker.private_ip_address
       )
     end
 
@@ -502,8 +540,13 @@ class OpenStudioAwsWrapper
         a[:tested] = false
       end
 
-      # TODO: in 1.6.0 just stop putting in the cc2workers.
-      if ami[:tags_hash][:openstudio_version].to_version >= '1.5.0'
+      if ami[:tags_hash][:openstudio_version].to_version >= '1.6.0'
+        if ami[:name] =~ /Server/
+          a[:amis][:server] = ami[:image_id]
+        elsif ami[:name] =~ /Worker/
+          a[:amis][:worker] = ami[:image_id]
+        end
+      elsif ami[:tags_hash][:openstudio_version].to_version >= '1.5.0'
         if ami[:name] =~ /Server/
           a[:amis][:server] = ami[:image_id]
         elsif ami[:name] =~ /Worker/