openstudio-aws 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e3f40caa3eed2715d9774ae42a8913b3d605434
-  data.tar.gz: ac233b57d15ba24a027019e94c919da9a0850cf0
+  metadata.gz: 62fe65f799f5982237127e58ec68bdd92ed883e1
+  data.tar.gz: fc6ece5979bc471e2ef8c5ca67d02ad9ebb5f1e8
 SHA512:
-  metadata.gz: 76a729d74b97e3b9fe3e8e737b303c38939443567d2006e9ebfd46ce709995a7d0e1a29305951ceeb362122e36d7a9a55968aba2e6127f03dfefae29ef7c94cf
-  data.tar.gz: 5d46f753111b741a8b51349d2e83c1ef75689f51a42d006a590d5e0795a1d4e7f46acb25c259d43188a3bca411fa66afe469cca186b14a04c849498b8458d092
+  metadata.gz: e01949a6660d24ebce0feafbe9a46f34e389ef302bcb015f92f4bbf9619b988f17ff63c9722de7b4d423503cd2d3ba791e9f427b0864026e24ba3f081ed359f8
+  data.tar.gz: a6f8f3a8d246b3bafbc1c61be9ecbba0ebc6c116b09a404ccac45c12826a0e53ca4a3584ac0164d1d63b72672bffd357a6dfaef5872a1623e5ed4c49e126642f

data/lib/openstudio/aws/aws.rb

@@ -55,12 +55,12 @@ module OpenStudio
           else
             proxy_uri = "https://#{options[:proxy][:host]}:#{options[:proxy][:port]}"
           end
-          # todo: remove this proxy_uri and make a method to format correctly
+          # TODO: remove this proxy_uri and make a method to format correctly
           options[:proxy_uri] = proxy_uri
 
-          # todo: do we need to escape a couple of the argument of username and password
+          # TODO: do we need to escape a couple of the argument of username and password
 
-          # todo: set some environment variables for system based proxy
+          # TODO: set some environment variables for system based proxy
         end
 
         # puts "Final options are: #{options.inspect}"
@@ -80,8 +80,6 @@ module OpenStudio
         @default_amis = OpenStudioAmis.new(options[:ami_lookup_version], ami_options).get_amis
       end
 
-
-
       # # def load_data_from_json(json_filename)
       #   @os_aws.load_data_from_json
       #
@@ -108,8 +106,8 @@ module OpenStudio
 
         # for backwards compatibilty, still allow security_group
         if options[:security_group]
-          warn "Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0"
-          options[:security_groups] = [ options[:security_group] ]
+          warn 'Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0'
+          options[:security_groups] = [options[:security_group]]
         end
 
         if options[:aws_key_pair_name]
@@ -136,10 +134,10 @@ module OpenStudio
         end
 
         server_options = {
-            user_id: options[:user_id],
-            tags: options[:tags],
-            subnet_id: options[:subnet_id],
-            associate_public_ip_address: options[:associate_public_ip_address],
+          user_id: options[:user_id],
+          tags: options[:tags],
+          subnet_id: options[:subnet_id],
+          associate_public_ip_address: options[:associate_public_ip_address]
         }
 
         # save the worker pem and public to the directory
@@ -181,11 +179,10 @@ module OpenStudio
 
         # for backwards compatibilty, still allow security_group
         if options[:security_group]
-          warn "Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0"
-          options[:security_groups] = [ options[:security_group] ]
+          warn 'Pass security_groups as an array instead of security_group. security_group will be deprecated in 0.4.0'
+          options[:security_groups] = [options[:security_group]]
         end
 
-
         # Get the right worker AMI ids based on the type of instance
         if options[:image_id].nil?
           options[:image_id] = determine_image_type(options[:instance_type])
@@ -198,10 +195,10 @@ module OpenStudio
           puts 'No workers requested'
         else
           worker_options = {
-              user_id: options[:user_id],
-              tags: options[:tags],
-              subnet_id: options[:subnet_id],
-              associate_public_ip_address: options[:associate_public_ip_address]
+            user_id: options[:user_id],
+            tags: options[:tags],
+            subnet_id: options[:subnet_id],
+            associate_public_ip_address: options[:associate_public_ip_address]
           }
 
           # if options[:ebs_volume_size]
@@ -288,7 +285,6 @@ module OpenStudio
           return false
         end
 
-        # load the worker nodes someday
         true
       end
 

data/lib/openstudio/aws/version.rb

@@ -1,5 +1,5 @@
 module OpenStudio
   module Aws
-    VERSION = '0.3.0'
+    VERSION = '0.3.1'
   end
 end

data/lib/openstudio/lib/ami_list.rb

@@ -99,7 +99,6 @@ class OpenStudioAmis
       # just grab the most recent server
       # need to do a sort to get the most recent because we can't promise that they are in order
       json[:openstudio].each do |_k, _v|
-
       end
       key, value = json[:openstudio_server].first
 
@@ -115,9 +114,27 @@ class OpenStudioAmis
 
   private
 
+  # fetch the URL with redirects
+  def fetch(uri_str, limit = 10)
+    # You should choose better exception.
+    fail ArgumentError, 'HTTP redirect too deep' if limit == 0
+
+    url = URI.parse(uri_str)
+    req = Net::HTTP::Get.new(url.path)
+    response = Net::HTTP.start(url.host, url.port) { |http| http.request(req) }
+    case response
+      when Net::HTTPSuccess then
+        response
+      when Net::HTTPRedirection then
+        fetch(response['location'], limit - 1)
+      else
+        response.error!
+    end
+  end
+
   def retrieve_json(endpoint)
     result = nil
-    resp = Net::HTTP.get_response(@options[:host], endpoint)
+    resp = fetch("http://#{@options[:host]}/#{endpoint}")
     if resp.code == '200'
       result = JSON.parse(resp.body, symbolize_names: true)
     else

data/lib/openstudio/lib/openstudio_aws_instance.rb

@@ -27,7 +27,6 @@ class OpenStudioAwsInstance
   attr_reader :private_ip_address
   attr_reader :group_uuid
 
-
   # param security_groups can be a single instance or an array
   def initialize(aws_session, openstudio_instance_type, key_pair_name, security_groups, group_uuid, private_key,
                  private_key_file_name, subnet_id, proxy = nil)
@@ -78,10 +77,10 @@ class OpenStudioAwsInstance
     # need to wait until it is available
 
     @aws.attach_volume(
-          volume_id: resp[:volume_id],
-          instance_id: instance_id,
-          # required
-          device: '/dev/sdh'
+      volume_id: resp[:volume_id],
+      instance_id: instance_id,
+      # required
+      device: '/dev/sdh'
     )
 
     # Wait for the volume to attach
@@ -92,14 +91,14 @@ class OpenStudioAwsInstance
   def launch_instance(image_id, instance_type, user_data, user_id, options = {})
     # determine the instance type of the server
     instance = {
-        image_id: image_id,
-        key_name: @key_pair_name,
-        security_group_ids: @security_groups,
-        subnet_id: options[:subnet_id],
-        user_data: Base64.encode64(user_data),
-        instance_type: instance_type,
-        min_count: 1,
-        max_count: 1
+      image_id: image_id,
+      key_name: @key_pair_name,
+      security_group_ids: @security_groups,
+      subnet_id: options[:subnet_id],
+      user_data: Base64.encode64(user_data),
+      instance_type: instance_type,
+      min_count: 1,
+      max_count: 1
     }
 
     if options[:availability_zone]
@@ -113,19 +112,18 @@ class OpenStudioAwsInstance
       # You have to delete the security group and subnet_id from the instance hash and put into the network interface
       # otherwise you will get an error on launch with an InvalidParameterCombination error.
       instance[:network_interfaces] = [
-          {
-              subnet_id: instance.delete(:subnet_id),
-              groups: instance.delete(:security_group_ids),
-              device_index: 0,
-              associate_public_ip_address: true
-          }
+        {
+          subnet_id: instance.delete(:subnet_id),
+          groups: instance.delete(:security_group_ids),
+          device_index: 0,
+          associate_public_ip_address: true
+        }
       ]
     end
 
     # Documentation for run_instances is here: http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/EC2/Client.html#run_instances-instance_method
     result = @aws.run_instances(instance)
 
-
     # determine how many processors are suppose to be in this image (lookup for now?)
     processors = find_processors(instance_type)
 
@@ -160,15 +158,14 @@ class OpenStudioAwsInstance
       tries ||= 3
       aws_instance = result.data.instances.first
       @aws.create_tags(
-          resources: [aws_instance.instance_id],
-          tags: aws_tags
+        resources: [aws_instance.instance_id],
+        tags: aws_tags
       )
     rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
       sleep 1000
       retry unless (tries -= 1).zero?
     end
 
-
     # get the instance information
     test_result = @aws.describe_instance_status(instance_ids: [aws_instance.instance_id]).data.instance_statuses.first
     begin
@@ -190,7 +187,7 @@ class OpenStudioAwsInstance
     end
 
     # now grab information about the instance
-    # todo: check lengths on all of arrays
+    # TODO: check lengths on all of arrays
     instance_data = @aws.describe_instances(instance_ids: [aws_instance.instance_id]).data.reservations.first.instances.first.to_hash
     logger.info "instance description is: #{instance_data}"
 
@@ -427,13 +424,13 @@ class OpenStudioAwsInstance
   def create_struct(instance, procs)
     instance_struct = Struct.new(:instance, :id, :ip, :dns, :procs, :availability_zone, :private_ip_address)
     s = instance_struct.new(
-        instance,
-        instance[:instance_id],
-        instance[:public_ip_address],
-        instance[:public_dns_name],
-        procs,
-        instance[:placement][:availability_zone],
-        instance[:private_ip_address]
+      instance,
+      instance[:instance_id],
+      instance[:public_ip_address],
+      instance[:public_dns_name],
+      procs,
+      instance[:placement][:availability_zone],
+      instance[:private_ip_address]
     )
 
     # store some values into the member variables

data/lib/openstudio/lib/openstudio_aws_wrapper.rb

@@ -82,20 +82,20 @@ class OpenStudioAwsWrapper
       logger.info 'security group not found --- will create a new one'
       @aws.create_security_group(group_name: tmp_name, description: "group dynamically created by #{__FILE__}")
       @aws.authorize_security_group_ingress(
-          group_name: tmp_name,
-          ip_permissions: [
-            { ip_protocol: 'tcp', from_port: 22, to_port: 22, ip_ranges: [cidr_ip: '0.0.0.0/0'] }, # Eventually make this only the user's IP address seen by the internet
-            { ip_protocol: 'tcp', from_port: 80, to_port: 80, ip_ranges: [cidr_ip: '0.0.0.0/0'] },
-            { ip_protocol: 'tcp', from_port: 443, to_port: 443, ip_ranges: [cidr_ip: '0.0.0.0/0'] },
-            { ip_protocol: 'tcp', from_port: 0, to_port: 65535, user_id_group_pairs: [{ group_name: tmp_name}]}, # allow all machines in the security group talk to each other openly
-            { ip_protocol: 'icmp', from_port: -1, to_port: -1, ip_ranges: [cidr_ip: '0.0.0.0/0'] }
-          ]
+        group_name: tmp_name,
+        ip_permissions: [
+          { ip_protocol: 'tcp', from_port: 22, to_port: 22, ip_ranges: [cidr_ip: '0.0.0.0/0'] }, # Eventually make this only the user's IP address seen by the internet
+          { ip_protocol: 'tcp', from_port: 80, to_port: 80, ip_ranges: [cidr_ip: '0.0.0.0/0'] },
+          { ip_protocol: 'tcp', from_port: 443, to_port: 443, ip_ranges: [cidr_ip: '0.0.0.0/0'] },
+          { ip_protocol: 'tcp', from_port: 0, to_port: 65535, user_id_group_pairs: [{ group_name: tmp_name }] }, # allow all machines in the security group talk to each other openly
+          { ip_protocol: 'icmp', from_port: -1, to_port: -1, ip_ranges: [cidr_ip: '0.0.0.0/0'] }
+        ]
       )
 
       # reload group information
       group = @aws.describe_security_groups(filters: [{ name: 'group-name', values: [tmp_name] }])
     else
-      logger.info "Found existing security group"
+      logger.info 'Found existing security group'
     end
 
     @security_groups = [group.data.security_groups.first.group_id]
@@ -134,11 +134,11 @@ class OpenStudioAwsWrapper
     resp = nil
     if group_uuid
       resp = @aws.describe_instances(
-          filters: [
-            { name: 'instance-state-code', values: [0.to_s, 16.to_s] }, # running or pending
-            { name: 'tag-key', values: ['GroupUUID'] },
-            { name: 'tag-value', values: [group_uuid.to_s] }
-          ]
+        filters: [
+          { name: 'instance-state-code', values: [0.to_s, 16.to_s] }, # running or pending
+          { name: 'tag-key', values: ['GroupUUID'] },
+          { name: 'tag-value', values: [group_uuid.to_s] }
+        ]
       )
     else
       resp = @aws.describe_instances
@@ -153,7 +153,7 @@ class OpenStudioAwsWrapper
           if group_uuid && openstudio_instance_type
             # {:key=>"Purpose", :value=>"OpenStudioWorker"}
             if i_h[:tags].any? { |h| (h[:key] == 'Purpose') && (h[:value] == "OpenStudio#{openstudio_instance_type.capitalize}") } &&
-                i_h[:tags].any? { |h| (h[:key] == 'GroupUUID') && (h[:value] == group_uuid.to_s) }
+               i_h[:tags].any? { |h| (h[:key] == 'GroupUUID') && (h[:value] == group_uuid.to_s) }
               instance_data << i_h
             end
           elsif group_uuid
@@ -177,7 +177,7 @@ class OpenStudioAwsWrapper
   def describe_amis(filter = nil, image_ids = [], owned_by_me = true)
     resp = nil
 
-    # todo: test the filter.  i don't think that it is exposed in the AWS gem?
+    # TODO: test the filter.  i don't think that it is exposed in the AWS gem?
     if owned_by_me
       if filter
         resp = @aws.describe_images(owners: [:self], filter: filter).data
@@ -212,8 +212,8 @@ class OpenStudioAwsWrapper
 
   def stop_instances(ids)
     resp = @aws.stop_instances(
-        instance_ids: ids,
-        force: true
+      instance_ids: ids,
+      force: true
     )
 
     resp
@@ -222,7 +222,7 @@ class OpenStudioAwsWrapper
   def terminate_instances(ids)
     begin
       resp = @aws.terminate_instances(
-          instance_ids: ids
+        instance_ids: ids
       )
     rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
       # Log that the instances couldn't be found?
@@ -264,7 +264,7 @@ class OpenStudioAwsWrapper
   end
 
   def load_private_key(filename)
-    if !File.exist? filename
+    unless File.exist? filename
       # check if the file basename exists in your user directory
       filename = File.expand_path("~/.ssh/#{File.basename(filename)}")
       if File.exist? filename
@@ -290,23 +290,23 @@ class OpenStudioAwsWrapper
   end
 
   # save off the worker public/private keys that were created
-  def save_worker_keys(directory='.')
-    File.open("#{directory}/ec2_worker_key.pem", 'w') { |f| f << @worker_keys.private_key}
-    File.open("#{directory}/ec2_worker_key.pub", 'w') { |f| f << @worker_keys.public_key}
+  def save_worker_keys(directory = '.')
+    File.open("#{directory}/ec2_worker_key.pem", 'w') { |f| f << @worker_keys.private_key }
+    File.open("#{directory}/ec2_worker_key.pub", 'w') { |f| f << @worker_keys.public_key }
   end
 
   def launch_server(image_id, instance_type, launch_options = {})
     defaults = {
-        user_id: 'unknown_user',
-        tags: [],
-        ebs_volume_size: nil
+      user_id: 'unknown_user',
+      tags: [],
+      ebs_volume_size: nil
     }
     launch_options = defaults.merge(launch_options)
 
     # replace the server_script.sh.template with the keys to add
     user_data = File.read(File.expand_path(File.dirname(__FILE__)) + '/server_script.sh.template')
     user_data.gsub!(/SERVER_HOSTNAME/, 'openstudio.server')
-    user_data.gsub!(/WORKER_PRIVATE_KEY_TEMPLATE/, worker_keys.private_key.gsub("\n","\\n"))
+    user_data.gsub!(/WORKER_PRIVATE_KEY_TEMPLATE/, worker_keys.private_key.gsub("\n", '\\n'))
     user_data.gsub!(/WORKER_PUBLIC_KEY_TEMPLATE/, worker_keys.ssh_public_key)
 
     @server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, @security_groups, @group_uuid, @private_key,
@@ -321,11 +321,11 @@ class OpenStudioAwsWrapper
 
   def launch_workers(image_id, instance_type, num, launch_options = {})
     defaults = {
-        user_id: 'unknown_user',
-        tags: [],
-        ebs_volume_size: nil,
-        availability_zone: @server.data.availability_zone,
-        worker_public_key: ''
+      user_id: 'unknown_user',
+      tags: [],
+      ebs_volume_size: nil,
+      availability_zone: @server.data.availability_zone,
+      worker_public_key: ''
     }
     launch_options = defaults.merge(launch_options)
 
@@ -351,8 +351,8 @@ class OpenStudioAwsWrapper
     end
     threads.each(&:join)
 
-    # todo: do we need to have a flag if the worker node is successful?
-    # todo: do we need to check the current list of running workers?
+    # TODO: do we need to have a flag if the worker node is successful?
+    # TODO: do we need to check the current list of running workers?
   end
 
   # blocking method that waits for servers and workers to be fully configured (i.e. execution of user_data has
@@ -400,17 +400,18 @@ class OpenStudioAwsWrapper
     logger.info "finding the server for groupid of #{group_uuid}"
     fail 'no group uuid defined either in member variable or method argument' if group_uuid.nil?
 
- 	@server = nil
+    # This should really just be a single call to describe running instances
+    @server = nil
     resp = describe_running_instances(group_uuid, :server)
     if resp
       fail "more than one server running with group uuid of #{group_uuid} found, expecting only one" if resp.size > 1
       resp = resp.first
       if !@server
-      	if resp
-        	logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]"
-        	@server = OpenStudioAwsInstance.new(@aws, :server, resp[:key_name], resp[:security_groups].first[:group_name], group_uuid, @private_key, @private_key_file_name, @proxy)
+        if resp
+          logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]"
+          @server = OpenStudioAwsInstance.new(@aws, :server, resp[:key_name], resp[:security_groups].first[:group_name], group_uuid, @private_key, @private_key_file_name, @proxy)
 
-        	@server.load_instance_data(resp)
+          @server.load_instance_data(resp)
         end
       else
         logger.info "Server instance is already defined with instance #{resp[:instance_id]}"
@@ -419,6 +420,19 @@ class OpenStudioAwsWrapper
       puts 'could not find a running server instance'
     end
 
+    # find the workers
+    if @workers.size == 0
+      resp = describe_running_instances(group_uuid, :worker)
+      if resp
+        resp.each do |r|
+          @workers << OpenStudioAwsInstance.new(@aws, :worker, r[:key_name], r[:security_groups].first[:group_name], group_uuid, @private_key, @private_key_file_name, @proxy)
+          @workers.last.load_instance_data(r)
+        end
+      end
+    else
+      logger.info 'Worker nodes are already defined'
+    end
+
     # Really don't need to return anything because this sets the class instance variable
     @server
   end
@@ -439,7 +453,7 @@ class OpenStudioAwsWrapper
       version1 = {}
 
       # now grab the good keys - they should be sorted newest to older... so go backwards
-      amis[:openstudio_server].keys.reverse.each do |key|
+      amis[:openstudio_server].keys.reverse_each do |key|
         next if amis[:openstudio_server][key][:deprecate]
 
         a = amis[:openstudio_server][key]
@@ -470,12 +484,12 @@ class OpenStudioAwsWrapper
     worker_hash = []
     @workers.each do |worker|
       worker_hash.push(
-          id: worker.data.id,
-          ip: "http://#{worker.data.ip}",
-          dns: worker.data.dns,
-          procs: worker.data.procs,
-          private_key_file_name: worker.private_key_file_name,
-          private_ip_address: worker.private_ip_address
+        id: worker.data.id,
+        ip: "http://#{worker.data.ip}",
+        dns: worker.data.dns,
+        procs: worker.data.procs,
+        private_key_file_name: worker.private_key_file_name,
+        private_ip_address: worker.private_ip_address
       )
     end
 
@@ -495,9 +509,7 @@ class OpenStudioAwsWrapper
     end
 
     # get the max version in the list_of_svs
-    while list_of_svs.include?(b.to_s)
-      b.patch += 1
-    end
+    b.patch += 1 while list_of_svs.include?(b.to_s)
 
     # return the value back as a string
     b.to_s