openstudio-aws 0.4.0.alpha2 → 0.4.0.alpha3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eba77b1f1302604cb57c69a087796ead6aaf2c98
-  data.tar.gz: 113a23d7859a8da882a7f5e426098cbf6ed61986
+  metadata.gz: 2f024e917704db3001f98d70663da27d0a998d54
+  data.tar.gz: 43af587c2f58ff2690c1a91ccd242b4f133c0171
 SHA512:
-  metadata.gz: b01c424a5369cddce55c78adfb87b3c330158faa4e0b25f7148106c5947db6f8df7dbd91e4d1e3d4749e6548476743b5edd5865477e3b864b6d034539a3c8bab
-  data.tar.gz: 99026903cb962cbb0e8990aa97236ce645336346d1977be7027a4aee58757768d835b4869957dd5965f4629ee0f17dc2da1f62ede3c3fb803873253181e87333
+  metadata.gz: 8f7b216034b3aaee08482319454aef815a9b3437c21e1dc2821fe2393cf483179c870e3eaf0d05987cd82d62d1dca795cce4a287d871ffcd622cb0f11c56a5d2
+  data.tar.gz: 3d087837a34a9c847b4f72100ac94ca1328ea2e88909009fa4eb44d7f2db6750010a9f75b14938d96b111e6513adea75c77d4f4d7ed9260daab02dcae156d4f0

data/CHANGELOG.md

@@ -1,6 +1,11 @@
 OpenStudio AWS Gem Change Log
 ==================================
 
+Version 0.4.0.alpha3
+-------------
+* Load the key name and security groups from the AWS instance information. 
+* Add method on Aws to get the group_uuid 
+
 Version 0.4.0.alpha2
 -------------
 * Load worker keys from disk (if they exist) when constructing the OpenStudioAwsWrapper class

data/lib/openstudio/aws/aws.rb

@@ -138,7 +138,7 @@ module OpenStudio
           @os_aws.private_key_file_name = options[:private_key_file_name]
         else
           # Save the private key if you did not pass in an already existing key_pair_name
-          @os_aws.save_private_key(@save_directory)
+          @os_aws.save_private_key @save_directory
         end
 
         server_options = {
@@ -150,32 +150,16 @@ module OpenStudio
 
         # save the worker pem and public to the directory
         # presently, this will always overwrite the worker key, is that okay? Is this really needed later?
-        @os_aws.save_worker_keys(@save_directory)
-
-        # if instance_data[:ebs_volume_id]
-        #   server_options[:ebs_volume_id] = instance_data[:ebs_volume_id]
-        # end
+        @os_aws.save_worker_keys @save_directory
 
         @os_aws.launch_server(options[:image_id], options[:instance_type], server_options)
       end
 
-      # Write out to the terminal the connection information for the servers and workers
+      # create workers after the server has been created.
       #
-      # @return [nil] Only prints to the screen. No return is expected
-      def print_connection_info
-        # Print out some debugging commands (probably work on mac/linux only)
-        puts ''
-        puts 'Server SSH Command:'
-        puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
-        if @os_aws.workers.size > 0
-          puts ''
-          puts 'Worker SSH Command:'
-          @os_aws.workers.each do |worker|
-            puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{worker.data[:dns]}"
-          end
-        end
-      end
-
+      # @param number_of_instances [Integer] Number of worker instances to create
+      # @param options [Hash]
+      # @option options [String] :instance_type Type of server to start (e.g. m3.medium, m3.xlarge, etc.)
       def create_workers(number_of_instances, options = {}, user_id = 'unknown_user')
         defaults = {
           instance_type: 'm2.4xlarge',
@@ -226,13 +210,27 @@ module OpenStudio
         @os_aws.configure_server_and_workers
       end
 
+      # Write out to the terminal the connection information for the servers and workers
+      #
+      # @return [nil] Only prints to the screen. No return is expected
+      def print_connection_info
+        # Print out some debugging commands (probably work on mac/linux only)
+        puts ''
+        puts 'Server SSH Command:'
+        puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
+        if @os_aws.workers.size > 0
+          puts ''
+          puts 'Worker SSH Command:'
+          @os_aws.workers.each do |worker|
+            puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{worker.data[:dns]}"
+          end
+        end
+      end
+
       # Return information on the cluster instances as a hash. This includes IP addresses, host names, number of processors, etc.
       # @return [Hash] Data about the configured cluster
       def cluster_info
-        h = @os_aws.server.to_os_hash
-        h[:workers] = @os_aws.to_os_worker_hash[:workers]
-
-        h
+        @os_aws.to_os_hash
       end
 
       # Save a JSON with information about the cluster that was configured.
@@ -416,6 +414,13 @@ module OpenStudio
         image
       end
 
+      # Return the Group UUID as defined in the AWS wrapper
+      #
+      # @return [String] UUID
+      def group_uuid
+        @os_aws.group_uuid
+      end
+
       private
 
       def os_aws_file_location

data/lib/openstudio/aws/version.rb

@@ -1,5 +1,5 @@
 module OpenStudio
   module Aws
-    VERSION = '0.4.0.alpha2'
+    VERSION = '0.4.0.alpha3'
   end
 end

data/lib/openstudio/lib/openstudio_aws_wrapper.rb

@@ -65,7 +65,7 @@ class OpenStudioAwsWrapper
     work_dir = options[:save_directory] || '.'
     if File.exist?(File.join(work_dir, 'ec2_worker_key.pem')) && File.exist?(File.join(work_dir, 'ec2_worker_key.pub'))
       logger.info "Worker keys already exist, loading from #{work_dir}"
-      @worker_keys = SSHKey.new(File.read(File.join(work_dir, 'ec2_worker_key.pem')))
+      load_worker_key(File.join(work_dir, 'ec2_worker_key.pem'))
     else
       logger.info 'Generating new worker keys'
       @worker_keys = SSHKey.generate
@@ -338,13 +338,24 @@ class OpenStudioAwsWrapper
     @private_key = File.read(filename)
   end
 
+  # Load the worker key for communicating between the server and worker instances on AWS. The public key
+  # will be automatically created when loading the private key
+  #
+  # @param private_key_filename [String] Fully qualified path to the worker private key
+  def load_worker_key(private_key_filename)
+    logger.info "Loading worker keys from #{private_key_filename}"
+    @worker_keys_filename = private_key_filename
+    @worker_keys = SSHKey.new(File.read(@worker_keys_filename))
+  end
+
+  # Save the private key to disk
   def save_private_key(directory = '.', filename = 'ec2_server_key.pem')
     if @private_key
       @private_key_file_name = File.expand_path "#{directory}/#{filename}"
       logger.info "Saving server private key in #{@private_key_file_name}"
       File.open(@private_key_file_name, 'w') { |f| f << @private_key }
       logger.info 'Setting permissions of server private key to 0600'
-
+      File.chmod(0600, @private_key_file_name)
     else
       fail "No private key found in which to persist with filename #{filename}"
     end
@@ -352,11 +363,11 @@ class OpenStudioAwsWrapper
 
   # save off the worker public/private keys that were created
   def save_worker_keys(directory = '.')
-    wk = "#{directory}/ec2_worker_key.pem"
-    logger.info "Saving worker private key in #{wk}"
-    File.open(wk, 'w') { |f| f << @worker_keys.private_key }
+    @worker_keys_filename = "#{directory}/ec2_worker_key.pem"
+    logger.info "Saving worker private key in #{@worker_keys_filename}"
+    File.open(@worker_keys_filename, 'w') { |f| f << @worker_keys.private_key }
     logger.info 'Setting permissions of worker private key to 0600'
-    File.chmod(0600, wk)
+    File.chmod(0600, @worker_keys_filename)
 
     wk = "#{directory}/ec2_worker_key.pub"
     logger.info "Saving worker public key in #{wk}"
@@ -403,7 +414,6 @@ class OpenStudioAwsWrapper
     logger.info("worker user_data #{user_data.inspect}")
 
     # thread the launching of the workers
-
     num.times do
       @workers << OpenStudioAwsInstance.new(@aws, :worker, @key_pair_name, @security_groups, @group_uuid,
                                             @private_key, @private_key_file_name, @proxy)
@@ -458,14 +468,19 @@ class OpenStudioAwsWrapper
   end
 
   # method to query the amazon api to find the server (if it exists), based on the group id
-  # if it is found, then it will set the @server member variable.
+  # if it is found, then it will set the @server instance variable. The security groups are assigned from the
+  # server node information on AWS if the security groups have not been initialized yet.
+  #
   # Note that the information around keys and security groups is pulled from the instance information.
+  # @param server_data_hash [Hash] Server data
+  # @option server_data_hash [String] :group_id Group ID of the analysis
+  # @option server_data_hash [String] :server.private_key_file_name Name of the private key to communicate to the server
   def find_server(server_data_hash)
     @group_uuid = server_data_hash[:group_id] || @group_uuid
     load_private_key(server_data_hash[:server][:private_key_file_name])
 
     logger.info "Finding the server for GroupUUID of #{group_uuid}"
-    fail 'no GroupUUID defined either in member variable or method argument' if group_uuid.nil?
+    fail 'no GroupUUID defined either in member variable or method argument' if @group_uuid.nil?
 
     # This should really just be a single call to describe running instances
     @server = nil
@@ -476,7 +491,17 @@ class OpenStudioAwsWrapper
       if !@server
         if resp
           logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]"
-          @server = OpenStudioAwsInstance.new(@aws, :server, resp[:key_name], resp[:security_groups].first[:group_name], group_uuid, @private_key, @private_key_file_name, @proxy)
+
+          sg = resp[:security_groups].map { |s| s[:group_id] }
+          # Set the security groups of the object if these groups haven't been assigned yet.
+          @security_groups = sg if @security_groups.empty?
+          logger.info "The security groups in aws wrapper are #{@security_groups}"
+
+          # set the key name from AWS if it isn't yet assigned
+          logger.info 'Setting the keyname in the aws wrapper'
+          @key_pair_name = resp[:key_name] unless @key_pair_name
+
+          @server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, sg, @group_uuid, @private_key, @private_key_file_name, @proxy)
 
           @server.load_instance_data(resp)
         end
@@ -487,12 +512,12 @@ class OpenStudioAwsWrapper
       logger.info 'could not find a running server instance'
     end
 
-    # find the workers
+    # Find the worker instances.
     if @workers.size == 0
       resp = describe_running_instances(group_uuid, :worker)
       if resp
         resp.each do |r|
-          @workers << OpenStudioAwsInstance.new(@aws, :worker, r[:key_name], r[:security_groups].first[:group_name], group_uuid, @private_key, @private_key_file_name, @proxy)
+          @workers << OpenStudioAwsInstance.new(@aws, :worker, r[:key_name], r[:security_groups].map { |s| s[:group_id] }, @group_uuid, @private_key, @private_key_file_name, @proxy)
           @workers.last.load_instance_data(r)
         end
       end
@@ -500,6 +525,10 @@ class OpenStudioAwsWrapper
       logger.info 'Worker nodes are already defined'
     end
 
+    # set the private key from the hash
+    load_private_key server_data_hash[:server][:private_key_file_name]
+    load_worker_key server_data_hash[:server][:worker_private_key_file_name]
+
     # Really don't need to return anything because this sets the class instance variable
     @server
   end
@@ -558,23 +587,23 @@ class OpenStudioAwsWrapper
     amis
   end
 
-  def to_os_worker_hash
-    worker_hash = []
-    @workers.each do |worker|
-      worker_hash.push(
+  # save off the instance configuration and instance information into a JSON file for later use
+  def to_os_hash
+    h = @server.to_os_hash
+
+    h[:server][:worker_private_key_file_name] = @worker_keys_filename
+    h[:workers] = @workers.map do |worker|
+      {
         id: worker.data.id,
         ip: "http://#{worker.data.ip}",
         dns: worker.data.dns,
         procs: worker.data.procs,
         private_key_file_name: worker.private_key_file_name,
         private_ip_address: worker.private_ip_address
-      )
+      }
     end
 
-    out = { workers: worker_hash }
-    logger.info out
-
-    out
+    h
   end
 
   # take the base version and increment the patch until

data/spec/aws_instances/aws_spec_api.rb

@@ -290,4 +290,58 @@ describe OpenStudio::Aws::Aws do
       expect(@aws_2.os_aws.worker_keys.public_key).not_to be_nil
     end
   end
+
+  context 'stateful creation of server and worker' do
+    before(:all) do
+      @config = OpenStudio::Aws::Config.new
+      @aws = OpenStudio::Aws::Aws.new
+    end
+
+    it 'should create the server and save the state' do
+      options = {
+        instance_type: 'm3.medium',
+        image_id: SERVER_AMI,
+        tags: [
+          'ci_tests=true',
+          'ServerOnly=true'
+        ]
+      }
+
+      test_pem_file = 'ec2_server_key.pem'
+      FileUtils.rm_f test_pem_file if File.exist? test_pem_file
+      FileUtils.rm_f 'server_data.json' if File.exist? 'server_data.json'
+
+      @aws.create_server(options)
+      @aws.save_cluster_info 'server_data.json'
+
+      h = @aws.os_aws.server.to_os_hash
+      expect(h[:group_id]).to be_a String
+      expect(h[:group_id]).to match /^[\d\S]{32}$/
+      expect(h[:location]).to eq 'AWS'
+    end
+
+    it 'should load server information from json and launch worker' do
+      aws2 = OpenStudio::Aws::Aws.new
+      aws2.load_instance_info_from_file('server_data.json')
+
+      options = {
+        instance_type: 'm3.medium',
+        image_id: WORKER_AMI
+      }
+
+      aws2.create_workers(1, options)
+      aws2.save_cluster_info 'server_data.json'
+
+      expect(File.exist?('server_data.json')).to eq true
+      # check if file exists
+
+      h = aws2.cluster_info # to make sure that the settings are correct
+      expect(h[:server][:worker_private_key_file_name]).to match /.*ec2_worker_key.pem/
+      expect(h[:workers].size).to eq 1
+    end
+
+    after :all do
+      @aws.terminate
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: openstudio-aws
 version: !ruby/object:Gem::Version
-  version: 0.4.0.alpha2
+  version: 0.4.0.alpha3
 platform: ruby
 authors:
 - Nicholas Long
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-19 00:00:00.000000000 Z
+date: 2015-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-scp