openstudio-aws 0.1.26 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e6440da99b4622bfa5010fcd957a30dd3ff27581
-  data.tar.gz: 840fbca886816df6a5d6994c90b8799ef25dbb01
+  metadata.gz: 3a6943903d87487ab5447ec19e96377828e2521b
+  data.tar.gz: 76dbc1bbe397a96f2324d2955dc7810dbf0bd210
 SHA512:
-  metadata.gz: 491563e8b5dac430f43bb89c7db36430c65263da3778c4f6334cb06811a13e546a59f49b3dbaa229d6d2e3052d8daf4db601c9caec862ce956f754fb5025d6d5
-  data.tar.gz: 1410e466c2c92c0f8ada366361aa37ca20ca3fbe297d4c2fe3e8598ee2b9c52cf197f9b40a9e82460095698f7e787fe60d32abc9ef76216075b5c6a13200cb8d
+  metadata.gz: 032ae2643fc3a677c1d20f21c25f8aa47e4393894beb0a704d31ddeab50fbb9b0116c0021cd6e40c5a43581c07b8a996d541304f91d55d39ff7fb652734d6a5c
+  data.tar.gz: 1945051b63a42f00ab601879e88bd12c5dcd47f5b0571860653014ead0334182a43ab3248c09a8f709e04ce8c9268f8254baddbfd93b03ca6bb8a49acecd1076

data/README.md

@@ -17,4 +17,4 @@ Development Notes
 There is an underlying attempt to merge several OpenStudio based gems into one and have a general 
 OpenStudio namespace for ruby classes.
 
-The AWS SDK gem is custom built and distributed with this gem in order to remove the JSON dependency.  If you are running ruby 1.9 or greater, then JSON is installed in teh std lib and should be called out in the gemspec.
+The AWS SDK gem is custom built and distributed with this gem in order to remove the JSON dependency.  If you are running ruby 1.9 or greater, then JSON is installed in the std lib and should be called out in the gemspec.

data/Rakefile

@@ -1,76 +1,75 @@
-require "bundler" 
+require 'bundler'
 Bundler.setup
 
-require "rake"
-require "rspec/core/rake_task"
+require 'rake'
+require 'rspec/core/rake_task'
 
-$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
-require "openstudio/aws/version"
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+require 'openstudio/aws/version'
 
-task :gem => :build
+task gem: :build
 task :build do
-  system "gem build openstudio-aws.gemspec"
+  system 'gem build openstudio-aws.gemspec'
 end
 
-desc "build and install gem locally"
-task :install => :build do
+desc 'build and install gem locally'
+task install: :build do
   system "gem install openstudio-aws-#{OpenStudio::Aws::VERSION}.gem --no-ri --no-rdoc"
 end
 
-desc "build and release the gem using rubygems and git tags"
-task :release => :build do
+desc 'build and release the gem using rubygems and git tags'
+task release: :build do
   system "git tag -a v#{OpenStudio::Aws::VERSION} -m 'Tagging #{OpenStudio::Aws::VERSION}'"
-  system "git push --tags"
+  system 'git push --tags'
   system "gem push openstudio-aws-#{OpenStudio::Aws::VERSION}.gem"
   system "rm openstudio-aws-#{OpenStudio::Aws::VERSION}.gem"
 end
 
-RSpec::Core::RakeTask.new("spec") do |spec|
+RSpec::Core::RakeTask.new('spec') do |spec|
   spec.rspec_opts = %w(--format progress --format CI::Reporter::RSpec)
-  spec.pattern = "spec/**/*_spec.rb"
+  spec.pattern = 'spec/**/*_spec.rb'
 end
 
-RSpec::Core::RakeTask.new("spec:api") do |spec|
+RSpec::Core::RakeTask.new('spec:api') do |spec|
   spec.rspec_opts = %w(--format progress --format CI::Reporter::RSpec)
-  spec.pattern = "spec/**/*_spec_api.rb"
+  spec.pattern = 'spec/**/*_spec_api.rb'
 end
 
-task :default => :spec
+task default: :spec
 
-desc "import files from other repos"
+desc 'import files from other repos'
 task :import_files do
-  puts "Importing data from other repos until this repo is self contained"
+  puts 'Importing data from other repos until this repo is self contained'
   # Copy data from github openstudio source
-  
+
   # this has been forked at the moment until openstudio team supports the new aws-sdk gem
-  #os_file = "./lib/openstudio/lib/os-aws.rb"
-  #system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/aws.rb.in"
-  #if File.exists?(os_file)
+  # os_file = "./lib/openstudio/lib/os-aws.rb"
+  # system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/aws.rb.in"
+  # if File.exists?(os_file)
   #  system "ruby -i -pe 'puts \"# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files\" if $.==1' #{os_file}"
   #  system "sed -i '' 's/\${CMAKE_VERSION_MAJOR}.\${CMAKE_VERSION_MINOR}.\${CMAKE_VERSION_PATCH}/#{OpenStudio::Aws::OPENSTUDIO_VERSION}/g' #{os_file}"
-  #end      
+  # end
   #
-  os_file = "./lib/openstudio/lib/mongoid.yml.template"
+  os_file = './lib/openstudio/lib/mongoid.yml.template'
   system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/mongoid.yml.template"
 
-  os_file = "./lib/openstudio/lib/server_script.sh"
+  os_file = './lib/openstudio/lib/server_script.sh'
   system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/server_script.sh"
-  if File.exists?(os_file)
+  if File.exist?(os_file)
     system "ruby -i -pe 'puts \"# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files\" if $.==2' #{os_file}"
   end
 
-  os_file = "./lib/openstudio/lib/worker_script.sh.template"
+  os_file = './lib/openstudio/lib/worker_script.sh.template'
   system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/worker_script.sh.template"
-  if File.exists?(os_file)
+  if File.exist?(os_file)
     system "ruby -i -pe 'puts \"# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files\" if $.==2' #{os_file}"
   end
 end
 
-desc "uninstall all openstudio-aws gems"
+desc 'uninstall all openstudio-aws gems'
 task :uninstall do
-  system "gem uninstall openstudio-aws -a"
+  system 'gem uninstall openstudio-aws -a'
 end
 
-desc "reinstall the gem (uninstall, build, and reinstall"
-task :reinstall => [:uninstall, :install]
-
+desc 'reinstall the gem (uninstall, build, and reinstall'
+task reinstall: [:uninstall, :install]

data/lib/openstudio/aws/aws.rb

@@ -7,6 +7,7 @@ module OpenStudio
     ]
 
     class Aws
+      # Deprecate OS_AWS object
       attr_reader :os_aws
       attr_reader :default_amis
 
@@ -26,7 +27,7 @@ module OpenStudio
           region: 'us-east-1',
           ssl_verify_peer: false,
           host: 'developer.nrel.gov',
-          url: '/downloads/buildings/openstudio/api'
+          url: '/downloads/buildings/openstudio/api',
         }
         options = defaults.merge(options)
 
@@ -65,7 +66,8 @@ module OpenStudio
         # puts "Final options are: #{options.inspect}"
 
         @os_aws = OpenStudioAwsWrapper.new(options)
-        @local_key_file_name = nil
+
+        @instances_json = nil
 
         # this will grab the default version of openstudio ami versions
         # get the arugments for the AMI lookup
@@ -78,44 +80,92 @@ module OpenStudio
         @default_amis = OpenStudioAmis.new(options[:ami_lookup_version], ami_options).get_amis
       end
 
+      # def load_data_from_json(json_filename)
+      #   @os_aws.load_data_from_json
+      #
+      #
+      # end
+
       # command line call to create a new instance.  This should be more tightly integrated with teh os-aws.rb gem
-      def create_server(instance_data = {}, server_json_filename = 'server_data.json', user_id = 'unknown_user')
-        defaults = { instance_type: 'm2.xlarge', image_id: @default_amis[:server] }
-        instance_data = defaults.merge(instance_data)
+      def create_server(options = {}, instances_json = 'server_data.json')
+        defaults = {
+          instance_type: 'm2.xlarge',
+          security_group: 'openstudio-worker-sg-v1',
+          image_id: @default_amis[:server],
+          user_id: 'unknown_user',
+
+          # optional -- will default later
+          ebs_volume_id: nil,
+          aws_key_pair_name: nil,
+          private_key_file_name: nil, # required if using an existing "aws_key_pair_name"
+        }
+        options = defaults.merge(options)
+
+        if options[:aws_key_pair_name]
+          fail 'Must pass in the private_key_file_name' unless options[:private_key_file_name]
+          fail "Private key was not found: #{options[:private_key_file_name]}" unless File.exist? options[:private_key_file_name]
+        end
+
+        @os_aws.create_or_retrieve_security_group options[:security_group]
+        @os_aws.create_or_retrieve_key_pair options[:aws_key_pair_name]
 
-        @os_aws.create_or_retrieve_security_group('openstudio-worker-sg-v1')
-        @os_aws.create_or_retrieve_key_pair
+        # If using an already_existing key_pair, then you must pass in the private key file name
+        if options[:aws_key_pair_name]
+          @os_aws.load_private_key options[:private_key_file_name]
+          @os_aws.private_key_file_name = options[:private_key_file_name]
+        else
+          # Save the private key if you did not pass in an already existing key_pair_name
+          @os_aws.save_private_key('ec2_server_key.pem')
+        end
 
-        @local_key_file_name = 'ec2_server_key.pem'
-        @os_aws.save_private_key(@local_key_file_name)
-        @os_aws.launch_server(instance_data[:image_id], instance_data[:instance_type], user_id: user_id)
+        server_options = { user_id: options[:user_id] }
+        # if instance_data[:ebs_volume_id]
+        #   server_options[:ebs_volume_id] = instance_data[:ebs_volume_id]
+        # end
 
-        puts @os_aws.server.to_os_hash.to_json
+        @os_aws.launch_server(options[:image_id], options[:instance_type], server_options)
 
-        File.open(server_json_filename, 'w') { |f| f << JSON.pretty_generate(@os_aws.server.to_os_hash) }
+        @instances_json = instances_json
+        File.open(@instances_json, 'w') { |f| f << JSON.pretty_generate(@os_aws.server.to_os_hash) }
 
         # Print out some debugging commands (probably work on mac/linux only)
         puts ''
         puts 'Server SSH Command:'
 
-        puts "ssh -i #{@local_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
+        puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
       end
 
-      def create_workers(number_of_instances, instance_data = {}, user_id = 'unknown_user')
-        defaults = { instance_type: 'm2.4xlarge' }
-        instance_data = defaults.merge(instance_data)
+      def create_workers(number_of_instances, options = {}, _user_id = 'unknown_user')
+        defaults = {
+          instance_type: 'm2.4xlarge',
+          security_group: 'openstudio-worker-sg-v1',
+          image_id: @default_amis[:server],
+          user_id: 'unknown_user',
 
-        if instance_data[:image_id].nil?
-          if instance_data[:instance_type] =~ /cc2|c3/
-            instance_data[:image_id] = @default_amis[:cc2worker]
+          # optional -- will default later
+          ebs_volume_id: nil,
+          aws_key_pair_name: nil,
+          private_key_file_name: nil, # required if using an existing "aws_key_pair_name"
+        }
+        options = defaults.merge(options)
+
+        # Get the right worker AMI ids based on the type of instance
+        if options[:image_id].nil?
+          if options[:instance_type] =~ /cc2|c3/
+            options[:image_id] = @default_amis[:cc2worker]
           else
-            instance_data[:image_id] = @default_amis[:worker]
+            options[:image_id] = @default_amis[:worker]
           end
         end
 
         fail "Can't create workers without a server instance running" if @os_aws.server.nil?
 
-        @os_aws.launch_workers(instance_data[:image_id], instance_data[:instance_type], number_of_instances, user_id: user_id)
+        worker_options = { user_id: options[:user_id] }
+        # if options[:ebs_volume_size]
+        #   worker_options[:ebs_volume_size] = options[:ebs_volume_size]
+        # end
+
+        @os_aws.launch_workers(options[:image_id], options[:instance_type], number_of_instances, worker_options)
 
         ## append the information to the server_data hash that already exists
         # @server_data[:instance_type] = instance_data[:instance_type]
@@ -129,10 +179,15 @@ module OpenStudio
         # File.open("worker_data.json", "w") { |f| f << JSON.pretty_generate(worker_data) }
         #
         ## Print out some debugging commands (probably work on mac/linux only)
+        # Add the worker data to the JSON
+        h = JSON.parse(File.read(@instances_json))
+        h[:workers] = @os_aws.to_os_worker_hash[:workers]
+        File.open(@instances_json, 'w') { |f| f << JSON.pretty_generate(h) }
+
         puts ''
         puts 'Worker SSH Command:'
         @os_aws.workers.each do |worker|
-          puts "ssh -i #{@local_key_file_name} ubuntu@#{worker.data[:dns]}"
+          puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{worker.data[:dns]}"
         end
 
         puts ''
@@ -141,8 +196,77 @@ module OpenStudio
         @os_aws.configure_server_and_workers
       end
 
-      def kill_instances
-        # Add this method to kill all the running instances
+      # openstudio_instance_type as symbol
+      def stop_instances(group_id, openstudio_instance_type)
+        instances = @os_aws.describe_running_instances(group_id, openstudio_instance_type.to_sym)
+        ids = instances.map { |k, _| k[:instance_id] }
+
+        resp = []
+        resp = @os_aws.stop_instances(ids).to_hash unless ids.empty?
+        resp
+      end
+
+      # @params(ids): array of instances
+      def terminate_instances(ids)
+        puts "Terminating the following instances #{ids}"
+        resp = []
+        resp = @os_aws.terminate_instances(ids).to_hash unless ids.empty?
+        resp
+      end
+
+      # Warning, this appears that it terminates all the instances of the openstudio_instance_type
+      def terminate_instances_by_group_id(group_id, openstudio_instance_type)
+        instances = @os_aws.describe_running_instances(group_id, openstudio_instance_type.to_sym)
+        ids = instances.map { |k, _| k[:instance_id] }
+
+        puts "Terminating the following instances #{ids}"
+        resp = []
+        resp = @os_aws.terminate_instances(ids).to_hash unless ids.empty?
+        resp
+      end
+
+      def load_instance_info_from_file(filename)
+        fail 'Could not find instance description JSON file' unless File.exist? filename
+
+        h = JSON.parse(File.read(filename), symbolize_names: true)
+        @os_aws.find_server(h)
+
+        # load the worker nodes someday
+      end
+
+      # Send a file to the server or worker nodes
+      def upload_file(server_or_workers, local_file, remote_file)
+        case server_or_workers
+          when :server
+            fail 'Server node is nil' unless @os_aws.server
+            return @os_aws.server.upload_file(local_file, remote_file)
+          when :worker
+            fail 'Worker list is empty' if @os_aws.workers.empty?
+            return @os_aws.workers.each { |w| w.upload_file(local_file, remote_file) }
+        end
+      end
+
+      def shell_command(server_or_workers, command, load_env = true)
+        case server_or_workers
+          when :server
+            fail 'Server node is nil' unless @os_aws.server
+            return @os_aws.server.shell_command(command, load_env)
+          when :worker
+            fail 'Worker list is empty' if @os_aws.workers.empty?
+            return @os_aws.workers.each { |w| w.shell_command(command, load_env) }
+        end
+      end
+
+      # Download remote files that are on the server or worker.  note that the worker at the moment
+      # will not work because it would simply overwrite the downloaded filas at this time.
+      def download_remote_file(server_or_workers, remote_file, local_file)
+        case server_or_workers
+          when :server
+            fail 'Server node is nil' unless @os_aws.server
+            return @os_aws.server.download_file(remote_file, local_file)
+          when :worker
+            fail 'Worker file download is not available'
+        end
       end
 
       private

data/lib/openstudio/aws/config.rb

@@ -7,6 +7,7 @@ module OpenStudio
       def initialize(yml_config_file = nil)
         @yml_config_file = yml_config_file
         @config = nil
+
         if @yml_config_file.nil?
           @yml_config_file = File.join(File.expand_path('~'), 'aws_config.yml')
           unless File.exist?(@yml_config_file)

data/lib/openstudio/aws/version.rb

@@ -1,5 +1,5 @@
 module OpenStudio
   module Aws
-    VERSION = '0.1.26'
+    VERSION = '0.2.0'
   end
 end

data/lib/openstudio/lib/ami_list.rb

@@ -98,7 +98,7 @@ class OpenStudioAmis
     if @options[:openstudio_server_version].to_sym == :default
       # just grab the most recent server
       # need to do a sort to get the most recent because we can't promise that they are in order
-      json[:openstudio].each do |k, v|
+      json[:openstudio].each do |_k, _v|
 
       end
       key, value = json[:openstudio_server].first