openstudio-aws 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d4c759c2c748eb2f907a53b3f87390246b188a9e
-  data.tar.gz: 614223c0a1d4195d0abfd0cfeb9c0e9e879640c0
+  metadata.gz: 883acd75472bdf8a4c5e0bbb2736c8b42360d498
+  data.tar.gz: 41253c1ce3d409e47a80785c2e8c8c51e154badb
 SHA512:
-  metadata.gz: 5f41d38ccbb517dbcf10f189dfcc01dd85f2dac7a4bcc5900d89e278834990041538d00247f6bd4a907b91f0af9dfefeb1d6498fe5b204ee822fecea22eecdaf
-  data.tar.gz: 3703c7acb4ded02951821d9dca099dc0616c6475d42a050dc45a52a211cbd5f6790bd696c1a8d9493026a58334ae66bb650fc40a1fdf620221ed254ba5a6f1ff
+  metadata.gz: 07039c60796d51fda00f35310a33f30078487cbfafdf1a4654ead5bc35d7ac7f788e3ef78aac505038524f0d019519a1a4ca1a4a166d56e0cf1d49f23e0c54b5
+  data.tar.gz: 0dbf6db0cb50dae855d71a7da96e600e872417462226131d5f69f62c36f833488ab482a564d8980bf1bc5b0791ca1f8dfd55532ba500316c4307d57ac3927ecb

data/Rakefile

@@ -1,6 +1,5 @@
 require "bundler" 
 Bundler.setup
-#Bundler.require
 
 require "rake"
 require "rspec/core/rake_task"
@@ -18,6 +17,7 @@ task :install => :build do
   system "gem install openstudio-aws-#{OpenStudio::Aws::VERSION}.gem --no-ri --no-rdoc"
 end
 
+desc "build and release the gem using rubygems and git tags"
 task :release => :build do
   system "git tag -a v#{OpenStudio::Aws::VERSION} -m 'Tagging #{OpenStudio::Aws::VERSION}'"
   system "git push --tags"
@@ -26,13 +26,13 @@ task :release => :build do
 end
 
 RSpec::Core::RakeTask.new("spec") do |spec|
-  spec.rspec_opts = %w(--format progress)
-  spec.pattern = "spec/**/config_spec.rb"
+  spec.rspec_opts = %w(--format progress --format CI::Reporter::RSpec)
+  spec.pattern = "spec/**/*_spec.rb"
 end
 
-RSpec::Core::RakeTask.new("spec:all") do |spec|
-  spec.rspec_opts = %w(--format progress)
-  spec.pattern = "spec/**/*_spec.rb"
+RSpec::Core::RakeTask.new("spec:api") do |spec|
+  spec.rspec_opts = %w(--format progress --format CI::Reporter::RSpec)
+  spec.pattern = "spec/**/*_spec_api.rb"
 end
 
 task :default => :spec
@@ -42,13 +42,14 @@ task :import_files do
   puts "Importing data from other repos until this repo is self contained"
   # Copy data from github openstudio source
   
-  os_file = "./lib/openstudio/lib/os-aws.rb"
-  system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/aws.rb.in"
-  if File.exists?(os_file)
-    system "ruby -i -pe 'puts \"# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files\" if $.==1' #{os_file}"
-    system "sed -i '' 's/\${CMAKE_VERSION_MAJOR}.\${CMAKE_VERSION_MINOR}.\${CMAKE_VERSION_PATCH}/#{OpenStudio::Aws::OPENSTUDIO_VERSION}/g' #{os_file}"
-  end      
-  
+  # this has been forked at the moment until openstudio team supports the new aws-sdk gem
+  #os_file = "./lib/openstudio/lib/os-aws.rb"
+  #system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/aws.rb.in"
+  #if File.exists?(os_file)
+  #  system "ruby -i -pe 'puts \"# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files\" if $.==1' #{os_file}"
+  #  system "sed -i '' 's/\${CMAKE_VERSION_MAJOR}.\${CMAKE_VERSION_MINOR}.\${CMAKE_VERSION_PATCH}/#{OpenStudio::Aws::OPENSTUDIO_VERSION}/g' #{os_file}"
+  #end      
+  #
   os_file = "./lib/openstudio/lib/mongoid.yml.template"
   system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/mongoid.yml.template"
 

data/lib/openstudio-aws.rb

@@ -1,20 +1,22 @@
 require 'json'
 require 'net/scp'
+require 'net/http'
 require 'yaml'
+require 'logger'
 
-# AWS SDK
+# AWS SDK CORE
 begin
-  puts `gem list`
-  gem "aws-sdk", ">= 1.30.1"
-  require 'aws-sdk'
+  require 'aws-sdk-core'
 rescue LoadError
-  puts "Failed to AWS-SDK gem"
-  puts "  gem install aws-sdk"
+  puts "Failed to load AWS-SDK-CORE gem"
+  puts "  try running: gem install aws-sdk-core"
   exit
 end
 
 require 'openstudio/aws/version'
 require 'openstudio/aws/aws'
 require 'openstudio/aws/config'
-require 'openstudio/aws/send_data'
+require 'openstudio/lib/openstudio_aws_instance'
+require 'openstudio/lib/openstudio_aws_wrapper'
+require 'openstudio/lib/ami_list'
 

data/lib/openstudio/aws/aws.rb

@@ -1,83 +1,96 @@
 # This class is a wrapper around the command line version that is in the OpenStudio repository.
-
-
 module OpenStudio
   module Aws
     class Aws
-      attr_reader :server_data
-      attr_reader :worker_data
+      attr_reader :os_aws
 
       def initialize()
         # read in the config.yml file to get the secret/private key
         @config = OpenStudio::Aws::Config.new()
-        @server_data = nil
-        @worker_data = nil
+
+        config = {:access_key_id => @config.access_key, :secret_access_key => @config.secret_key, :region => "us-east-1", :ssl_verify_peer => false}
+        @os_aws = OpenStudioAwsWrapper.new(config)
+        @local_key_file_name = nil
       end
 
       # command line call to create a new instance.  This should be more tightly integrated with teh os-aws.rb gem
       def create_server(instance_data = {})
-        # TODO: find a way to override the instance ids here in case we want to prototype
         defaults = {instance_type: "m2.xlarge"}
         instance_data = defaults.merge(instance_data)
 
-        # Since this is a command line call then make sure to escape the quotes in the JSON
-        instance_string = instance_data.to_json.gsub("\"", "\\\\\"")
+        # todo: grab the image id the instance_data if not set
+        
+        @os_aws.create_or_retrieve_security_group("openstudio-worker-sg-v1")
+        @os_aws.create_or_retrieve_key_pair
+
+        @local_key_file_name = "ec2_server_key.pem"
+        @os_aws.save_private_key(@local_key_file_name)
+        @os_aws.launch_server(instance_data[:image_id], instance_data[:instance_type])
 
+        puts @os_aws.server.to_os_hash.to_json
 
         # Call the openstudio script to start the ec2 instance 
-        start_string = "ruby #{os_aws_file_location} #{@config.access_key} #{@config.secret_key} us-east-1 EC2 launch_server \"#{instance_string}\""
-        puts "Server Command: #{start_string}"
-        server_data_str = `#{start_string}`
-        @server_data = JSON.parse(server_data_str, :symbolize_names => true)
+        #start_string = "ruby #{os_aws_file_location} #{@config.access_key} #{@config.secret_key} us-east-1 EC2 launch_server \"#{instance_string}\""
+        #puts "Server Command: #{start_string}"
+        #server_data_str = `#{start_string}`
+        #@server_data = JSON.parse(server_data_str, :symbolize_names => true)
 
         # Save pieces of the data for passing to the worker node
-        server_key_file = "ec2_server_key.pem"
-        File.open(server_key_file, "w") { |f| f << @server_data[:private_key] }
-        File.chmod(0600, server_key_file)
+
+        #File.open(server_key_file, "w") { |f| f << @server_data[:private_key] }
+        #File.chmod(0600, server_key_file)
 
         # Save off the server data to be loaded into the worker nodes.  The Private key needs to e read from a
         # file in the worker node, so save that name instead in the HASH along with a couple other changes
-        @server_data[:private_key] = server_key_file
-        if @server_data[:server]
-          @server_data[:server_id] = @server_data[:server][:id]
-          @server_data[:server_ip] = @server_data[:server][:ip]
-          @server_data[:server_dns] = @server_data[:server][:dns]
-        end
+        #@server_data[:private_key] = server_key_file
+        #if @server_data[:server]
+        #  @server_data[:server_id] = @server_data[:server][:id]
+        #  @server_data[:server_ip] = @server_data[:server][:ip]
+        #  @server_data[:server_dns] = @server_data[:server][:dns]
+        #end
 
-        File.open("server_data.json", "w") { |f| f << JSON.pretty_generate(@server_data) }
+        File.open("server_data.json", "w") { |f| f << JSON.pretty_generate(@os_aws.server.to_os_hash) }
 
         # Print out some debugging commands (probably work on mac/linux only)
         puts ""
         puts "Server SSH Command:"
 
-        puts "ssh -i #{server_key_file} ubuntu@#{@server_data[:server_dns]}"
+        puts "ssh -i #{@local_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
       end
 
       def create_workers(number_of_instances, instance_data = {})
         defaults = {instance_type: "m2.4xlarge"}
         instance_data = defaults.merge(instance_data)
 
-        raise "Can't create workers without a server instance running" if @server_data.nil?
-
-        # append the information to the server_data hash that already exists
-        @server_data[:instance_type] = instance_data[:instance_type]
-        @server_data[:num] = number_of_instances
-        server_string = @server_data.to_json.gsub("\"", "\\\\\"")
+        raise "Can't create workers without a server instance running" if @os_aws.server.nil?
 
-        start_string = "ruby #{os_aws_file_location} #{@config.access_key} #{@config.secret_key} us-east-1 EC2 launch_workers \"#{server_string}\""
-        puts "Worker Command: #{start_string}"
-        worker_data_string = `#{start_string}`
-        @worker_data = JSON.parse(worker_data_string, :symbolize_names => true)
-        File.open("worker_data.json", "w") { |f| f << JSON.pretty_generate(worker_data) }
-
-        # Print out some debugging commands (probably work on mac/linux only)
-        @worker_data[:workers].each do |worker|
+      
+        @os_aws.launch_workers(instance_data[:image_id], instance_data[:instance_type], number_of_instances)
+
+        ## append the information to the server_data hash that already exists
+        #@server_data[:instance_type] = instance_data[:instance_type]
+        #@server_data[:num] = number_of_instances
+        #server_string = @server_data.to_json.gsub("\"", "\\\\\"")
+        #
+        #start_string = "ruby #{os_aws_file_location} #{@config.access_key} #{@config.secret_key} us-east-1 EC2 launch_workers \"#{server_string}\""
+        #puts "Worker Command: #{start_string}"
+        #worker_data_string = `#{start_string}`
+        #@worker_data = JSON.parse(worker_data_string, :symbolize_names => true)
+        #File.open("worker_data.json", "w") { |f| f << JSON.pretty_generate(worker_data) }
+        #
+        ## Print out some debugging commands (probably work on mac/linux only)
+        @os_aws.workers.each do |worker|
           puts ""
           puts "Worker SSH Command:"
-          puts "ssh -i #{@server_data[:private_key]} ubuntu@#{worker[:dns]}"
+          puts "ssh -i #{@local_key_file_name} ubuntu@#{worker.data[:dns]}"
         end
+        
+        puts ""
+        puts "Waiting for server/worker configurations"
+        
+        @os_aws.configure_server_and_workers
       end
-      
+
       def kill_instances()
         # Add this method to kill all the running instances
       end

data/lib/openstudio/aws/config.rb

@@ -9,7 +9,6 @@ module OpenStudio
         @config = nil
         if @yml_config_file.nil?
           @yml_config_file = File.join(File.expand_path("~"), "aws_config.yml")
-          puts @yml_config_file
           if !File.exists?(@yml_config_file)
             write_config_file
             puts "No Config File in user home directory. A template has been added, please edit and save: #{@yml_config_file}"

data/lib/openstudio/aws/version.rb

@@ -1,6 +1,6 @@
 module OpenStudio
   module Aws
-    VERSION = "0.1.7"
-    OPENSTUDIO_VERSION = "1.1.4"
+    VERSION = "0.1.8"
+    OPENSTUDIO_VERSION = "1.2.1"
   end
 end

data/lib/openstudio/lib/ami_list.rb

@@ -0,0 +1,79 @@
+# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files
+######################################################################
+#  Copyright (c) 2008-2014, Alliance for Sustainable Energy.  
+#  All rights reserved.
+#  
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2.1 of the License, or (at your option) any later version.
+#  
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#  
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+######################################################################
+
+# Class for managing the AMI ids based on the openstudio version and the openstudio-server version
+
+class OpenStudioAmis
+
+  def initialize(version = 1, openstudio_version = 'default', openstudio_server_version = 'default')
+    @version = 1
+    @openstudio_version = openstudio_version
+    @openstudio_server_version = openstudio_server_version
+  end
+
+  def get_amis()
+    amis = nil
+    command = "get_ami_version_#{@version}"
+    if OpenStudioAmis.method_defined?(command)
+      amis = send(command)
+    else
+      raise "Unknown api version command #{command}"
+    end
+
+    raise "Could not find any amis for #{@version}" if amis.nil?
+
+    puts amis.inspect
+    amis
+  end
+
+  protected 
+  
+  def get_ami_version_1()
+    endpoint = '/downloads/buildings/openstudio/rsrc/amis.json'
+
+    json = retrieve_json(endpoint)
+    version = json.has_key?(@openstudio_version) ? @openstudio_version : 'default'
+
+    json[version]
+  end
+
+  def get_ami_version_2()
+    endpoint = '/downloads/buildings/openstudio/rsrc/amis_v2.json'
+
+    json = retrieve_json(endpoint)
+    version = json.has_key?(@openstudio_version) ? @openstudio_version : 'default'
+
+    #logic logic
+  end
+  
+  private
+  
+  def retrieve_json(endpoint)
+    result = nil
+    resp = Net::HTTP.get_response('developer.nrel.gov', endpoint)
+    if resp.code == '200'
+      result = JSON.parse(resp.body)
+    else
+      raise "#{resp.code} Unable to download AMI IDs"
+    end
+
+    result
+  end
+end

data/lib/openstudio/lib/openstudio_aws_instance.rb

@@ -0,0 +1,138 @@
+# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files
+######################################################################
+#  Copyright (c) 2008-2014, Alliance for Sustainable Energy.  
+#  All rights reserved.
+#  
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2.1 of the License, or (at your option) any later version.
+#  
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#  
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+######################################################################
+
+require 'securerandom'
+require_relative 'openstudio_aws_logger'
+require_relative 'openstudio_aws_methods'
+
+class OpenStudioAwsInstance
+  include Logging
+  include OpenStudioAwsMethods
+
+  attr_reader :openstudio_instance_type
+  attr_reader :data
+
+  def initialize(aws_session, openstudio_instance_type, key_pair_name, security_group_name, group_uuid, private_key)
+    @data = nil # stored information about the instance
+    @aws = aws_session
+    @openstudio_instance_type = openstudio_instance_type # :server, :worker
+    @key_pair_name = key_pair_name
+    @security_group_name = security_group_name
+    @group_uuid = group_uuid.to_s
+    @private_key = private_key
+  end
+
+
+  def launch_instance(image_id, instance_type, user_data)
+    logger.info("user_data #{user_data.inspect}")
+    result = @aws.run_instances(
+        {
+            :image_id => image_id,
+            :key_name => @key_pair_name,
+            :security_groups => [@security_group_name],
+            :user_data => Base64.encode64(user_data),
+            :instance_type => instance_type,
+            :min_count => 1,
+            :max_count => 1
+        }
+    )
+
+    # determine how many processors are suppose to be in this image (lookup for now?)
+    processors = find_processors(instance_type)
+
+    # only asked for 1 instance, so therefore it should be the first 
+    aws_instance = result.data.instances.first
+    @aws.create_tags(
+        {
+            :resources => [aws_instance.instance_id],
+            :tags => [
+                {:key => 'Name', :value => "OpenStudio-#{@openstudio_instance_type.capitalize}"}, # todo: abstract out the server and version
+                {:key => 'GroupUUID', :value => @group_uuid},
+                {:key => 'NumberOfProcessors', :value => processors.to_s},
+                {:key => 'Purpose', :value => "OpenStudio#{@openstudio_instance_type.capitalize}"}
+            ]
+        }
+    )
+
+    # get the instance information 
+    test_result = @aws.describe_instance_status({:instance_ids => [aws_instance.instance_id]}).data.instance_statuses.first
+    begin
+      Timeout::timeout(600) {# 10 minutes
+        while test_result.nil? || test_result.instance_state.name != "running"
+          # refresh the server instance information
+
+          sleep 5
+          test_result = @aws.describe_instance_status({:instance_ids => [aws_instance.instance_id]}).data.instance_statuses.first
+          logger.info "... waiting for instance to be running ..."
+        end
+      }
+    rescue TimeoutError
+      raise "Intance was unable to launch due to timeout #{aws_instance.instance_id}"
+    end
+
+    # now grab information about the instance 
+    # todo: check lengths on all of arrays
+    instance_data = @aws.describe_instances({:instance_ids => [aws_instance.instance_id]}).data.reservations.first.instances.first.to_hash
+    logger.info "instance description is: #{instance_data}"
+
+    @data = create_struct(instance_data, processors)
+  end
+  
+  # if the server already exists, then load the data about the server into the object
+  # instance_data is passed in and in the form of the instance data (as a hash) structured as the 
+  # result of the amazon describe instance
+  def load_instance_data(instance_data)
+    @data = create_struct(instance_data, find_processors(instance_data[:instance_type]))     
+  end
+
+  # Format of the OS JSON that is used for the command line based script
+  def to_os_hash
+    h = ""
+    if @openstudio_instance_type == :server
+      h = {
+          :timestamp => @group_uuid,
+          #:private_key => @private_key, # need to stop printing this out
+          :server => {
+              :id => @data.id,
+              :ip => 'http://' + @data.ip,
+              :dns => @data.dns,
+              :procs => @data.procs
+          }
+      }
+    else
+      raise "do not know how to convert :worker instance to_os_hash. Use the os_aws.to_worker_hash method"
+    end
+    
+
+    logger.info("server info #{h}")
+
+    h
+  end
+  
+  private
+
+  # store the data into a custom struct.  The instance is the full description.  The remaining fields are
+  # just easier accessors to the data in the raw request except for procs which is a custom request.
+  def create_struct(instance, procs)
+    instance_struct = Struct.new(:instance, :id, :ip, :dns, :procs)
+    return instance_struct.new(instance, instance[:instance_id], instance[:public_ip_address], instance[:public_dns_name], procs)
+  end
+
+end