openstudio-aws 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3f236cadc25bfed979fd052337a7cc9712186075
+  data.tar.gz: 79ece9147f4fc4408fb39cdba032c4ef5c4cfa0c
+SHA512:
+  metadata.gz: 644b891e5a9cdb2476df32d3b7847bad5f30aa0d9ba9adab099ab9c99df4b9be44f87d78f0ca5ef74653eb0f3a97070b5bf848485e9b617f733ae361439fd54d
+  data.tar.gz: bdc69e4e5293cedb1ee3454143a7f226197e5adac1810378426c90a7f7ea809ad1044d0e1aa99949d16057d12408227c294cb2b424ff9b0dabeff1cf729f0324

data/README.md

@@ -0,0 +1,18 @@
+OpenStudio AWS Gem
+==================
+
+OpenStudio AWS uses the OpenStudio AWS ruby class to launch a server and multiple workers for doing 
+OpenStudio/EnergyPlus Analyses using Amazon AWS/EC2
+
+Instructions
+------------
+
+Typically this gem is used in conjunction with other gems such as OpenStudio-Analysis.
+
+To use this make sure to have ruby 2.0 in your path.
+
+Development Notes
+-----------------
+
+There is an underlying attempt to merge several OpenStudio based gems into one and have a general 
+OpenStudio namespace for ruby classes.

data/Rakefile

@@ -0,0 +1,75 @@
+require "bundler" 
+Bundler.setup
+
+require "rake"
+require "rspec/core/rake_task"
+
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+require "openstudio/aws/version"
+
+task :gem => :build
+task :build do
+  system "gem build openstudio-aws.gemspec"
+end
+
+desc "build and install gem locally"
+task :install => :build do
+  system "gem install openstudio-aws-#{OpenStudio::Aws::VERSION}.gem --no-ri --no-rdoc"
+end
+
+task :release => :build do
+#  system "git tag -a v#{OpenStudio::Aws::VERSION} -m 'Tagging #{OpenStudio::Aws::VERSION}'"
+#  system "git push --tags"
+  system "gem push openstudio-aws-#{OpenStudio::Aws::VERSION}.gem"
+  system "rm openstudio-aws-#{OpenStudio::Aws::VERSION}.gem"
+end
+
+
+RSpec::Core::RakeTask.new("spec") do |spec|
+  spec.pattern = "spec/**/*_spec.rb"
+end
+
+RSpec::Core::RakeTask.new('spec:progress') do |spec|
+  spec.rspec_opts = %w(--format progress)
+  spec.pattern = "spec/**/*_spec.rb"
+end
+
+task :default => :spec
+
+desc "import files from other repos"
+task :import_files do
+  puts "Importing data from other repos until this repo is self contained"
+  # Copy data from github openstudio source
+  
+  os_file = "./lib/openstudio/lib/os-aws.rb"
+  system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/aws.rb.in"
+  if File.exists?(os_file)
+    system "ruby -i -pe 'puts \"# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files\" if $.==1' #{os_file}"
+    system "sed -i '' 's/\${CMAKE_VERSION_MAJOR}.\${CMAKE_VERSION_MINOR}.\${CMAKE_VERSION_PATCH}/#{OpenStudio::Aws::OPENSTUDIO_VERSION}/g' #{os_file}"
+  end      
+  
+  os_file = "./lib/openstudio/lib/mongoid.yml.template"
+  system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/mongoid.yml.template"
+
+  os_file = "./lib/openstudio/lib/server_script.sh"
+  system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/server_script.sh"
+  if File.exists?(os_file)
+    system "ruby -i -pe 'puts \"# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files\" if $.==2' #{os_file}"
+  end
+
+  os_file = "./lib/openstudio/lib/worker_script.sh.template"
+  system "curl -S -s -L -o #{os_file} https://raw.github.com/NREL/OpenStudio/develop/openstudiocore/ruby/cloud/worker_script.sh.template"
+  if File.exists?(os_file)
+    system "ruby -i -pe 'puts \"# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files\" if $.==2' #{os_file}"
+  end
+end
+
+desc "uninstall all openstudio-aws gems"
+task :uninstall do
+  
+  system "gem uninstall openstudio-aws -a"
+end
+
+desc "reinstall the gem (uninstall, build, and reinstall"
+task :reinstall => [:uninstall, :install]
+

data/lib/openstudio-aws.rb

@@ -0,0 +1,9 @@
+require 'json'
+require 'net/scp'
+require 'yaml'
+
+require 'openstudio/aws/version'
+require 'openstudio/aws/aws'
+require 'openstudio/aws/config'
+require 'openstudio/aws/send_data'
+

data/lib/openstudio/aws/aws.rb

@@ -0,0 +1,97 @@
+# This class is a wrapper around the command line version that is in the OpenStudio repository.
+
+
+module OpenStudio
+  module Aws
+    class Aws
+      attr_reader :server_data
+      attr_reader :worker_data
+
+      def initialize()
+        # read in the config.yml file to get the secret/private key
+        @config = OpenStudio::Aws::Config.new()
+        @server_data = nil
+        @worker_data = nil
+      end
+
+      # command line call to create a new instance.  This should be more tightly integrated with teh os-aws.rb gem
+      def create_server(instance_data = {})
+        # TODO: find a way to override the instance ids here in case we want to prototype
+        defaults = {instance_type: "m2.xlarge"}
+        instance_data = defaults.merge(instance_data)
+
+        # Since this is a command line call then make sure to escape the quotes in the JSON
+        instance_string = instance_data.to_json.gsub("\"", "\\\\\"")
+
+
+        # Call the openstudio script to start the ec2 instance 
+        start_string = "ruby #{os_aws_file_location} #{@config.access_key} #{@config.secret_key} us-east-1 EC2 launch_server \"#{instance_string}\""
+        puts "Server Command: #{start_string}"
+        server_data_str = `#{start_string}`
+        @server_data = JSON.parse(server_data_str, :symbolize_names => true)
+
+        # Save pieces of the data for passing to the worker node
+        server_key_file = "ec2_server_key.pem"
+        File.open(server_key_file, "w") { |f| f << @server_data[:private_key] }
+        File.chmod(0600, server_key_file)
+
+        # Save off the server data to be loaded into the worker nodes.  The Private key needs to e read from a
+        # file in the worker node, so save that name instead in the HASH along with a couple other changes
+        @server_data[:private_key] = server_key_file
+        if @server_data[:server]
+          @server_data[:server_id] = @server_data[:server][:id]
+          @server_data[:server_ip] = @server_data[:server][:ip]
+          @server_data[:server_dns] = @server_data[:server][:dns]
+        end
+
+        File.open("server_data.json", "w") { |f| f << JSON.pretty_generate(@server_data) }
+
+        # Print out some debugging commands (probably work on mac/linux only)
+        puts ""
+        puts "Server SSH Command:"
+
+        puts "ssh -i #{server_key_file} ubuntu@#{@server_data[:server_dns]}"
+      end
+
+      def create_workers(number_of_instances, instance_data = {})
+        defaults = {instance_type: "m2.4xlarge"}
+        instance_data = defaults.merge(instance_data)
+
+        raise "Can't create workers without a server instance running" if @server_data.nil?
+
+        # append the information to the server_data hash that already exists
+        @server_data[:instance_type] = instance_data[:instance_type]
+        @server_data[:num] = number_of_instances
+        server_string = @server_data.to_json.gsub("\"", "\\\\\"")
+
+        start_string = "ruby #{os_aws_file_location} #{@config.access_key} #{@config.secret_key} us-east-1 EC2 launch_workers \"#{server_string}\""
+        puts "Worker Command: #{start_string}"
+        worker_data_string = `#{start_string}`
+        @worker_data = JSON.parse(worker_data_string, :symbolize_names => true)
+        File.open("worker_data.json", "w") { |f| f << JSON.pretty_generate(worker_data) }
+
+        # Print out some debugging commands (probably work on mac/linux only)
+        @worker_data[:workers].each do |worker|
+          puts ""
+          puts "Worker SSH Command:"
+          puts "ssh -i #{@server_data[:private_key]} ubuntu@#{worker[:dns]}"
+        end
+      end
+      
+      def kill_instances()
+        # Add this method to kill all the running instances
+      end
+
+      private
+
+      def os_aws_file_location
+        # Get the location of the os-aws.rb file.  Use the relative path from where this file exists
+        os_aws_file = File.expand_path(File.join(File.dirname(__FILE__), "..", "lib", "os-aws.rb"))
+        raise "os_aws_file does not exist where it is expected: #{os_aws_file}" unless File.exists?(os_aws_file)
+
+        os_aws_file
+      end
+
+    end
+  end
+end

data/lib/openstudio/aws/config.rb

@@ -0,0 +1,39 @@
+module OpenStudio
+  module Aws
+    class Config
+      attr_accessor :access_key
+      attr_accessor :secret_key
+
+      def initialize(yml_config_file = nil)
+        @yml_config_file = yml_config_file
+        @config = nil
+        if @yml_config_file.nil?
+          @yml_config_file = File.join(File.expand_path("~"), "aws_config.yml")
+          puts @yml_config_file
+          if !File.exists?(@yml_config_file)
+            write_config_file
+            puts "No Config File in user home directory. A template has been added, please edit and save: #{@yml_config_file}"
+            exit 1
+          end
+        end
+
+        begin
+          @config = YAML.load(File.read(@yml_config_file))
+          @access_key = @config['access_key_id']
+          @secret_key = @config['secret_access_key']
+        rescue
+          raise "Couldn't read config file #{@yml_config_file}. Delete file then recreate by rerunning script"
+        end
+      end
+
+      private
+
+      def write_config_file
+        File.open(@yml_config_file, 'w') do |f|
+          f << "access_key_id: YOUR_ACCESS_KEY_ID\n"
+          f << "secret_access_key: YOUR_SECRET_ACCESS_KEY\n"
+        end
+      end
+    end
+  end
+end

data/lib/openstudio/aws/send_data.rb

@@ -0,0 +1,42 @@
+# This is taken out of the OpenStudio AWS config file to do some testing
+#def send_command(host, command, key)
+#  require 'net/http'
+#  require 'net/scp'
+#  require 'net/ssh'
+#
+#  retries = 0
+#  begin
+#    puts "connecting.."
+#    output = ''
+#    Net::SSH.start(host, 'ubuntu', :key_data => [key]) do |ssh|
+#      response = ssh.exec!(command)
+#      output += response if !response.nil?
+#    end
+#    return output
+#  rescue Net::SSH::HostKeyMismatch => e
+#    e.remember_host!
+#    # key mismatch, retry
+#    return if retries == 2
+#    retries += 1
+#    sleep 1
+#    retry
+#  rescue Net::SSH::AuthenticationFailed
+#    error(-1, "Incorrect private key")
+#  rescue SystemCallError, Timeout::Error => e
+#    # port 22 might not be available immediately after the instance finishes launching
+#    return if retries == 2
+#    retries += 1
+#    sleep 1
+#    retry
+#  rescue Exception => e
+#    puts e.message
+#    puts e.backtrace.inspect
+#  end
+#end
+#
+#server_json = JSON.parse(File.read("server_data.json"), :symbolize_names => true)
+#
+#b = Time.now
+#delta = b.to_f - a.to_f
+#puts "startup time is #{delta}"
+##puts send_command(server_json[:server_ip], 'nproc | tr -d "\n"', File.read("ec2_server_key.pem"))

data/lib/openstudio/aws/version.rb

@@ -0,0 +1,6 @@
+module OpenStudio
+  module Aws
+    VERSION = "0.1"
+    OPENSTUDIO_VERSION = "1.1.2"
+  end
+end

data/lib/openstudio/lib/mongoid.yml.template

@@ -0,0 +1,19 @@
+development:
+  sessions:
+    default:
+      database: os_dev
+      hosts:
+        - SERVER_IP:27017
+      options:
+        allow_dynamic_fields: true 
+        raise_not_found_error: false
+production:
+  sessions:
+    default:
+      database: os_prod
+      hosts:
+        - SERVER_IP:27017
+      options:
+        allow_dynamic_fields: true 
+        raise_not_found_error: false
+                

data/lib/openstudio/lib/os-aws.rb

@@ -0,0 +1,556 @@
+# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files
+######################################################################
+#  Copyright (c) 2008-2013, Alliance for Sustainable Energy.  
+#  All rights reserved.
+#  
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2.1 of the License, or (at your option) any later version.
+#  
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#  
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+######################################################################
+
+######################################################################
+# == Synopsis
+#
+#   Uses the aws-sdk gem to communicate with AWS
+#
+# == Usage
+#
+#  ruby aws.rb access_key secret_key us-east-1 EC2 launch_server "{\"instance_type\":\"t1.micro\"}"
+#
+#  ARGV[0] - Access Key
+#  ARGV[1] - Secret Key
+#  ARGV[2] - Region
+#  ARGV[3] - Service (e.g. "EC2" or "CloudWatch")
+#  ARGV[4] - Command (e.g. "launch_server")
+#  ARGV[5] - Optional json with parameters associated with command
+#
+######################################################################
+
+require 'aws-sdk'
+require 'json'
+require 'logger'
+require 'net/http'
+require 'net/scp'
+require 'net/ssh'
+require 'tempfile'
+require 'time'
+
+OPENSTUDIO_VERSION='1.1.2'
+
+def error(code, msg)
+  puts ({:error => {:code => code, :message => msg},
+         :arguments => ARGV[2..-1]}.to_json)
+  exit(1)
+end
+
+if ARGV.length < 5
+  error(-1, 'Invalid number of args')
+end
+
+if ARGV[0].empty? || ARGV[1].empty?
+  error(401, 'Missing authentication arguments')
+end
+
+if ARGV[2].empty?
+  error(-1, 'Missing region argument')
+end
+
+if ARGV[3].empty?
+  error(-1, 'Missing service argument')
+end
+
+if ARGV[4].empty?
+  error(-1, 'Missing command argument')
+end
+
+AWS.config(
+    :access_key_id => ARGV[0],
+    :secret_access_key => ARGV[1],
+    :region => ARGV[2],
+    :ssl_verify_peer => false
+)
+
+if ARGV[3] == 'EC2'
+  @aws = AWS::EC2.new
+elsif ARGV[3] == 'CloudWatch'
+  @aws = AWS::CloudWatch.new
+else
+  error(-1, "Unrecognized AWS service: #{ARGV[3]}")
+end
+
+if ARGV.length == 6
+  @params = JSON.parse(ARGV[5])
+end
+
+resp = Net::HTTP.get_response('developer.nrel.gov','/downloads/buildings/openstudio/rsrc/amis.json')
+if resp.code == '200'
+  result = JSON.parse(resp.body)
+  version = result.has_key?(OPENSTUDIO_VERSION) ? OPENSTUDIO_VERSION : 'default'
+
+  @server_image_id = result[version]['server']
+  if ARGV.length >= 6 && @params['instance_type'] == 'cc2.8xlarge'
+    @worker_image_id = result[version]['cc2worker']
+  else
+    @worker_image_id = result[version]['worker']
+  end
+else
+  error(resp.code, 'Unable to download AMI IDs')
+end
+
+def create_struct(instance, procs)
+  instance_struct = Struct.new(:instance, :id, :ip, :dns, :procs)
+  return instance_struct.new(instance, instance.instance_id, instance.ip_address, instance.dns_name, procs)
+end
+
+def find_processors(instance)
+  processors = 1
+  case instance
+    when 'cc2.8xlarge'
+      processors = 16
+    when 'c1.xlarge'
+      processors = 8
+    when 'm2.4xlarge'
+      processors = 8
+    when 'm2.2xlarge'
+      processors = 4
+    when 'm2.xlarge'
+      processors = 2
+    when 'm1.xlarge'
+      processors = 4
+    when 'm1.large'
+      processors = 2
+    when 'm3.xlarge'
+      processors = 2
+    when 'm3.2xlarge'
+      processors = 4
+  end
+
+  return processors
+end
+
+def launch_server
+  user_data = File.read(File.expand_path(File.dirname(__FILE__))+'/server_script.sh')
+  @logger.info("server user_data #{user_data.inspect}")
+  @server = @aws.instances.create(:image_id => @server_image_id,
+                                  :key_pair => @key_pair,
+                                  :security_groups => @group,
+                                  :user_data => user_data,
+                                  :instance_type => @server_instance_type)
+  @server.add_tag('Name', :value => "OpenStudio-Server V#{OPENSTUDIO_VERSION}")
+  sleep 5 while @server.status == :pending
+  if @server.status != :running
+    error(-1, "Server status: #{@server.status}")
+  end
+
+  processors = find_processors(@server_instance_type)
+  #processors = send_command(@server.ip_address, 'nproc | tr -d "\n"')
+  #processors = 0 if processors.nil?  # sometimes this returns nothing, so put in a default
+  @server = create_struct(@server, processors)
+end
+
+def launch_workers(num, server_ip)
+  user_data = File.read(File.expand_path(File.dirname(__FILE__))+'/worker_script.sh.template')
+  user_data.gsub!(/SERVER_IP/, server_ip)
+  user_data.gsub!(/SERVER_HOSTNAME/, 'master')
+  user_data.gsub!(/SERVER_ALIAS/, '')
+  @logger.info("worker user_data #{user_data.inspect}")
+  instances = []
+  num.times do
+    worker = @aws.instances.create(:image_id => @worker_image_id,
+                                   :key_pair => @key_pair,
+                                   :security_groups => @group,
+                                   :user_data => user_data,
+                                   :instance_type => @worker_instance_type)
+    worker.add_tag('Name', :value => "OpenStudio-Worker V#{OPENSTUDIO_VERSION}")
+    instances.push(worker)
+  end
+  sleep 5 while instances.any? { |instance| instance.status == :pending }
+  if instances.any? { |instance| instance.status != :running }
+    error(-1, "Worker status: Not running")
+  end
+
+  # todo: fix this - sometimes returns nil
+  processors = find_processors(@worker_instance_type)
+  #processors = send_command(instances[0].ip_address, 'nproc | tr -d "\n"')
+  #processors = 0 if processors.nil?  # sometimes this returns nothing, so put in a default
+  instances.each { |instance| @workers.push(create_struct(instance, processors)) }
+end
+
+def upload_file(host, local_path, remote_path)
+  retries = 0
+  begin
+    Net::SCP.start(host, 'ubuntu', :key_data => [@private_key]) do |scp|
+      scp.upload! local_path, remote_path
+    end
+  rescue SystemCallError, Timeout::Error => e
+    # port 22 might not be available immediately after the instance finishes launching
+    return if retries == 5
+    retries += 1
+    sleep 1
+    retry
+  rescue
+    # Unknown upload error, retry
+    return if retries == 5
+    retries += 1
+    sleep 1
+    retry
+  end
+end
+
+
+def send_command(host, command)
+  #retries = 0
+  begin
+    output = ''
+    Net::SSH.start(host, 'ubuntu', :key_data => [@private_key]) do |ssh|
+      response = ssh.exec!(command)
+      output += response if !response.nil?
+    end
+    return output
+  rescue Net::SSH::HostKeyMismatch => e
+    e.remember_host!
+    # key mismatch, retry
+    #return if retries == 5
+    #retries += 1
+    sleep 1
+    retry
+  rescue Net::SSH::AuthenticationFailed
+    error(-1, "Incorrect private key")
+  rescue SystemCallError, Timeout::Error => e
+    # port 22 might not be available immediately after the instance finishes launching
+    #return if retries == 5
+    #retries += 1
+    sleep 1
+    retry
+  rescue Exception => e
+    puts e.message
+    puts e.backtrace.inspect
+  end
+end
+
+#======================= send command ======================#
+# Send a command through SSH Shell to an instance.
+# Need to pass instance object and the command as a string.
+def shell_command(host, command)
+  begin
+    @logger.info("ssh_command #{command}")
+    Net::SSH.start(host, 'ubuntu', :key_data => [@private_key]) do |ssh|
+      channel = ssh.open_channel do |ch|
+        ch.exec "#{command}" do |ch, success|
+          raise "could not execute #{command}" unless success
+
+          # "on_data" is called when the process writes something to stdout
+          ch.on_data do |c, data|
+            #$stdout.print data
+            @logger.info("#{data.inspect}")
+          end
+
+          # "on_extended_data" is called when the process writes something to stderr
+          ch.on_extended_data do |c, type, data|
+            #$stderr.print data
+            @logger.info("#{data.inspect}")
+          end
+        end
+      end
+    end
+  rescue Net::SSH::HostKeyMismatch => e
+    e.remember_host!
+    @logger.info("key mismatch, retry")
+    sleep 1
+    retry
+  rescue SystemCallError, Timeout::Error => e
+    # port 22 might not be available immediately after the instance finishes launching
+    sleep 1
+    @logger.info("Not Yet")
+    retry
+  end
+end
+
+def wait_command(host, command)
+  begin
+    flag = 0
+    while flag == 0 do
+      @logger.info("wait_command #{command}")
+      Net::SSH.start(host, 'ubuntu', :key_data => [@private_key]) do |ssh|
+        channel = ssh.open_channel do |ch|
+          ch.exec "#{command}" do |ch, success|
+            raise "could not execute #{command}" unless success
+
+            # "on_data" is called when the process writes something to stdout
+            ch.on_data do |c, data|
+              @logger.info("#{data.inspect}")
+              if data.chomp == "true"
+                @logger.info("wait_command #{command} is true")
+                flag = 1
+              else
+                sleep 5
+              end
+            end
+
+            # "on_extended_data" is called when the process writes something to stderr
+            ch.on_extended_data do |c, type, data|
+              @logger.info("#{data.inspect}")
+              if data == "true"
+                @logger.info("wait_command #{command} is true")
+                flag = 1
+              else
+                sleep 5
+              end
+            end
+          end
+        end
+      end
+    end
+  rescue Net::SSH::HostKeyMismatch => e
+    e.remember_host!
+    @logger.info("key mismatch, retry")
+    sleep 1
+    retry
+  rescue SystemCallError, Timeout::Error => e
+    # port 22 might not be available immediately after the instance finishes launching
+    sleep 1
+    @logger.info("Not Yet")
+    retry
+  end
+end
+
+def download_file(host, remote_path, local_path)
+  retries = 0
+  begin
+    Net::SCP.start(host, 'ubuntu', :key_data => [@private_key]) do |scp|
+      scp.download! remote_path, local_path
+    end
+  rescue SystemCallError, Timeout::Error => e
+    # port 22 might not be available immediately after the instance finishes launching
+    return if retries == 5
+    retries += 1
+    sleep 1
+    retry
+  rescue
+    return if retries == 5
+    retries += 1
+    sleep 1
+    retry
+  end
+end
+
+begin
+  @logger = Logger.new("aws.log")
+  @logger.info("initialized")
+  case ARGV[4]
+    when 'describe_availability_zones'
+      resp = @aws.client.describe_availability_zones
+      puts resp.data.to_json
+      @logger.info("availability_zones #{resp.data.to_json}")
+    when 'total_instances'
+      resp = @aws.client.describe_instance_status
+      puts ({:total_instances => resp.data[:instance_status_set].length,
+             :region => ARGV[2]}.to_json)
+    when 'instance_status'
+      resp = nil
+      if ARGV.length < 6
+        resp = @aws.client.describe_instance_status
+      else
+        resp = @aws.client.describe_instance_status({:instance_ids => [@params['instance_id']]})
+      end
+      output = Hash.new
+      resp.data[:instance_status_set].each { |instance|
+        output[instance[:instance_id]] = instance[:instance_state][:name]
+      }
+      puts output.to_json
+    when 'launch_server'
+      if ARGV.length < 6
+        error(-1, 'Invalid number of args')
+      end
+
+      @timestamp = Time.now.to_i
+
+      # find if an existing openstudio-server-vX security group exists and use that
+      @group = @aws.security_groups.filter('group-name', 'openstudio-server-sg-v1').first
+      if @group.nil?
+        @group = @aws.security_groups.create('openstudio-server-sg-v1')
+        @group.allow_ping() # allow ping
+        @group.authorize_ingress(:tcp, 1..65535) # all traffic
+      end
+      @logger.info("server_group #{@group}")
+      @server_instance_type = @params['instance_type']
+
+      @key_pair = @aws.key_pairs.create("key-pair-#{@timestamp}")
+      @private_key = @key_pair.private_key
+
+      launch_server()
+
+      puts ({:timestamp => @timestamp,
+             :private_key => @private_key,
+             :server => {
+                 :id => @server.id,
+                 :ip => 'http://' + @server.ip,
+                 :dns => @server.dns,
+                 :procs => @server.procs
+             }}.to_json)
+      @logger.info("server info #{({:timestamp => @timestamp, :private_key => @private_key, :server => {:id => @server.id, :ip => @server.ip, :dns => @server.dns, :procs => @server.procs}}.to_json)}")
+    when 'launch_workers'
+      if ARGV.length < 6
+        error(-1, 'Invalid number of args')
+      end
+      if @params['num'] < 1
+        error(-1, 'Invalid number of worker nodes, must be greater than 0')
+      end
+
+      @workers = []
+      @timestamp = @params['timestamp']
+
+      # find if an existing openstudio-server-vX security group exists and use that
+      @group = @aws.security_groups.filter('group-name', 'openstudio-worker-sg-v1').first
+      if @group.nil?
+        @group = @aws.security_groups.create('openstudio-worker-sg-v1')
+        @group.allow_ping() # allow ping
+        @group.authorize_ingress(:tcp, 1..65535) # all traffic
+      end
+      @logger.info("worker_group #{@group}")
+      @key_pair = @aws.key_pairs.filter('key-name', "key-pair-#{@timestamp}").first
+      @private_key = File.read(@params['private_key'])
+      @worker_instance_type = @params['instance_type']
+      @server = @aws.instances[@params['server_id']]
+      error(-1, 'Server node does not exist') unless @server.exists?
+      @server = create_struct(@server, @params['server_procs'])
+
+      launch_workers(@params['num'], @server.ip)
+      #@workers.push(create_struct(@aws.instances['i-xxxxxxxx'], 1))
+      #processors = send_command(@workers[0].ip, 'nproc | tr -d "\n"')
+      #@workers[0].procs = processors
+
+      #wait for user_data to complete execution
+      @logger.info("server user_data")
+      wait_command(@server.ip, '[ -e /home/ubuntu/user_data_done ] && echo "true"')
+      @logger.info("worker user_data")
+      @workers.each { |worker| wait_command(worker.ip, '[ -e /home/ubuntu/user_data_done ] && echo "true"') }
+      #wait_command(@workers.first.ip, "[ -e /home/ubuntu/user_data_done ] && echo 'true'") 
+
+
+      ips = "master|#{@server.ip}|#{@server.dns}|#{@server.procs}|ubuntu|ubuntu\n"
+      @workers.each { |worker| ips << "worker|#{worker.ip}|#{worker.dns}|#{worker.procs}|ubuntu|ubuntu|true\n" }
+      file = Tempfile.new('ip_addresses')
+      file.write(ips)
+      file.close
+      upload_file(@server.ip, file.path, 'ip_addresses')
+      file.unlink
+      @logger.info("ips #{ips}")
+      shell_command(@server.ip, 'chmod 664 /home/ubuntu/ip_addresses')
+      shell_command(@server.ip, '~/setup-ssh-keys.sh')
+      shell_command(@server.ip, '~/setup-ssh-worker-nodes.sh ip_addresses')
+
+      mongoid = File.read(File.expand_path(File.dirname(__FILE__))+'/mongoid.yml.template')
+      mongoid.gsub!(/SERVER_IP/, @server.ip)
+      file = Tempfile.new('mongoid.yml')
+      file.write(mongoid)
+      file.close
+      upload_file(@server.ip, file.path, '/mnt/openstudio/rails-models/mongoid.yml')
+      @workers.each { |worker| upload_file(worker.ip, file.path, '/mnt/openstudio/rails-models/mongoid.yml') }
+      file.unlink
+
+      # Does this command crash it?
+      shell_command(@server.ip, 'chmod 664 /mnt/openstudio/rails-models/mongoid.yml')
+      @workers.each { |worker| shell_command(worker.ip, 'chmod 664 /mnt/openstudio/rails-models/mongoid.yml') }
+
+      worker_json = []
+      @workers.each { |worker|
+        worker_json.push({
+                             :id => worker.id,
+                             :ip => 'http://' + worker.ip,
+                             :dns => worker.dns,
+                             :procs => worker.procs
+                         })
+      }
+      puts ({:workers => worker_json}.to_json)
+      @logger.info("workers #{({:workers => worker_json}.to_json)}")
+    when 'terminate_session'
+      if ARGV.length < 6
+        error(-1, 'Invalid number of args')
+      end
+      instances = []
+
+      server = @aws.instances[@params['server_id']]
+      error(-1, "Server node #{@params['server_id']} does not exist") unless server.exists?
+
+      #@timestamp = @aws.client.describe_instances({:instance_ids=>[@params['server_id']]}).data[:instance_index][@params['server_id']][:key_name][9,10]
+      @timestamp = server.key_name[9, 10]
+
+      instances.push(server)
+      @params['worker_ids'].each { |worker_id|
+        worker = @aws.instances[worker_id]
+        error(-1, "Worker node #{worker_id} does not exist") unless worker.exists?
+        instances.push(worker)
+      }
+
+      instances.each { |instance|
+        instance.terminate
+      }
+      sleep 5 while instances.any? { |instance| instance.status != :terminated }
+
+      # When session is fully terminated, then delete the key pair
+      #@aws.client.delete_security_group({:group_name=>'openstudio-server-sg-v1'}"})
+      #@aws.client.delete_security_group({:group_name=>'openstudio-worker-sg-v1'}"})
+      @aws.client.delete_key_pair({:key_name => "key-pair-#{@timestamp}"})
+
+    when 'termination_status'
+      if ARGV.length < 6
+        error(-1, 'Invalid number of args')
+      end
+      notTerminated = 0
+
+      server = @aws.instances[@params['server_id']]
+      notTerminated += 1 if (server.exists? && server.status != :terminated)
+
+      @params['worker_ids'].each { |worker_id|
+        worker = @aws.instances[worker_id]
+        notTerminated += 1 if (worker.exists? && worker.status != :terminated)
+      }
+
+      puts ({:all_instances_terminated => (notTerminated == 0)}.to_json)
+
+    when 'session_uptime'
+      if ARGV.length < 6
+        error(-1, 'Invalid number of args')
+      end
+      server_id = @params['server_id']
+      #No need to call AWS, but we can
+      #minutes = (Time.now.to_i - @aws.client.describe_instances({:instance_ids=>[server_id]}).data[:instance_index][server_id][:launch_time].to_i)/60
+      minutes = (Time.now.to_i - @params['timestamp'].to_i)/60
+      puts ({:session_uptime => minutes}.to_json)
+
+    when 'estimated_charges'
+      endTime = Time.now.utc
+      startTime = endTime - 86400
+      resp = @aws.client.get_metric_statistics({:dimensions => [{:name => 'ServiceName', :value => 'AmazonEC2'}, {:name => 'Currency', :value => 'USD'}], :metric_name => 'EstimatedCharges', :namespace => 'AWS/Billing', :start_time => startTime.iso8601, :end_time => endTime.iso8601, :period => 300, :statistics => ['Maximum']})
+      error(-1, 'No Billing Data') if resp.data[:datapoints].length == 0
+      datapoints = resp.data[:datapoints]
+      datapoints.sort! { |a, b| a[:timestamp] <=> b[:timestamp] }
+      puts ({:estimated_charges => datapoints[-1][:maximum],
+             :timestamp => datapoints[-1][:timestamp].to_i}.to_json)
+
+    else
+      error(-1, "Unknown command: #{ARGV[4]} (#{ARGV[3]})")
+  end
+    #puts \"Status: #{resp.http_response.status}\"
+rescue SystemExit => e
+rescue Exception => e
+  if e.message == 'getaddrinfo: No such host is known. '
+    error(503, 'Offline')
+  elsif defined? e.http_response
+    error(e.http_response.status, e.code)
+  else
+    error(-1, "#{e}: #{e.backtrace}")
+  end
+
+end

data/lib/openstudio/lib/server_script.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files
+
+# Change Host File Entries
+ENTRY="localhost localhost master"
+FILE=/etc/hosts
+if grep -q "$ENTRY" $FILE; then
+  echo "entry already exists"
+else
+  sudo sh -c "echo $ENTRY >> /etc/hosts"
+fi
+
+# copy all the setup scripts to the appropriate home directory
+cp /data/launch-instance/setup* /home/ubuntu/
+chmod 775 /home/ubuntu/setup*
+chown ubuntu:ubuntu /home/ubuntu/setup*
+
+# Set permissions on rails apps folders
+sudo chmod 777 /var/www/rails/openstudio/public
+
+# Force the generation of OpenStudio on the EBS mount and copy worker files
+sudo rm -rf /mnt/openstudio
+sudo mkdir -p /mnt/openstudio
+sudo chmod -R 777 /mnt/openstudio
+cp -rf /data/worker-nodes/* /mnt/openstudio/
+
+# Unzip the worker files (including Mongoid models) and set permissions one last time
+# Note that the 777 is redundant but needed until we actually deploy ACL
+cd /mnt/openstudio/rails-models && unzip -o rails-models.zip
+sudo chmod -R 777 /mnt/openstudio
+
+# Force the generation of MongoDB dbpath on the EBS mount
+sudo rm -rf /mnt/mongodb/data
+sudo mkdir -p /mnt/mongodb/data
+sudo chown mongodb:nogroup /mnt/mongodb/data
+sudo service mongodb restart
+sudo service delayed_job restart
+
+# Add database indexes after it gets mounted for the first time
+cd /var/www/rails/openstudio
+sudo rake db:mongoid:create_indexes
+
+#file flag the user_data has completed
+cat /dev/null > /home/ubuntu/user_data_done
+
+

data/lib/openstudio/lib/worker_script.sh.template

@@ -0,0 +1,39 @@
+#!/bin/sh
+# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files
+
+# AWS Worker Bootstrap File
+
+# Change Host File Entries
+ENTRY="SERVER_IP SERVER_HOSTNAME SERVER_ALIAS"
+FILE=/etc/hosts
+if grep -q "$ENTRY" $FILE; then
+  echo "entry already exists"
+else
+  sudo sh -c "echo $ENTRY >> /etc/hosts"
+fi
+
+# copy all the setup scripts to the appropriate home directory
+cp /data/launch-instance/setup* /home/ubuntu/
+chmod 775 /home/ubuntu/setup*
+chown ubuntu:ubuntu /home/ubuntu/setup*
+
+# Force the generation of OpenStudio on the EBS mount and copy worker files
+sudo rm -rf /mnt/openstudio
+sudo mkdir -p /mnt/openstudio
+sudo chmod -R 777 /mnt/openstudio
+cp -rf /data/worker-nodes/* /mnt/openstudio/
+
+# Unzip the worker files (including Mongoid models) and set permissions one last time
+# Note that the 777 is redundant but needed until we actually deploy ACL
+cd /mnt/openstudio/rails-models && unzip -o rails-models.zip
+sudo chmod -R 777 /mnt/openstudio
+
+#turn off hyperthreading
+for cpunum in $(
+  cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | 
+  cut -s -d, -f2- | tr ',' '\n' | sort -un); do
+    echo 0 > /sys/devices/system/cpu/cpu$cpunum/online
+done
+
+#file flag the user_data has completed
+cat /dev/null > /home/ubuntu/user_data_done

data/spec/openstudio-aws/aws_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe OpenStudio::Aws do
+  context "create a new instance" do
+    before(:all) do
+      @aws = OpenStudio::Aws::Aws.new
+    end
+
+    it "should create a new instance" do
+      @aws.should_not be_nil
+    end
+
+    it "should ask the user to create a new config file" do
+    end
+
+    it "should create a server" do
+      # use the default instance type
+      #options = {instance_type: "t1.micro" }
+      options = {instance_type: "m1.small" }
+      @aws.create_server(options)
+      @aws.server_data[:server_dns].should_not be_nil 
+    end
+    
+    it "should create a 1 worker" do
+      #options = {instance_type: "t1.micro" }
+      options = {instance_type: "m1.small" }
+      #server_json[:instance_type] = "m2.4xlarge"
+      #server_json[:instance_type] = "m2.2xlarge"
+      #server_json[:instance_type] = "t1.micro"
+      @aws.create_workers(1, options)
+      
+      @aws.worker_data[:workers].size.should eq(1)
+      @aws.worker_data[:workers][0][:dns].should_not be_nil
+    end
+  end
+
+  context "workers before server" do
+    before(:all) do
+      @aws = OpenStudio::Aws::Aws.new
+    end
+
+    it "should not create any workers" do
+      
+      expect {@aws.create_workers(5)}.to raise_error
+    end
+
+  end
+  
+  context "larger cluster" do
+    
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'rspec'
+require 'openstudio-aws'
+
+
+

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: openstudio-aws
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Nicholas Long
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.26.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.26.0
+description: Custom classes for configuring clusters for OpenStudio & EnergyPlus analyses
+email:
+- Nicholas.Long@nrel.gov
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/openstudio/aws/aws.rb
+- lib/openstudio/aws/config.rb
+- lib/openstudio/aws/send_data.rb
+- lib/openstudio/aws/version.rb
+- lib/openstudio/lib/mongoid.yml.template
+- lib/openstudio/lib/os-aws.rb
+- lib/openstudio/lib/server_script.sh
+- lib/openstudio/lib/worker_script.sh.template
+- lib/openstudio-aws.rb
+- README.md
+- Rakefile
+- spec/openstudio-aws/aws_spec.rb
+- spec/spec_helper.rb
+homepage: http://openstudio.nrel.gov
+licenses:
+- LGPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.2
+signing_key: 
+specification_version: 4
+summary: Start AWS EC2 instances for running distributed OpenStudio-based analyses
+test_files:
+- spec/openstudio-aws/aws_spec.rb
+- spec/spec_helper.rb