openstudio-aws 0.2.5 → 0.3.0

data/lib/openstudio/lib/{server_script.sh → server_script.sh.template}

@@ -1,25 +1,48 @@
 #!/bin/bash
 
 # AWS Server Bootstrap File
-# This script is used to configure the AWS boxes
+# This script is used to configure the AWS boxes for OpenStudio-server
 
-# Unlock the user account
-sudo passwd -u ubuntu
-
-# Change Host File Entries
-ENTRY="localhost localhost master"
+ENTRY="127.0.0.1 SERVER_HOSTNAME"
 FILE=/etc/hosts
 if grep -q "$ENTRY" $FILE; then
-  echo "entry already exists"
+  echo "Hosts '$SERVER_HOSTNAME' entry already exists"
+else
+  sudo sh -c "echo $ENTRY >> /etc/hosts"
+fi
+
+WORKER_PRIVATE_KEY="WORKER_PRIVATE_KEY_TEMPLATE"
+WORKER_PUBLIC_KEY="WORKER_PUBLIC_KEY_TEMPLATE"
+
+USER=ubuntu
+
+mkdir -p /home/$USER/.ssh && chmod 700 /home/$USER/.ssh
+echo -e $WORKER_PRIVATE_KEY > /home/$USER/.ssh/id_rsa
+echo -e $WORKER_PUBLIC_KEY > /home/$USER/.ssh/id_rsa.pub
+chmod 600 /home/$USER/.ssh/id_rsa
+chmod 644 /home/$USER/.ssh/id_rsa.pub
+chown ubuntu.ubuntu /home/$USER/.ssh/id_rsa
+chown ubuntu.ubuntu /home/$USER/.ssh/id_rsa.pub
+
+# Only allow localhost (and localhosts IP) to connect ot itself with this key. The IP is used for R cluster
+ENTRY="from=\"localhost,127.0.0.1\" $WORKER_PUBLIC_KEY"
+FILE=/home/$USER/.ssh/authorized_keys
+if grep -q "$ENTRY" $FILE; then
+  echo "Key already exists in file"
 else
-  sh -c "echo $ENTRY >> /etc/hosts"
+  echo $ENTRY >> $FILE
 fi
+chmod 644 /home/$USER/.ssh/authorized_keys
+
+# SSH config
+echo -e "Host *\n\tStrictHostKeyChecking no" > /home/$USER/.ssh/config
+chmod 644 /home/$USER/.ssh/config && chown ubuntu.ubuntu /home/$USER/.ssh/config
 
-# copy all the setup scripts to the appropriate home directory
-# the scripts are called by the AWS connector for passwordless ssh config
-cp /data/launch-instance/setup* /home/ubuntu/
-chmod 775 /home/ubuntu/setup*
-chown ubuntu:ubuntu /home/ubuntu/setup*
+# ec2 instance information
+curl -o /usr/local/bin/ec2-metadata http://s3.amazonaws.com/ec2metadata/ec2-metadata
+chmod 775 /usr/local/bin/ec2-metadata
+mkdir -p /etc/openstudio-server
+ec2-metadata -a -i -t -h -o -z -p -v > /etc/openstudio-server/instance.yml
 
 # stop the various services that use mongo
 service delayed_job stop
@@ -67,20 +90,21 @@ su - ubuntu -c 'cd /var/www/rails/openstudio && bundle exec rake db:mongoid:crea
 
 ## Worker Data Configuration -- On Vagrant this is a separate file
 
-# Force the generation of various directories that are in the EBS mnt
+rm -f /tmp/snow.log
+
+# Force the generation of various directories that are in the EBS mount
 rm -rf /mnt/openstudio
 mkdir -p /mnt/openstudio
 chown -R ubuntu:www-data /mnt/openstudio
 chmod -R 775 /mnt/openstudio
 
 # save application files into the right directory
-cp -rf /data/worker-nodes/* /mnt/openstudio/
+rsync -a --chown ubuntu:www-data --exclude Gemfile.lock /data/worker-nodes/ /mnt/openstudio/
 
 # install workflow dependencies
-su - ubuntu -c 'cd /mnt/openstudio && rm -f'
-rm -f /mnt/openstudio/Gemfile.lock
-cd /mnt/openstudio && bundle update
-su - ubuntu -c 'cd /mnt/openstudio && bundle update'
+# note: vagrant/ubuntu are now members of rbenv but it still doesn't work to not call sudo on bundle
+# Give full path to bundle because sudoers path is not available with cloud-init root
+cd /mnt/openstudio && /opt/rbenv/shims/bundle
 
 # copy over the models needed for mongo
 cd /mnt/openstudio/rails-models && unzip -o rails-models.zip -d models
@@ -100,5 +124,3 @@ service delayed_job start
 
 #file flag the user_data has completed
 cat /dev/null > /home/ubuntu/user_data_done
-
-

data/lib/openstudio/lib/worker_script.sh.template

@@ -3,39 +3,55 @@
 # AWS Worker Bootstrap File
 # File used to configure worker nodes on AWS
 
-# Unlock the user account
-sudo passwd -u ubuntu
-
 # Change Host File Entries
-ENTRY="SERVER_IP SERVER_HOSTNAME SERVER_ALIAS"
+ENTRY="SERVER_IP SERVER_HOSTNAME"
 FILE=/etc/hosts
 if grep -q "$ENTRY" $FILE; then
   echo "entry already exists"
 else
-  sh -c "echo $ENTRY >> /etc/hosts"
+  sh -c "echo $ENTRY >> $FILE"
 fi
 
-# copy all the setup scripts to the appropriate home directory
-cp /data/launch-instance/setup* /home/ubuntu/
-chmod 775 /home/ubuntu/setup*
-chown ubuntu:ubuntu /home/ubuntu/setup*
+WORKER_PUBLIC_KEY="WORKER_PUBLIC_KEY_TEMPLATE"
+
+# add the public key to authorized keys
+USER=ubuntu
+
+mkdir -p /home/$USER/.ssh && chmod 700 /home/$USER/.ssh
+
+# Only access from the server node
+ENTRY="from=\"SERVER_HOSTNAME\" $WORKER_PUBLIC_KEY"
+FILE=/home/$USER/.ssh/authorized_keys
+if grep -q "$ENTRY" $FILE; then
+  echo "Key already exists in file"
+else
+  echo $ENTRY >> $FILE
+fi
+chmod 644 /home/$USER/.ssh/authorized_keys
+
+# ec2 instance information
+curl -o /usr/local/bin/ec2-metadata http://s3.amazonaws.com/ec2metadata/ec2-metadata
+chmod 775 /usr/local/bin/ec2-metadata
+mkdir -p /etc/openstudio-server
+ec2-metadata -a -i -t -h -o -z -p -v > /etc/openstudio-server/instance.yml
 
 ## Worker Data Configuration -- On Vagrant this is a separate file
 
-# Force the generation of various directories that are in the EBS mnt
+rm -f /tmp/snow.log
+
+# Force the generation of various directories that are in the EBS mount
 rm -rf /mnt/openstudio
 mkdir -p /mnt/openstudio
 chown -R ubuntu:www-data /mnt/openstudio
 chmod -R 775 /mnt/openstudio
 
 # save application files into the right directory
-cp -rf /data/worker-nodes/* /mnt/openstudio/
+rsync -a --chown ubuntu:www-data --exclude Gemfile.lock /data/worker-nodes/ /mnt/openstudio/
 
 # install workflow dependencies
-su - ubuntu -c 'cd /mnt/openstudio && rm -f'
-rm -f /mnt/openstudio/Gemfile.lock
-cd /mnt/openstudio && bundle update
-su - ubuntu -c 'cd /mnt/openstudio && bundle update'
+# note: vagrant/ubuntu are now members of rbenv but it still doesn't work to not call sudo on bundle
+# Give full path to bundle because sudoers path is not available with cloud-init root
+cd /mnt/openstudio && /opt/rbenv/shims/bundle
 
 # copy over the models needed for mongo
 cd /mnt/openstudio/rails-models && unzip -o rails-models.zip -d models

data/spec/aws_instances/aws_spec_api.rb

@@ -1,5 +1,8 @@
 require 'spec_helper'
 
+SERVER_AMI = 'ami-e0b38888'
+WORKER_AMI = 'ami-a8bc87c0'
+
 describe OpenStudio::Aws::Aws do
   context 'create a new instance' do
     before(:all) do
@@ -7,24 +10,29 @@ describe OpenStudio::Aws::Aws do
       @aws = OpenStudio::Aws::Aws.new
 
       @group_id = nil
+
+      FileUtils.rm_f 'ec2_server_key.pem' if File.exist? 'ec2_server_key.pem'
     end
 
     it 'should create a new instance' do
-      expect(@aws).to be_nil
+      expect(@aws.os_aws.server).to be_nil
+      expect(@aws.os_aws.workers).to be_empty
     end
 
     it 'should create a server' do
-      options = { instance_type: 'm3.medium', image_id: 'ami-3c0fbf54' }
+      options = { instance_type: 'm3.medium', image_id: SERVER_AMI }
 
-      test_pem_file = 'ec2_server_key.pem'
-      FileUtils.rm_f test_pem_file if File.exist? test_pem_file
+      FileUtils.rm_f 'ec2_server_key.pem' if File.exist? 'ec2_server_key.pem'
       FileUtils.rm_f 'server_data.json' if File.exist? 'server_data.json'
 
       @aws.create_server(options)
 
       expect(File.exist?('ec2_server_key.pem')).to be true
       expect(File.exist?('server_data.json')).to be true
+      expect(File.exist?('ec2_worker_key.pem')).to be true
+      expect(File.exist?('ec2_worker_key.pub')).to be true
       expect(@aws.os_aws.server).not_to be_nil
+      expect(@aws.os_aws.server.data.availability_zone).to match /us-east-../
 
       h = @aws.os_aws.server.to_os_hash
       expect(h[:group_id]).to be_a String
@@ -35,16 +43,17 @@ describe OpenStudio::Aws::Aws do
     end
 
     it 'should create a 1 worker' do
-      options = { instance_type: 'm3.medium', image_id: 'ami-040ebe6c' }
+      options = { instance_type: 'm3.medium', image_id: WORKER_AMI }
 
       @aws.create_workers(1, options)
 
       expect(@aws.os_aws.workers.size).to eq 1
       expect(@aws.os_aws.workers[0].data[:dns]).not_to be_nil
+      expect(@aws.os_aws.server.data.availability_zone).to eq @aws.os_aws.workers[0].data.availability_zone
     end
 
     it 'should be able to connect a worker to an existing server' do
-      options = { instance_type: 'm3.medium', image_id: 'ami-3c0fbf54' }
+      options = { instance_type: 'm3.medium', image_id: WORKER_AMI }
 
       # will require a new @aws class--but attached to same group_uuid
       @config = OpenStudio::Aws::Config.new
@@ -58,10 +67,6 @@ describe OpenStudio::Aws::Aws do
     it 'should kill running instances' do
       # how to test this?
     end
-
-    after :all do
-      FileUtils.rm_f 'ec2_server_key.pem' if File.exist? 'ec2_server_key.pem'
-    end
   end
 
   context 'upload data to a server' do
@@ -69,7 +74,7 @@ describe OpenStudio::Aws::Aws do
       config = OpenStudio::Aws::Config.new
       aws = OpenStudio::Aws::Aws.new
 
-      options = { instance_type: 'm3.medium', image_id: 'ami-3c0fbf54' }
+      options = { instance_type: 'm3.medium', image_id: SERVER_AMI }
 
       aws.create_server(options)
     end
@@ -132,7 +137,7 @@ describe OpenStudio::Aws::Aws do
       begin
         options = {
           instance_type: 'm3.medium',
-          image_id: 'ami-3c0fbf54',
+          image_id: SERVER_AMI,
           tags: [
             'ci_tests=true',
             'nothing=else',
@@ -179,7 +184,7 @@ describe OpenStudio::Aws::Aws do
         begin
           options = {
             instance_type: 'm3.medium',
-            image_id: 'ami-3c0fbf54', # server AMI
+            image_id: SERVER_AMI,
             tags: [
               'ci_tests=true',
               'ServerOnly=true'
@@ -207,19 +212,4 @@ describe OpenStudio::Aws::Aws do
       end
     end
   end
-
-  # context 'create ebs storage' do
-  #   before(:all) do
-  #     @aws = OpenStudio::Aws::Aws.new
-  #   end
-  #
-  #   it 'should create an EBS volume' do
-  #     options = {instance_type: 'm1.small', image_id: 'ami-29faca40', ebs_volume_size: 128}
-  #     @aws.create_volume()
-  #     @aws.create_server(options)
-  #     expect(@aws.os_aws.server).not_to be_nil
-  #   end
-  #
-  # end
-
 end

data/spec/openstudio-aws/ami_list_spec.rb

@@ -13,6 +13,7 @@ describe OpenStudioAmis do
       # From OpenStudio Version 1.5.0 expect cc2workers to be the same as the worker
       expect(amis[:cc2worker]).not_to be_nil
       expect(amis[:worker]).to eq amis[:cc2worker]
+      puts amis.inspect
     end
 
     it 'should return specific amis if passed a version' do
@@ -33,8 +34,14 @@ describe OpenStudioAmis do
   end
 
   context 'version 2' do
-    it 'should default the server version' do
+    it 'should return 1.8.0 versions correctly' do
+      a = OpenStudioAmis.new(2, openstudio_server_version: '1.8.0')
 
+      amis = a.get_amis
+
+      expect(amis[:server]).to eq('ami-3c0fbf54')
+      expect(amis[:worker]).to eq('ami-040ebe6c')
+      expect(amis[:cc2worker]).to eq('ami-040ebe6c')
     end
   end
 end

data/spec/openstudio-aws/aws_instance_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe OpenStudioAwsInstance do
   context 'processors' do
     before :all do
-      @aws_instance = OpenStudioAwsInstance.new(nil, nil, nil, nil, nil, nil, nil)
+      @aws_instance = OpenStudioAwsInstance.new(nil, nil, nil, nil, nil, nil, nil, nil)
     end
 
     it 'should default to 1 with a warning' do

data/spec/openstudio-aws/aws_spec.rb

@@ -21,6 +21,17 @@ describe OpenStudio::Aws::Aws do
 
       expect(aws.default_amis).not_to be_nil
     end
+
+    it 'should find the cc2 instances' do
+      options = {
+          ami_lookup_version: 2,
+          openstudio_server_version: '1.5.0'
+      }
+      aws = OpenStudio::Aws::Aws.new(options)
+
+      expect(aws.determine_image_type('cc2.2xlarge')).to eq 'ami-e582948c'
+      expect(aws.determine_image_type('c3.2xlarge')).to eq 'ami-918294f8'
+    end
   end
 
   context 'should error' do

data/spec/openstudio-aws/aws_wrapper_spec.rb

@@ -32,13 +32,12 @@ describe OpenStudioAwsWrapper do
       it 'should describe the zones' do
         resp = @aws.os_aws.describe_availability_zones
         expect(resp).not_to be_nil
-        expect(resp[:availability_zone_info].length).to eq(4)
+        expect(resp[:availability_zone_info].length).to eq(5)
 
-        expect(resp[:availability_zone_info].inspect).to eq("[{:zone_name=>\"us-east-1a\", :state=>\"available\", :region_name=>\"us-east-1\", :messages=>[]}, {:zone_name=>\"us-east-1b\", :state=>\"available\", :region_name=>\"us-east-1\", :messages=>[]}, {:zone_name=>\"us-east-1c\", :state=>\"available\", :region_name=>\"us-east-1\", :messages=>[]}, {:zone_name=>\"us-east-1d\", :state=>\"available\", :region_name=>\"us-east-1\", :messages=>[]}]")
+        expect(resp[:availability_zone_info].inspect).to eq("[{:zone_name=>\"us-east-1a\", :state=>\"available\", :region_name=>\"us-east-1\", :messages=>[]}, {:zone_name=>\"us-east-1b\", :state=>\"available\", :region_name=>\"us-east-1\", :messages=>[]}, {:zone_name=>\"us-east-1c\", :state=>\"available\", :region_name=>\"us-east-1\", :messages=>[]}, {:zone_name=>\"us-east-1d\", :state=>\"available\", :region_name=>\"us-east-1\", :messages=>[]}, {:zone_name=>\"us-east-1e\", :state=>\"available\", :region_name=>\"us-east-1\", :messages=>[]}]")
 
         resp = @aws.os_aws.describe_availability_zones_json
         expect(resp).not_to be_nil
-
       end
 
       it 'should list number of instances' do
@@ -97,4 +96,16 @@ describe OpenStudioAwsWrapper do
     end
   end
 
+  context 'security groups' do
+    before :all do
+      @aws = OpenStudio::Aws::Aws.new
+    end
+
+    it 'should create security group' do
+      sg = @aws.os_aws.create_or_retrieve_default_security_group
+      pp sg
+      expect(sg).to_not be nil
+    end
+  end
+
 end

data/spec/reports/SPEC-OpenStudio-Aws-Aws-ami-lists.0.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="OpenStudio::Aws::Aws ami lists" tests="2" time="0.069183" failures="0" errors="0" skipped="0" timestamp="2014-10-13T10:47:29-06:00">
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.027307">
+  </testcase>
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.041671">
+  </testcase>
+  <system-out>
+  </system-out>
+  <system-err>
+  </system-err>
+</testsuite>

data/spec/reports/SPEC-OpenStudio-Aws-Aws-ami-lists.1.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="OpenStudio::Aws::Aws ami lists" tests="2" time="0.044669" failures="0" errors="0" skipped="0" timestamp="2014-10-13T10:47:55-06:00">
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.016562">
+  </testcase>
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.027876">
+  </testcase>
+  <system-out>
+  </system-out>
+  <system-err>
+  </system-err>
+</testsuite>

data/spec/reports/SPEC-OpenStudio-Aws-Aws-ami-lists.10.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="OpenStudio::Aws::Aws ami lists" tests="3" time="0.052765" failures="1" errors="0" skipped="0" timestamp="2014-10-13T11:20:06-06:00">
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.016159">
+  </testcase>
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.027489">
+  </testcase>
+  <testcase name="OpenStudio::Aws::Aws ami lists should find the cc2 instances" time="0.008848">
+    <failure type="RSpec::Expectations::ExpectationNotMetError" message="...">
+
+expected: "ami-1284172938"
+     got: "ami-918294f8"
+
+(compared using ==)
+ (RSpec::Expectations::ExpectationNotMetError)
+
+expected: "ami-1284172938"
+     got: "ami-918294f8"
+
+(compared using ==)
+
+     ./spec/openstudio-aws/aws_spec.rb:33:in `block (3 levels) in &lt;top (required)&gt;'    </failure>
+  </testcase>
+  <system-out>
+  </system-out>
+  <system-err>
+  </system-err>
+</testsuite>

data/spec/reports/SPEC-OpenStudio-Aws-Aws-ami-lists.11.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="OpenStudio::Aws::Aws ami lists" tests="3" time="0.054594" failures="0" errors="0" skipped="0" timestamp="2014-10-13T11:20:32-06:00">
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.015859">
+  </testcase>
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.028364">
+  </testcase>
+  <testcase name="OpenStudio::Aws::Aws ami lists should find the cc2 instances" time="0.010115">
+  </testcase>
+  <system-out>
+  </system-out>
+  <system-err>
+  </system-err>
+</testsuite>

data/spec/reports/SPEC-OpenStudio-Aws-Aws-ami-lists.12.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="OpenStudio::Aws::Aws ami lists" tests="3" time="0.062048" failures="0" errors="0" skipped="0" timestamp="2014-10-13T11:46:41-06:00">
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.019056">
+  </testcase>
+  <testcase name="OpenStudio::Aws::Aws ami lists should allow a different region" time="0.030449">
+  </testcase>
+  <testcase name="OpenStudio::Aws::Aws ami lists should find the cc2 instances" time="0.01224">
+  </testcase>
+  <system-out>
+  </system-out>
+  <system-err>
+  </system-err>
+</testsuite>