openstudio-aws 0.1.22 → 0.1.23
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/openstudio-aws.rb +2 -3
- data/lib/openstudio/aws/aws.rb +44 -46
- data/lib/openstudio/aws/config.rb +3 -3
- data/lib/openstudio/aws/version.rb +1 -1
- data/lib/openstudio/lib/ami_list.rb +24 -26
- data/lib/openstudio/lib/mongoid.yml.template +0 -1
- data/lib/openstudio/lib/openstudio_aws_instance.rb +45 -48
- data/lib/openstudio/lib/openstudio_aws_logger.rb +5 -5
- data/lib/openstudio/lib/openstudio_aws_methods.rb +52 -54
- data/lib/openstudio/lib/openstudio_aws_wrapper.rb +89 -94
- data/lib/openstudio/lib/os-aws.rb +46 -46
- data/lib/openstudio/lib/server_script.sh +45 -16
- data/lib/openstudio/lib/worker_script.sh.template +15 -6
- data/spec/aws_instances/aws_spec_api.rb +15 -15
- data/spec/openstudio-aws/ami_list_spec.rb +14 -14
- data/spec/openstudio-aws/aws_methods_spec.rb +7 -9
- data/spec/openstudio-aws/aws_spec.rb +43 -47
- data/spec/openstudio-aws/aws_wrapper_spec.rb +39 -41
- data/spec/openstudio-aws/config_spec.rb +2 -2
- data/spec/reports/SPEC-OpenStudio-Aws-Aws-ami-lists.7.xml +11 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Aws-ami-lists.8.xml +11 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Aws-proxy-configuration.36.xml +13 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Aws-proxy-configuration.37.xml +13 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Aws-should-error.4.xml +9 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Aws-should-error.5.xml +9 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Aws-version-testing.4.xml +9 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Aws-version-testing.5.xml +9 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Aws.42.xml +7 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Aws.43.xml +7 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Config-create-a-new-config.61.xml +9 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Config-create-a-new-config.62.xml +9 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Config.61.xml +7 -0
- data/spec/reports/SPEC-OpenStudio-Aws-Config.62.xml +7 -0
- data/spec/reports/SPEC-OpenStudioAmis-version-1.63.xml +13 -0
- data/spec/reports/SPEC-OpenStudioAmis-version-1.64.xml +13 -0
- data/spec/reports/SPEC-OpenStudioAmis-version-2.61.xml +9 -0
- data/spec/reports/SPEC-OpenStudioAmis-version-2.62.xml +9 -0
- data/spec/reports/SPEC-OpenStudioAmis.63.xml +7 -0
- data/spec/reports/SPEC-OpenStudioAmis.64.xml +7 -0
- data/spec/reports/SPEC-OpenStudioAwsMethods-processors.63.xml +11 -0
- data/spec/reports/SPEC-OpenStudioAwsMethods-processors.64.xml +11 -0
- data/spec/reports/SPEC-OpenStudioAwsMethods.63.xml +7 -0
- data/spec/reports/SPEC-OpenStudioAwsMethods.64.xml +7 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-ami-list.1.xml +9 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-ami-list.2.xml +9 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-availability.62.xml +11 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-availability.63.xml +11 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-create-new-ami-json.53.xml +11 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-create-new-ami-json.54.xml +11 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-new-instance.62.xml +9 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-new-instance.63.xml +9 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-version-1.11.xml +87 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-version-1.12.xml +87 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-version-2.11.xml +821 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session-version-2.12.xml +821 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session.62.xml +7 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-authenticated-session.63.xml +7 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-unauthenticated-session.63.xml +9 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper-unauthenticated-session.64.xml +9 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper.63.xml +7 -0
- data/spec/reports/SPEC-OpenStudioAwsWrapper.64.xml +7 -0
- data/spec/spec_helper.rb +1 -4
- metadata +85 -1
@@ -1,18 +1,18 @@
|
|
1
1
|
# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files
|
2
2
|
######################################################################
|
3
|
-
# Copyright (c) 2008-2014, Alliance for Sustainable Energy.
|
3
|
+
# Copyright (c) 2008-2014, Alliance for Sustainable Energy.
|
4
4
|
# All rights reserved.
|
5
|
-
#
|
5
|
+
#
|
6
6
|
# This library is free software; you can redistribute it and/or
|
7
7
|
# modify it under the terms of the GNU Lesser General Public
|
8
8
|
# License as published by the Free Software Foundation; either
|
9
9
|
# version 2.1 of the License, or (at your option) any later version.
|
10
|
-
#
|
10
|
+
#
|
11
11
|
# This library is distributed in the hope that it will be useful,
|
12
12
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14
14
|
# Lesser General Public License for more details.
|
15
|
-
#
|
15
|
+
#
|
16
16
|
# You should have received a copy of the GNU Lesser General Public
|
17
17
|
# License along with this library; if not, write to the Free Software
|
18
18
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
@@ -53,7 +53,7 @@ module Logging
|
|
53
53
|
end
|
54
54
|
|
55
55
|
def configure_logger_for(classname)
|
56
|
-
logger = Logger.new(File.expand_path(
|
56
|
+
logger = Logger.new(File.expand_path('~/.aws.log'))
|
57
57
|
logger.progname = classname
|
58
58
|
logger
|
59
59
|
end
|
@@ -1,18 +1,18 @@
|
|
1
1
|
# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files
|
2
2
|
######################################################################
|
3
|
-
# Copyright (c) 2008-2014, Alliance for Sustainable Energy.
|
3
|
+
# Copyright (c) 2008-2014, Alliance for Sustainable Energy.
|
4
4
|
# All rights reserved.
|
5
|
-
#
|
5
|
+
#
|
6
6
|
# This library is free software; you can redistribute it and/or
|
7
7
|
# modify it under the terms of the GNU Lesser General Public
|
8
8
|
# License as published by the Free Software Foundation; either
|
9
9
|
# version 2.1 of the License, or (at your option) any later version.
|
10
|
-
#
|
10
|
+
#
|
11
11
|
# This library is distributed in the hope that it will be useful,
|
12
12
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14
14
|
# Lesser General Public License for more details.
|
15
|
-
#
|
15
|
+
#
|
16
16
|
# You should have received a copy of the GNU Lesser General Public
|
17
17
|
# License along with this library; if not, write to the Free Software
|
18
18
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
@@ -26,63 +26,62 @@
|
|
26
26
|
# == Usage
|
27
27
|
#
|
28
28
|
# Inside the class in which this file is included make sure to implement the following
|
29
|
-
#
|
29
|
+
#
|
30
30
|
# Member Variables:
|
31
31
|
# private_key : the in memory private key
|
32
32
|
# logger : logger class in which to write log messages
|
33
|
-
# @proxy : proxy setting if available
|
33
|
+
# @proxy : proxy setting if available
|
34
34
|
######################################################################
|
35
35
|
|
36
36
|
module OpenStudioAwsMethods
|
37
37
|
# This list of processors can be pulled out of the ../../doc/amazon_prices.xlsx file
|
38
38
|
def find_processors(instance)
|
39
39
|
lookup = {
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
40
|
+
'm3.medium' => 1,
|
41
|
+
'm3.large' => 2,
|
42
|
+
'm3.xlarge' => 4,
|
43
|
+
'm3.2xlarge' => 8,
|
44
|
+
'c3.large' => 2,
|
45
|
+
'c3.xlarge' => 4,
|
46
|
+
'c3.2xlarge' => 8,
|
47
|
+
'c3.4xlarge' => 8,
|
48
|
+
'c3.8xlarge' => 16,
|
49
|
+
'r3.large' => 2,
|
50
|
+
'r3.xlarge' => 4,
|
51
|
+
'r3.2xlarge' => 8,
|
52
|
+
'r3.4xlarge' => 16,
|
53
|
+
'r3.8xlarge' => 32,
|
54
|
+
't1.micro' => 1,
|
55
|
+
'm1.small' => 1,
|
56
56
|
}
|
57
57
|
|
58
58
|
processors = 1
|
59
|
-
if lookup.
|
59
|
+
if lookup.key?(instance)
|
60
60
|
processors = lookup[instance]
|
61
61
|
else
|
62
|
-
#logger.warn "Could not find the number of processors for instance type of #{instance}" if logger
|
62
|
+
# logger.warn "Could not find the number of processors for instance type of #{instance}" if logger
|
63
63
|
end
|
64
64
|
|
65
65
|
processors
|
66
66
|
end
|
67
67
|
|
68
|
-
def get_proxy
|
68
|
+
def get_proxy
|
69
69
|
proxy = nil
|
70
70
|
if @proxy
|
71
71
|
if @proxy[:username]
|
72
|
-
proxy = Net::SSH::Proxy::HTTP.new(@proxy[:host], @proxy[:port], :
|
72
|
+
proxy = Net::SSH::Proxy::HTTP.new(@proxy[:host], @proxy[:port], user: @proxy[:username], password: proxy[:password])
|
73
73
|
else
|
74
74
|
proxy = Net::SSH::Proxy::HTTP.new(@proxy[:host], @proxy[:port])
|
75
75
|
end
|
76
76
|
end
|
77
|
-
|
77
|
+
|
78
78
|
proxy
|
79
79
|
end
|
80
|
-
|
81
|
-
|
80
|
+
|
82
81
|
def upload_file(host, local_path, remote_path)
|
83
82
|
retries = 0
|
84
83
|
begin
|
85
|
-
Net::SCP.start(host, 'ubuntu', :
|
84
|
+
Net::SCP.start(host, 'ubuntu', proxy: get_proxy, key_data: [@private_key]) do |scp|
|
86
85
|
scp.upload! local_path, remote_path
|
87
86
|
end
|
88
87
|
rescue SystemCallError, Timeout::Error => e
|
@@ -100,29 +99,28 @@ module OpenStudioAwsMethods
|
|
100
99
|
end
|
101
100
|
end
|
102
101
|
|
103
|
-
|
104
102
|
def send_command(host, command)
|
105
|
-
#retries = 0
|
103
|
+
# retries = 0
|
106
104
|
begin
|
107
105
|
output = ''
|
108
|
-
Net::SSH.start(host, 'ubuntu', :
|
106
|
+
Net::SSH.start(host, 'ubuntu', proxy: get_proxy, key_data: [@private_key]) do |ssh|
|
109
107
|
response = ssh.exec!(command)
|
110
|
-
output += response
|
108
|
+
output += response unless response.nil?
|
111
109
|
end
|
112
110
|
return output
|
113
111
|
rescue Net::SSH::HostKeyMismatch => e
|
114
112
|
e.remember_host!
|
115
113
|
# key mismatch, retry
|
116
|
-
#return if retries == 5
|
117
|
-
#retries += 1
|
114
|
+
# return if retries == 5
|
115
|
+
# retries += 1
|
118
116
|
sleep 1
|
119
117
|
retry
|
120
118
|
rescue Net::SSH::AuthenticationFailed
|
121
|
-
error(-1,
|
119
|
+
error(-1, 'Incorrect private key')
|
122
120
|
rescue SystemCallError, Timeout::Error => e
|
123
121
|
# port 22 might not be available immediately after the instance finishes launching
|
124
|
-
#return if retries == 5
|
125
|
-
#retries += 1
|
122
|
+
# return if retries == 5
|
123
|
+
# retries += 1
|
126
124
|
sleep 1
|
127
125
|
retry
|
128
126
|
rescue Exception => e
|
@@ -131,26 +129,26 @@ module OpenStudioAwsMethods
|
|
131
129
|
end
|
132
130
|
end
|
133
131
|
|
134
|
-
|
132
|
+
# ======================= send command ======================#
|
135
133
|
# Send a command through SSH Shell to an instance.
|
136
134
|
# Need to pass instance object and the command as a string.
|
137
135
|
def shell_command(host, command)
|
138
136
|
begin
|
139
137
|
@logger.info("ssh_command #{command}")
|
140
|
-
Net::SSH.start(host, 'ubuntu', :
|
138
|
+
Net::SSH.start(host, 'ubuntu', proxy: get_proxy, key_data: [@private_key]) do |ssh|
|
141
139
|
channel = ssh.open_channel do |ch|
|
142
140
|
ch.exec "#{command}" do |ch, success|
|
143
|
-
|
141
|
+
fail "could not execute #{command}" unless success
|
144
142
|
|
145
143
|
# "on_data" is called when the process writes something to stdout
|
146
144
|
ch.on_data do |c, data|
|
147
|
-
|
145
|
+
# $stdout.print data
|
148
146
|
@logger.info("#{data.inspect}")
|
149
147
|
end
|
150
148
|
|
151
149
|
# "on_extended_data" is called when the process writes something to stderr
|
152
150
|
ch.on_extended_data do |c, type, data|
|
153
|
-
|
151
|
+
# $stderr.print data
|
154
152
|
@logger.info("#{data.inspect}")
|
155
153
|
end
|
156
154
|
end
|
@@ -158,13 +156,13 @@ module OpenStudioAwsMethods
|
|
158
156
|
end
|
159
157
|
rescue Net::SSH::HostKeyMismatch => e
|
160
158
|
e.remember_host!
|
161
|
-
@logger.info(
|
159
|
+
@logger.info('key mismatch, retry')
|
162
160
|
sleep 1
|
163
161
|
retry
|
164
162
|
rescue SystemCallError, Timeout::Error => e
|
165
163
|
# port 22 might not be available immediately after the instance finishes launching
|
166
164
|
sleep 1
|
167
|
-
@logger.info(
|
165
|
+
@logger.info('Not Yet')
|
168
166
|
retry
|
169
167
|
end
|
170
168
|
end
|
@@ -172,17 +170,17 @@ module OpenStudioAwsMethods
|
|
172
170
|
def wait_command(host, command)
|
173
171
|
begin
|
174
172
|
flag = 0
|
175
|
-
while flag == 0
|
173
|
+
while flag == 0
|
176
174
|
@logger.info("wait_command #{command}")
|
177
|
-
Net::SSH.start(host, 'ubuntu', :
|
175
|
+
Net::SSH.start(host, 'ubuntu', proxy: get_proxy, key_data: [@private_key]) do |ssh|
|
178
176
|
channel = ssh.open_channel do |ch|
|
179
177
|
ch.exec "#{command}" do |ch, success|
|
180
|
-
|
178
|
+
fail "could not execute #{command}" unless success
|
181
179
|
|
182
180
|
# "on_data" is called when the process writes something to stdout
|
183
181
|
ch.on_data do |c, data|
|
184
182
|
@logger.info("#{data.inspect}")
|
185
|
-
if data.chomp ==
|
183
|
+
if data.chomp == 'true'
|
186
184
|
@logger.info("wait_command #{command} is true")
|
187
185
|
flag = 1
|
188
186
|
else
|
@@ -193,7 +191,7 @@ module OpenStudioAwsMethods
|
|
193
191
|
# "on_extended_data" is called when the process writes something to stderr
|
194
192
|
ch.on_extended_data do |c, type, data|
|
195
193
|
@logger.info("#{data.inspect}")
|
196
|
-
if data ==
|
194
|
+
if data == 'true'
|
197
195
|
@logger.info("wait_command #{command} is true")
|
198
196
|
flag = 1
|
199
197
|
else
|
@@ -206,13 +204,13 @@ module OpenStudioAwsMethods
|
|
206
204
|
end
|
207
205
|
rescue Net::SSH::HostKeyMismatch => e
|
208
206
|
e.remember_host!
|
209
|
-
@logger.info(
|
207
|
+
@logger.info('key mismatch, retry')
|
210
208
|
sleep 5
|
211
209
|
retry
|
212
210
|
rescue SystemCallError, Timeout::Error => e
|
213
211
|
# port 22 might not be available immediately after the instance finishes launching
|
214
212
|
sleep 5
|
215
|
-
@logger.info(
|
213
|
+
@logger.info('Timeout. Perhaps there is a communication error to EC2? Will try again')
|
216
214
|
retry
|
217
215
|
end
|
218
216
|
end
|
@@ -220,7 +218,7 @@ module OpenStudioAwsMethods
|
|
220
218
|
def download_file(host, remote_path, local_path)
|
221
219
|
retries = 0
|
222
220
|
begin
|
223
|
-
Net::SCP.start(host, 'ubuntu', :
|
221
|
+
Net::SCP.start(host, 'ubuntu', proxy: get_proxy, key_data: [@private_key]) do |scp|
|
224
222
|
scp.download! remote_path, local_path
|
225
223
|
end
|
226
224
|
rescue SystemCallError, Timeout::Error => e
|
@@ -1,18 +1,18 @@
|
|
1
1
|
# NOTE: Do not modify this file as it is copied over. Modify the source file and rerun rake import_files
|
2
2
|
######################################################################
|
3
|
-
# Copyright (c) 2008-2014, Alliance for Sustainable Energy.
|
3
|
+
# Copyright (c) 2008-2014, Alliance for Sustainable Energy.
|
4
4
|
# All rights reserved.
|
5
|
-
#
|
5
|
+
#
|
6
6
|
# This library is free software; you can redistribute it and/or
|
7
7
|
# modify it under the terms of the GNU Lesser General Public
|
8
8
|
# License as published by the Free Software Foundation; either
|
9
9
|
# version 2.1 of the License, or (at your option) any later version.
|
10
|
-
#
|
10
|
+
#
|
11
11
|
# This library is distributed in the hope that it will be useful,
|
12
12
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14
14
|
# Lesser General Public License for more details.
|
15
|
-
#
|
15
|
+
#
|
16
16
|
# You should have received a copy of the GNU Lesser General Public
|
17
17
|
# License along with this library; if not, write to the Free Software
|
18
18
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
@@ -50,7 +50,7 @@ class OpenStudioAwsWrapper
|
|
50
50
|
|
51
51
|
# todo: put this into a module
|
52
52
|
VALID_OPTIONS = [
|
53
|
-
|
53
|
+
:proxy, :credentials
|
54
54
|
]
|
55
55
|
|
56
56
|
def initialize(options = {}, group_uuid = nil)
|
@@ -71,31 +71,31 @@ class OpenStudioAwsWrapper
|
|
71
71
|
|
72
72
|
def create_or_retrieve_security_group(sg_name = nil)
|
73
73
|
tmp_name = sg_name || 'openstudio-server-sg-v1'
|
74
|
-
group = @aws.describe_security_groups(
|
74
|
+
group = @aws.describe_security_groups(filters: [{ name: 'group-name', values: [tmp_name] }])
|
75
75
|
logger.info "Length of the security group is: #{group.data.security_groups.length}"
|
76
76
|
if group.data.security_groups.length == 0
|
77
|
-
logger.info
|
78
|
-
@aws.create_security_group(
|
77
|
+
logger.info 'server group not found --- will create a new one'
|
78
|
+
@aws.create_security_group(group_name: tmp_name, description: "group dynamically created by #{__FILE__}")
|
79
79
|
@aws.authorize_security_group_ingress(
|
80
|
-
|
81
|
-
:
|
82
|
-
:
|
83
|
-
|
80
|
+
|
81
|
+
group_name: tmp_name,
|
82
|
+
ip_permissions: [
|
83
|
+
{ ip_protocol: 'tcp', from_port: 1, to_port: 65_535, ip_ranges: [cidr_ip: '0.0.0.0/0'] }
|
84
84
|
]
|
85
|
-
|
85
|
+
|
86
86
|
)
|
87
87
|
@aws.authorize_security_group_ingress(
|
88
|
-
|
89
|
-
:
|
90
|
-
:
|
91
|
-
|
92
|
-
|
88
|
+
|
89
|
+
group_name: tmp_name,
|
90
|
+
ip_permissions: [
|
91
|
+
{ ip_protocol: 'icmp', from_port: -1, to_port: -1, ip_ranges: [cidr_ip: '0.0.0.0/0']
|
92
|
+
}
|
93
93
|
]
|
94
|
-
|
94
|
+
|
95
95
|
)
|
96
96
|
|
97
97
|
# reload group information
|
98
|
-
group = @aws.describe_security_groups(
|
98
|
+
group = @aws.describe_security_groups(filters: [{ name: 'group-name', values: [tmp_name] }])
|
99
99
|
end
|
100
100
|
@security_group_name = group.data.security_groups.first.group_name
|
101
101
|
logger.info("server_group #{group.data.security_groups.first.group_name}")
|
@@ -108,7 +108,7 @@ class OpenStudioAwsWrapper
|
|
108
108
|
map << zn.to_hash
|
109
109
|
end
|
110
110
|
|
111
|
-
{:
|
111
|
+
{ availability_zone_info: map }
|
112
112
|
end
|
113
113
|
|
114
114
|
def describe_availability_zones_json
|
@@ -118,8 +118,8 @@ class OpenStudioAwsWrapper
|
|
118
118
|
def describe_total_instances
|
119
119
|
resp = @aws.describe_instance_status
|
120
120
|
|
121
|
-
region = resp.instance_statuses.length > 0 ? resp.instance_statuses.first.availability_zone :
|
122
|
-
{:
|
121
|
+
region = resp.instance_statuses.length > 0 ? resp.instance_statuses.first.availability_zone : 'no_instances'
|
122
|
+
{ total_instances: resp.instance_statuses.length, region: region }
|
123
123
|
end
|
124
124
|
|
125
125
|
def describe_total_instances_json
|
@@ -128,23 +128,22 @@ class OpenStudioAwsWrapper
|
|
128
128
|
|
129
129
|
# return all of the running instances, or filter by the group_uuid & instance type
|
130
130
|
def describe_running_instances(group_uuid = nil, openstudio_instance_type = nil)
|
131
|
-
|
132
131
|
resp = nil
|
133
132
|
if group_uuid
|
134
133
|
resp = @aws.describe_instances(
|
135
|
-
|
136
|
-
:
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
#{:name => "tag-value", :values => [group_uuid.to_s, "OpenStudio#{@openstudio_instance_type.capitalize}"]}
|
141
|
-
#{:name => "tag:key=value", :values => ["GroupUUID=#{group_uuid.to_s}"]}
|
134
|
+
|
135
|
+
filters: [
|
136
|
+
{ name: 'instance-state-code', values: [0.to_s, 16.to_s] }, # running or pending
|
137
|
+
{ name: 'tag-key', values: ['GroupUUID'] },
|
138
|
+
{ name: 'tag-value', values: [group_uuid.to_s] } # todo: how to check for the server versions
|
139
|
+
# {:name => "tag-value", :values => [group_uuid.to_s, "OpenStudio#{@openstudio_instance_type.capitalize}"]}
|
140
|
+
# {:name => "tag:key=value", :values => ["GroupUUID=#{group_uuid.to_s}"]}
|
142
141
|
]
|
143
|
-
|
142
|
+
|
144
143
|
)
|
145
144
|
else
|
146
145
|
# todo: need to restrict this to only the current user
|
147
|
-
resp = @aws.describe_instances
|
146
|
+
resp = @aws.describe_instances
|
148
147
|
end
|
149
148
|
|
150
149
|
instance_data = nil
|
@@ -157,10 +156,9 @@ class OpenStudioAwsWrapper
|
|
157
156
|
instance_data << i.to_hash
|
158
157
|
end
|
159
158
|
|
160
|
-
|
161
159
|
end
|
162
160
|
else
|
163
|
-
logger.info
|
161
|
+
logger.info 'no running instances found'
|
164
162
|
end
|
165
163
|
end
|
166
164
|
|
@@ -173,15 +171,15 @@ class OpenStudioAwsWrapper
|
|
173
171
|
# todo: test the filter. i don't think that it is exposed in the AWS gem?
|
174
172
|
if owned_by_me
|
175
173
|
if filter
|
176
|
-
resp = @aws.describe_images(
|
174
|
+
resp = @aws.describe_images(owners: [:self], filter: filter).data
|
177
175
|
else
|
178
|
-
resp = @aws.describe_images(
|
176
|
+
resp = @aws.describe_images(owners: [:self]).data
|
179
177
|
end
|
180
178
|
else
|
181
179
|
if filter
|
182
|
-
resp = @aws.describe_images(
|
180
|
+
resp = @aws.describe_images(filter: filter).data
|
183
181
|
else
|
184
|
-
resp = @aws.describe_images(
|
182
|
+
resp = @aws.describe_images(image_ids: image_ids).data
|
185
183
|
end
|
186
184
|
end
|
187
185
|
|
@@ -209,8 +207,8 @@ class OpenStudioAwsWrapper
|
|
209
207
|
# the describe_key_pairs method will raise an expection if it can't find the keypair, so catch it
|
210
208
|
resp = nil
|
211
209
|
begin
|
212
|
-
resp = @aws.describe_key_pairs(
|
213
|
-
|
210
|
+
resp = @aws.describe_key_pairs(key_names: [tmp_name]).data
|
211
|
+
fail 'looks like there are 2 key pairs with the same name' if resp.key_pairs.size >= 2
|
214
212
|
rescue
|
215
213
|
logger.info "could not find key pair '#{tmp_name}'"
|
216
214
|
end
|
@@ -219,7 +217,7 @@ class OpenStudioAwsWrapper
|
|
219
217
|
# create the new key_pair
|
220
218
|
# check if the key pair name exists
|
221
219
|
# create a new key pair everytime
|
222
|
-
keypair = @aws.create_key_pair(
|
220
|
+
keypair = @aws.create_key_pair(key_name: tmp_name)
|
223
221
|
|
224
222
|
# save the private key to memory (which can later be persisted via the save_private_key method)
|
225
223
|
@private_key = keypair.data.key_material
|
@@ -244,27 +242,27 @@ class OpenStudioAwsWrapper
|
|
244
242
|
File.open(filename, 'w') { |f| f << @private_key }
|
245
243
|
File.chmod(0600, filename)
|
246
244
|
else
|
247
|
-
logger.error
|
245
|
+
logger.error 'no private key found in which to persist'
|
248
246
|
end
|
249
247
|
end
|
250
248
|
|
251
249
|
def launch_server(image_id, instance_type, options = {})
|
252
|
-
defaults = {:
|
250
|
+
defaults = { user_id: 'unknown_user' }
|
253
251
|
options = defaults.merge(options)
|
254
252
|
|
255
|
-
user_data = File.read(File.expand_path(File.dirname(__FILE__))+'/server_script.sh')
|
253
|
+
user_data = File.read(File.expand_path(File.dirname(__FILE__)) + '/server_script.sh')
|
256
254
|
@server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, @security_group_name, @group_uuid, @private_key, @proxy)
|
257
255
|
|
258
|
-
|
259
|
-
|
256
|
+
fail 'image_id is nil' unless image_id
|
257
|
+
fail 'instance type is nil' unless instance_type
|
260
258
|
@server.launch_instance(image_id, instance_type, user_data, options[:user_id])
|
261
259
|
end
|
262
260
|
|
263
261
|
def launch_workers(image_id, instance_type, num, options = {})
|
264
|
-
defaults = {:
|
262
|
+
defaults = { user_id: 'unknown_user' }
|
265
263
|
options = defaults.merge(options)
|
266
264
|
|
267
|
-
user_data = File.read(File.expand_path(File.dirname(__FILE__))+'/worker_script.sh.template')
|
265
|
+
user_data = File.read(File.expand_path(File.dirname(__FILE__)) + '/worker_script.sh.template')
|
268
266
|
user_data.gsub!(/SERVER_IP/, @server.data.ip)
|
269
267
|
user_data.gsub!(/SERVER_HOSTNAME/, 'master')
|
270
268
|
user_data.gsub!(/SERVER_ALIAS/, '')
|
@@ -291,10 +289,10 @@ class OpenStudioAwsWrapper
|
|
291
289
|
# blocking method that waits for servers and workers to be fully configured (i.e. execution of user_data has
|
292
290
|
# occured on all nodes)
|
293
291
|
def configure_server_and_workers
|
294
|
-
#todo: add a timeout here!
|
295
|
-
logger.info(
|
292
|
+
# todo: add a timeout here!
|
293
|
+
logger.info('waiting for server user_data to complete')
|
296
294
|
@server.wait_command(@server.data.ip, '[ -e /home/ubuntu/user_data_done ] && echo "true"')
|
297
|
-
@logger.info(
|
295
|
+
@logger.info('waiting for worker user_data to complete')
|
298
296
|
@workers.each { |worker| worker.wait_command(worker.data.ip, '[ -e /home/ubuntu/user_data_done ] && echo "true"') }
|
299
297
|
|
300
298
|
ips = "master|#{@server.data.ip}|#{@server.data.dns}|#{@server.data.procs}|ubuntu|ubuntu|true\n"
|
@@ -309,7 +307,7 @@ class OpenStudioAwsWrapper
|
|
309
307
|
@server.shell_command(@server.data.ip, '~/setup-ssh-keys.sh')
|
310
308
|
@server.shell_command(@server.data.ip, '~/setup-ssh-worker-nodes.sh ip_addresses')
|
311
309
|
|
312
|
-
mongoid = File.read(File.expand_path(File.dirname(__FILE__))+'/mongoid.yml.template')
|
310
|
+
mongoid = File.read(File.expand_path(File.dirname(__FILE__)) + '/mongoid.yml.template')
|
313
311
|
mongoid.gsub!(/SERVER_IP/, @server.data.ip)
|
314
312
|
file = Tempfile.new('mongoid.yml')
|
315
313
|
file.write(mongoid)
|
@@ -325,7 +323,6 @@ class OpenStudioAwsWrapper
|
|
325
323
|
true
|
326
324
|
end
|
327
325
|
|
328
|
-
|
329
326
|
# method to query the amazon api to find the server (if it exists), based on the group id
|
330
327
|
# if it is found, then it will set the @server member variable.
|
331
328
|
# Note that the information around keys and security groups is pulled from the instance information.
|
@@ -333,11 +330,11 @@ class OpenStudioAwsWrapper
|
|
333
330
|
group_uuid = group_uuid || @group_uuid
|
334
331
|
|
335
332
|
logger.info "finding the server for groupid of #{group_uuid}"
|
336
|
-
|
333
|
+
fail 'no group uuid defined either in member variable or method argument' if group_uuid.nil?
|
337
334
|
|
338
335
|
resp = describe_running_instances(group_uuid, :server)
|
339
336
|
if resp
|
340
|
-
|
337
|
+
fail "more than one server running with group uuid of #{group_uuid} found, expecting only one" if resp.size > 1
|
341
338
|
resp = resp.first
|
342
339
|
if !@server
|
343
340
|
logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]"
|
@@ -347,23 +344,23 @@ class OpenStudioAwsWrapper
|
|
347
344
|
logger.info "Server instance is already defined with instance #{resp[:instance_id]}"
|
348
345
|
end
|
349
346
|
else
|
350
|
-
|
347
|
+
fail 'could not find a running server instance'
|
351
348
|
end
|
352
349
|
end
|
353
350
|
|
354
|
-
# method to hit the existing list of available amis and compare to the list of AMIs on Amazon and then generate the
|
351
|
+
# method to hit the existing list of available amis and compare to the list of AMIs on Amazon and then generate the
|
355
352
|
# new ami list
|
356
353
|
def create_new_ami_json(version = 1)
|
357
354
|
# get the list of existing amis from developer.nrel.gov
|
358
355
|
existing_amis = OpenStudioAmis.new(1).list
|
359
356
|
|
360
357
|
# list of available AMIs from AWS
|
361
|
-
available_amis = describe_amis
|
358
|
+
available_amis = describe_amis
|
362
359
|
|
363
360
|
amis = transform_ami_lists(existing_amis, available_amis)
|
364
361
|
|
365
362
|
if version == 1
|
366
|
-
puts
|
363
|
+
puts 'Creating version 1 of the AMI file'
|
367
364
|
version1 = {}
|
368
365
|
|
369
366
|
# now grab the good keys - they should be sorted newest to older... so go backwards
|
@@ -376,8 +373,8 @@ class OpenStudioAwsWrapper
|
|
376
373
|
end
|
377
374
|
|
378
375
|
# create the default version. First sort, then grab the first hash's values
|
379
|
-
|
380
|
-
version1.sort_by
|
376
|
+
|
377
|
+
version1.sort_by
|
381
378
|
default_v = nil
|
382
379
|
version1 = Hash[version1.sort_by { |k, _| k.to_s.to_version }.reverse]
|
383
380
|
default_v = version1.keys[0]
|
@@ -397,15 +394,15 @@ class OpenStudioAwsWrapper
|
|
397
394
|
def to_os_worker_hash
|
398
395
|
worker_hash = []
|
399
396
|
@workers.each { |worker|
|
400
|
-
worker_hash.push(
|
401
|
-
:
|
402
|
-
:
|
403
|
-
:
|
404
|
-
:
|
405
|
-
|
397
|
+
worker_hash.push(
|
398
|
+
id: worker.data.id,
|
399
|
+
ip: 'http://' + worker.data.ip,
|
400
|
+
dns: worker.data.dns,
|
401
|
+
procs: worker.data.procs
|
402
|
+
)
|
406
403
|
}
|
407
404
|
|
408
|
-
out = {:
|
405
|
+
out = { workers: worker_hash }
|
409
406
|
logger.info out
|
410
407
|
|
411
408
|
out
|
@@ -434,27 +431,26 @@ class OpenStudioAwsWrapper
|
|
434
431
|
# transform the available amis into an easier to read format
|
435
432
|
def transform_ami_lists(existing, available)
|
436
433
|
# initialize ami hash
|
437
|
-
amis = {:
|
434
|
+
amis = { openstudio_server: {}, openstudio: {} }
|
438
435
|
list_of_svs = []
|
439
436
|
|
440
437
|
available[:images].each do |ami|
|
441
438
|
sv = ami[:tags_hash][:openstudio_server_version]
|
442
439
|
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
sv = get_next_version("0.0.1", list_of_svs)
|
440
|
+
if sv.nil? || sv == ''
|
441
|
+
puts 'found nil version, incrementing from 0.0.1'
|
442
|
+
sv = get_next_version('0.0.1', list_of_svs)
|
447
443
|
end
|
448
444
|
list_of_svs << sv
|
449
445
|
|
450
|
-
amis[:openstudio_server][sv.to_sym] = {}
|
446
|
+
amis[:openstudio_server][sv.to_sym] = {} unless amis[:openstudio_server][sv.to_sym]
|
451
447
|
a = amis[:openstudio_server][sv.to_sym]
|
452
448
|
|
453
449
|
# initialize ami hash
|
454
|
-
a[:amis] = {}
|
450
|
+
a[:amis] = {} unless a[:amis]
|
455
451
|
|
456
452
|
# fill in data (this will override data currently)
|
457
|
-
a[:deprecate] = true if sv.to_version.satisfies(
|
453
|
+
a[:deprecate] = true if sv.to_version.satisfies('0.0.*')
|
458
454
|
a[:openstudio_version] = ami[:tags_hash][:openstudio_version] if ami[:tags_hash][:openstudio_version]
|
459
455
|
a[:openstudio_version_sha] = ami[:tags_hash][:openstudio_version_sha] if ami[:tags_hash][:openstudio_version_sha]
|
460
456
|
a[:user_uuid] = ami[:tags_hash][:user_uuid] if ami[:tags_hash][:user_uuid]
|
@@ -462,37 +458,37 @@ class OpenStudioAwsWrapper
|
|
462
458
|
a[:created_on] = ami[:tags_hash][:created_on] if ami[:tags_hash][:created_on]
|
463
459
|
a[:openstudio_server_version] = sv.to_s
|
464
460
|
if ami[:tags_hash][:tested]
|
465
|
-
a[:tested] = ami[:tags_hash][:tested].downcase ==
|
461
|
+
a[:tested] = ami[:tags_hash][:tested].downcase == 'true'
|
466
462
|
else
|
467
463
|
a[:tested] = false
|
468
464
|
end
|
469
465
|
|
470
466
|
if ami[:name] =~ /Worker|Cluster/
|
471
|
-
if ami[:virtualization_type] ==
|
467
|
+
if ami[:virtualization_type] == 'paravirtual'
|
472
468
|
a[:amis][:worker] = ami[:image_id]
|
473
|
-
elsif ami[:virtualization_type] ==
|
469
|
+
elsif ami[:virtualization_type] == 'hvm'
|
474
470
|
a[:amis][:cc2worker] = ami[:image_id]
|
475
471
|
else
|
476
|
-
|
472
|
+
fail "unknown virtualization_type in #{ami[:name]}"
|
477
473
|
end
|
478
474
|
elsif ami[:name] =~ /Server/
|
479
475
|
a[:amis][:server] = ami[:image_id]
|
480
476
|
end
|
481
477
|
end
|
482
|
-
#puts "Current AMIs: #{JSON.pretty_generate(amis)}"
|
478
|
+
# puts "Current AMIs: #{JSON.pretty_generate(amis)}"
|
483
479
|
|
484
480
|
# merge in the existing AMIs the existing amis into the 'unknown category, but don't flag them as 'deprecate'
|
485
|
-
#puts "Existing AMIs: #{JSON.pretty_generate(existing)}"
|
481
|
+
# puts "Existing AMIs: #{JSON.pretty_generate(existing)}"
|
486
482
|
existing.keys.each do |ami_key|
|
487
|
-
next if ami_key ==
|
483
|
+
next if ami_key == 'default'.to_sym # ignore default
|
488
484
|
|
489
485
|
# get next version
|
490
|
-
next_version = get_next_version(
|
486
|
+
next_version = get_next_version('0.0.1', list_of_svs)
|
491
487
|
list_of_svs << next_version
|
492
488
|
|
493
|
-
amis[:openstudio_server][next_version.to_sym] = {}
|
489
|
+
amis[:openstudio_server][next_version.to_sym] = {} unless amis[:openstudio_server][next_version.to_sym]
|
494
490
|
a = amis[:openstudio_server][next_version.to_sym]
|
495
|
-
a[:amis] = {}
|
491
|
+
a[:amis] = {} unless a[:amis]
|
496
492
|
|
497
493
|
a[:openstudio_version] = ami_key
|
498
494
|
a[:amis][:server] = existing[ami_key][:server]
|
@@ -501,18 +497,18 @@ class OpenStudioAwsWrapper
|
|
501
497
|
a[:openstudio_server_version] = next_version.to_s
|
502
498
|
end
|
503
499
|
|
504
|
-
#puts "After merge: #{JSON.pretty_generate(amis)}"
|
500
|
+
# puts "After merge: #{JSON.pretty_generate(amis)}"
|
505
501
|
# flip these around for openstudio server section
|
506
502
|
amis[:openstudio_server].keys.each do |key|
|
507
|
-
next if key.to_s.to_version.satisfies(
|
503
|
+
next if key.to_s.to_version.satisfies('0.0.*')
|
508
504
|
|
509
505
|
a = amis[:openstudio_server][key]
|
510
506
|
ov = a[:openstudio_version]
|
511
507
|
|
512
|
-
amis[:openstudio][ov] = {}
|
508
|
+
amis[:openstudio][ov] = {} unless amis[:openstudio][ov]
|
513
509
|
osv = key
|
514
|
-
amis[:openstudio][ov][osv] = {}
|
515
|
-
amis[:openstudio][ov][osv][:amis] = {}
|
510
|
+
amis[:openstudio][ov][osv] = {} unless amis[:openstudio][ov][osv]
|
511
|
+
amis[:openstudio][ov][osv][:amis] = {} unless amis[:openstudio][ov][osv][:amis]
|
516
512
|
amis[:openstudio][ov][osv][:amis][:server] = a[:amis][:server]
|
517
513
|
amis[:openstudio][ov][osv][:amis][:worker] = a[:amis][:worker]
|
518
514
|
amis[:openstudio][ov][osv][:amis][:cc2worker] = a[:amis][:cc2worker]
|
@@ -528,9 +524,8 @@ class OpenStudioAwsWrapper
|
|
528
524
|
end
|
529
525
|
amis[:openstudio] = Hash[amis[:openstudio].sort_by { |k, _| k.to_s.to_version }.reverse]
|
530
526
|
|
531
|
-
#puts "After sort: #{JSON.pretty_generate(amis)}"
|
527
|
+
# puts "After sort: #{JSON.pretty_generate(amis)}"
|
532
528
|
|
533
529
|
amis
|
534
530
|
end
|
535
|
-
|
536
531
|
end
|