openstudio-aws 0.4.0.alpha2 → 0.4.0.alpha3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/lib/openstudio/aws/aws.rb +31 -26
- data/lib/openstudio/aws/version.rb +1 -1
- data/lib/openstudio/lib/openstudio_aws_wrapper.rb +50 -21
- data/spec/aws_instances/aws_spec_api.rb +54 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2f024e917704db3001f98d70663da27d0a998d54
|
4
|
+
data.tar.gz: 43af587c2f58ff2690c1a91ccd242b4f133c0171
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f7b216034b3aaee08482319454aef815a9b3437c21e1dc2821fe2393cf483179c870e3eaf0d05987cd82d62d1dca795cce4a287d871ffcd622cb0f11c56a5d2
|
7
|
+
data.tar.gz: 3d087837a34a9c847b4f72100ac94ca1328ea2e88909009fa4eb44d7f2db6750010a9f75b14938d96b111e6513adea75c77d4f4d7ed9260daab02dcae156d4f0
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
OpenStudio AWS Gem Change Log
|
2
2
|
==================================
|
3
3
|
|
4
|
+
Version 0.4.0.alpha3
|
5
|
+
-------------
|
6
|
+
* Load the key name and security groups from the AWS instance information.
|
7
|
+
* Add method on Aws to get the group_uuid
|
8
|
+
|
4
9
|
Version 0.4.0.alpha2
|
5
10
|
-------------
|
6
11
|
* Load worker keys from disk (if they exist) when constructing the OpenStudioAwsWrapper class
|
data/lib/openstudio/aws/aws.rb
CHANGED
@@ -138,7 +138,7 @@ module OpenStudio
|
|
138
138
|
@os_aws.private_key_file_name = options[:private_key_file_name]
|
139
139
|
else
|
140
140
|
# Save the private key if you did not pass in an already existing key_pair_name
|
141
|
-
@os_aws.save_private_key
|
141
|
+
@os_aws.save_private_key @save_directory
|
142
142
|
end
|
143
143
|
|
144
144
|
server_options = {
|
@@ -150,32 +150,16 @@ module OpenStudio
|
|
150
150
|
|
151
151
|
# save the worker pem and public to the directory
|
152
152
|
# presently, this will always overwrite the worker key, is that okay? Is this really needed later?
|
153
|
-
@os_aws.save_worker_keys
|
154
|
-
|
155
|
-
# if instance_data[:ebs_volume_id]
|
156
|
-
# server_options[:ebs_volume_id] = instance_data[:ebs_volume_id]
|
157
|
-
# end
|
153
|
+
@os_aws.save_worker_keys @save_directory
|
158
154
|
|
159
155
|
@os_aws.launch_server(options[:image_id], options[:instance_type], server_options)
|
160
156
|
end
|
161
157
|
|
162
|
-
#
|
158
|
+
# create workers after the server has been created.
|
163
159
|
#
|
164
|
-
# @
|
165
|
-
|
166
|
-
|
167
|
-
puts ''
|
168
|
-
puts 'Server SSH Command:'
|
169
|
-
puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
|
170
|
-
if @os_aws.workers.size > 0
|
171
|
-
puts ''
|
172
|
-
puts 'Worker SSH Command:'
|
173
|
-
@os_aws.workers.each do |worker|
|
174
|
-
puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{worker.data[:dns]}"
|
175
|
-
end
|
176
|
-
end
|
177
|
-
end
|
178
|
-
|
160
|
+
# @param number_of_instances [Integer] Number of worker instances to create
|
161
|
+
# @param options [Hash]
|
162
|
+
# @option options [String] :instance_type Type of server to start (e.g. m3.medium, m3.xlarge, etc.)
|
179
163
|
def create_workers(number_of_instances, options = {}, user_id = 'unknown_user')
|
180
164
|
defaults = {
|
181
165
|
instance_type: 'm2.4xlarge',
|
@@ -226,13 +210,27 @@ module OpenStudio
|
|
226
210
|
@os_aws.configure_server_and_workers
|
227
211
|
end
|
228
212
|
|
213
|
+
# Write out to the terminal the connection information for the servers and workers
|
214
|
+
#
|
215
|
+
# @return [nil] Only prints to the screen. No return is expected
|
216
|
+
def print_connection_info
|
217
|
+
# Print out some debugging commands (probably work on mac/linux only)
|
218
|
+
puts ''
|
219
|
+
puts 'Server SSH Command:'
|
220
|
+
puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
|
221
|
+
if @os_aws.workers.size > 0
|
222
|
+
puts ''
|
223
|
+
puts 'Worker SSH Command:'
|
224
|
+
@os_aws.workers.each do |worker|
|
225
|
+
puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{worker.data[:dns]}"
|
226
|
+
end
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
229
230
|
# Return information on the cluster instances as a hash. This includes IP addresses, host names, number of processors, etc.
|
230
231
|
# @return [Hash] Data about the configured cluster
|
231
232
|
def cluster_info
|
232
|
-
|
233
|
-
h[:workers] = @os_aws.to_os_worker_hash[:workers]
|
234
|
-
|
235
|
-
h
|
233
|
+
@os_aws.to_os_hash
|
236
234
|
end
|
237
235
|
|
238
236
|
# Save a JSON with information about the cluster that was configured.
|
@@ -416,6 +414,13 @@ module OpenStudio
|
|
416
414
|
image
|
417
415
|
end
|
418
416
|
|
417
|
+
# Return the Group UUID as defined in the AWS wrapper
|
418
|
+
#
|
419
|
+
# @return [String] UUID
|
420
|
+
def group_uuid
|
421
|
+
@os_aws.group_uuid
|
422
|
+
end
|
423
|
+
|
419
424
|
private
|
420
425
|
|
421
426
|
def os_aws_file_location
|
@@ -65,7 +65,7 @@ class OpenStudioAwsWrapper
|
|
65
65
|
work_dir = options[:save_directory] || '.'
|
66
66
|
if File.exist?(File.join(work_dir, 'ec2_worker_key.pem')) && File.exist?(File.join(work_dir, 'ec2_worker_key.pub'))
|
67
67
|
logger.info "Worker keys already exist, loading from #{work_dir}"
|
68
|
-
|
68
|
+
load_worker_key(File.join(work_dir, 'ec2_worker_key.pem'))
|
69
69
|
else
|
70
70
|
logger.info 'Generating new worker keys'
|
71
71
|
@worker_keys = SSHKey.generate
|
@@ -338,13 +338,24 @@ class OpenStudioAwsWrapper
|
|
338
338
|
@private_key = File.read(filename)
|
339
339
|
end
|
340
340
|
|
341
|
+
# Load the worker key for communicating between the server and worker instances on AWS. The public key
|
342
|
+
# will be automatically created when loading the private key
|
343
|
+
#
|
344
|
+
# @param private_key_filename [String] Fully qualified path to the worker private key
|
345
|
+
def load_worker_key(private_key_filename)
|
346
|
+
logger.info "Loading worker keys from #{private_key_filename}"
|
347
|
+
@worker_keys_filename = private_key_filename
|
348
|
+
@worker_keys = SSHKey.new(File.read(@worker_keys_filename))
|
349
|
+
end
|
350
|
+
|
351
|
+
# Save the private key to disk
|
341
352
|
def save_private_key(directory = '.', filename = 'ec2_server_key.pem')
|
342
353
|
if @private_key
|
343
354
|
@private_key_file_name = File.expand_path "#{directory}/#{filename}"
|
344
355
|
logger.info "Saving server private key in #{@private_key_file_name}"
|
345
356
|
File.open(@private_key_file_name, 'w') { |f| f << @private_key }
|
346
357
|
logger.info 'Setting permissions of server private key to 0600'
|
347
|
-
|
358
|
+
File.chmod(0600, @private_key_file_name)
|
348
359
|
else
|
349
360
|
fail "No private key found in which to persist with filename #{filename}"
|
350
361
|
end
|
@@ -352,11 +363,11 @@ class OpenStudioAwsWrapper
|
|
352
363
|
|
353
364
|
# save off the worker public/private keys that were created
|
354
365
|
def save_worker_keys(directory = '.')
|
355
|
-
|
356
|
-
logger.info "Saving worker private key in #{
|
357
|
-
File.open(
|
366
|
+
@worker_keys_filename = "#{directory}/ec2_worker_key.pem"
|
367
|
+
logger.info "Saving worker private key in #{@worker_keys_filename}"
|
368
|
+
File.open(@worker_keys_filename, 'w') { |f| f << @worker_keys.private_key }
|
358
369
|
logger.info 'Setting permissions of worker private key to 0600'
|
359
|
-
File.chmod(0600,
|
370
|
+
File.chmod(0600, @worker_keys_filename)
|
360
371
|
|
361
372
|
wk = "#{directory}/ec2_worker_key.pub"
|
362
373
|
logger.info "Saving worker public key in #{wk}"
|
@@ -403,7 +414,6 @@ class OpenStudioAwsWrapper
|
|
403
414
|
logger.info("worker user_data #{user_data.inspect}")
|
404
415
|
|
405
416
|
# thread the launching of the workers
|
406
|
-
|
407
417
|
num.times do
|
408
418
|
@workers << OpenStudioAwsInstance.new(@aws, :worker, @key_pair_name, @security_groups, @group_uuid,
|
409
419
|
@private_key, @private_key_file_name, @proxy)
|
@@ -458,14 +468,19 @@ class OpenStudioAwsWrapper
|
|
458
468
|
end
|
459
469
|
|
460
470
|
# method to query the amazon api to find the server (if it exists), based on the group id
|
461
|
-
# if it is found, then it will set the @server
|
471
|
+
# if it is found, then it will set the @server instance variable. The security groups are assigned from the
|
472
|
+
# server node information on AWS if the security groups have not been initialized yet.
|
473
|
+
#
|
462
474
|
# Note that the information around keys and security groups is pulled from the instance information.
|
475
|
+
# @param server_data_hash [Hash] Server data
|
476
|
+
# @option server_data_hash [String] :group_id Group ID of the analysis
|
477
|
+
# @option server_data_hash [String] :server.private_key_file_name Name of the private key to communicate to the server
|
463
478
|
def find_server(server_data_hash)
|
464
479
|
@group_uuid = server_data_hash[:group_id] || @group_uuid
|
465
480
|
load_private_key(server_data_hash[:server][:private_key_file_name])
|
466
481
|
|
467
482
|
logger.info "Finding the server for GroupUUID of #{group_uuid}"
|
468
|
-
fail 'no GroupUUID defined either in member variable or method argument' if group_uuid.nil?
|
483
|
+
fail 'no GroupUUID defined either in member variable or method argument' if @group_uuid.nil?
|
469
484
|
|
470
485
|
# This should really just be a single call to describe running instances
|
471
486
|
@server = nil
|
@@ -476,7 +491,17 @@ class OpenStudioAwsWrapper
|
|
476
491
|
if !@server
|
477
492
|
if resp
|
478
493
|
logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]"
|
479
|
-
|
494
|
+
|
495
|
+
sg = resp[:security_groups].map { |s| s[:group_id] }
|
496
|
+
# Set the security groups of the object if these groups haven't been assigned yet.
|
497
|
+
@security_groups = sg if @security_groups.empty?
|
498
|
+
logger.info "The security groups in aws wrapper are #{@security_groups}"
|
499
|
+
|
500
|
+
# set the key name from AWS if it isn't yet assigned
|
501
|
+
logger.info 'Setting the keyname in the aws wrapper'
|
502
|
+
@key_pair_name = resp[:key_name] unless @key_pair_name
|
503
|
+
|
504
|
+
@server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, sg, @group_uuid, @private_key, @private_key_file_name, @proxy)
|
480
505
|
|
481
506
|
@server.load_instance_data(resp)
|
482
507
|
end
|
@@ -487,12 +512,12 @@ class OpenStudioAwsWrapper
|
|
487
512
|
logger.info 'could not find a running server instance'
|
488
513
|
end
|
489
514
|
|
490
|
-
#
|
515
|
+
# Find the worker instances.
|
491
516
|
if @workers.size == 0
|
492
517
|
resp = describe_running_instances(group_uuid, :worker)
|
493
518
|
if resp
|
494
519
|
resp.each do |r|
|
495
|
-
@workers << OpenStudioAwsInstance.new(@aws, :worker, r[:key_name], r[:security_groups].
|
520
|
+
@workers << OpenStudioAwsInstance.new(@aws, :worker, r[:key_name], r[:security_groups].map { |s| s[:group_id] }, @group_uuid, @private_key, @private_key_file_name, @proxy)
|
496
521
|
@workers.last.load_instance_data(r)
|
497
522
|
end
|
498
523
|
end
|
@@ -500,6 +525,10 @@ class OpenStudioAwsWrapper
|
|
500
525
|
logger.info 'Worker nodes are already defined'
|
501
526
|
end
|
502
527
|
|
528
|
+
# set the private key from the hash
|
529
|
+
load_private_key server_data_hash[:server][:private_key_file_name]
|
530
|
+
load_worker_key server_data_hash[:server][:worker_private_key_file_name]
|
531
|
+
|
503
532
|
# Really don't need to return anything because this sets the class instance variable
|
504
533
|
@server
|
505
534
|
end
|
@@ -558,23 +587,23 @@ class OpenStudioAwsWrapper
|
|
558
587
|
amis
|
559
588
|
end
|
560
589
|
|
561
|
-
|
562
|
-
|
563
|
-
@
|
564
|
-
|
590
|
+
# save off the instance configuration and instance information into a JSON file for later use
|
591
|
+
def to_os_hash
|
592
|
+
h = @server.to_os_hash
|
593
|
+
|
594
|
+
h[:server][:worker_private_key_file_name] = @worker_keys_filename
|
595
|
+
h[:workers] = @workers.map do |worker|
|
596
|
+
{
|
565
597
|
id: worker.data.id,
|
566
598
|
ip: "http://#{worker.data.ip}",
|
567
599
|
dns: worker.data.dns,
|
568
600
|
procs: worker.data.procs,
|
569
601
|
private_key_file_name: worker.private_key_file_name,
|
570
602
|
private_ip_address: worker.private_ip_address
|
571
|
-
|
603
|
+
}
|
572
604
|
end
|
573
605
|
|
574
|
-
|
575
|
-
logger.info out
|
576
|
-
|
577
|
-
out
|
606
|
+
h
|
578
607
|
end
|
579
608
|
|
580
609
|
# take the base version and increment the patch until
|
@@ -290,4 +290,58 @@ describe OpenStudio::Aws::Aws do
|
|
290
290
|
expect(@aws_2.os_aws.worker_keys.public_key).not_to be_nil
|
291
291
|
end
|
292
292
|
end
|
293
|
+
|
294
|
+
context 'stateful creation of server and worker' do
|
295
|
+
before(:all) do
|
296
|
+
@config = OpenStudio::Aws::Config.new
|
297
|
+
@aws = OpenStudio::Aws::Aws.new
|
298
|
+
end
|
299
|
+
|
300
|
+
it 'should create the server and save the state' do
|
301
|
+
options = {
|
302
|
+
instance_type: 'm3.medium',
|
303
|
+
image_id: SERVER_AMI,
|
304
|
+
tags: [
|
305
|
+
'ci_tests=true',
|
306
|
+
'ServerOnly=true'
|
307
|
+
]
|
308
|
+
}
|
309
|
+
|
310
|
+
test_pem_file = 'ec2_server_key.pem'
|
311
|
+
FileUtils.rm_f test_pem_file if File.exist? test_pem_file
|
312
|
+
FileUtils.rm_f 'server_data.json' if File.exist? 'server_data.json'
|
313
|
+
|
314
|
+
@aws.create_server(options)
|
315
|
+
@aws.save_cluster_info 'server_data.json'
|
316
|
+
|
317
|
+
h = @aws.os_aws.server.to_os_hash
|
318
|
+
expect(h[:group_id]).to be_a String
|
319
|
+
expect(h[:group_id]).to match /^[\d\S]{32}$/
|
320
|
+
expect(h[:location]).to eq 'AWS'
|
321
|
+
end
|
322
|
+
|
323
|
+
it 'should load server information from json and launch worker' do
|
324
|
+
aws2 = OpenStudio::Aws::Aws.new
|
325
|
+
aws2.load_instance_info_from_file('server_data.json')
|
326
|
+
|
327
|
+
options = {
|
328
|
+
instance_type: 'm3.medium',
|
329
|
+
image_id: WORKER_AMI
|
330
|
+
}
|
331
|
+
|
332
|
+
aws2.create_workers(1, options)
|
333
|
+
aws2.save_cluster_info 'server_data.json'
|
334
|
+
|
335
|
+
expect(File.exist?('server_data.json')).to eq true
|
336
|
+
# check if file exists
|
337
|
+
|
338
|
+
h = aws2.cluster_info # to make sure that the settings are correct
|
339
|
+
expect(h[:server][:worker_private_key_file_name]).to match /.*ec2_worker_key.pem/
|
340
|
+
expect(h[:workers].size).to eq 1
|
341
|
+
end
|
342
|
+
|
343
|
+
after :all do
|
344
|
+
@aws.terminate
|
345
|
+
end
|
346
|
+
end
|
293
347
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: openstudio-aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.0.
|
4
|
+
version: 0.4.0.alpha3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicholas Long
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-07-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: net-scp
|