openstudio-aws 0.4.0.alpha2 → 0.4.0.alpha3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/lib/openstudio/aws/aws.rb +31 -26
- data/lib/openstudio/aws/version.rb +1 -1
- data/lib/openstudio/lib/openstudio_aws_wrapper.rb +50 -21
- data/spec/aws_instances/aws_spec_api.rb +54 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2f024e917704db3001f98d70663da27d0a998d54
|
4
|
+
data.tar.gz: 43af587c2f58ff2690c1a91ccd242b4f133c0171
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f7b216034b3aaee08482319454aef815a9b3437c21e1dc2821fe2393cf483179c870e3eaf0d05987cd82d62d1dca795cce4a287d871ffcd622cb0f11c56a5d2
|
7
|
+
data.tar.gz: 3d087837a34a9c847b4f72100ac94ca1328ea2e88909009fa4eb44d7f2db6750010a9f75b14938d96b111e6513adea75c77d4f4d7ed9260daab02dcae156d4f0
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
OpenStudio AWS Gem Change Log
|
2
2
|
==================================
|
3
3
|
|
4
|
+
Version 0.4.0.alpha3
|
5
|
+
-------------
|
6
|
+
* Load the key name and security groups from the AWS instance information.
|
7
|
+
* Add method on Aws to get the group_uuid
|
8
|
+
|
4
9
|
Version 0.4.0.alpha2
|
5
10
|
-------------
|
6
11
|
* Load worker keys from disk (if they exist) when constructing the OpenStudioAwsWrapper class
|
data/lib/openstudio/aws/aws.rb
CHANGED
@@ -138,7 +138,7 @@ module OpenStudio
|
|
138
138
|
@os_aws.private_key_file_name = options[:private_key_file_name]
|
139
139
|
else
|
140
140
|
# Save the private key if you did not pass in an already existing key_pair_name
|
141
|
-
@os_aws.save_private_key
|
141
|
+
@os_aws.save_private_key @save_directory
|
142
142
|
end
|
143
143
|
|
144
144
|
server_options = {
|
@@ -150,32 +150,16 @@ module OpenStudio
|
|
150
150
|
|
151
151
|
# save the worker pem and public to the directory
|
152
152
|
# presently, this will always overwrite the worker key, is that okay? Is this really needed later?
|
153
|
-
@os_aws.save_worker_keys
|
154
|
-
|
155
|
-
# if instance_data[:ebs_volume_id]
|
156
|
-
# server_options[:ebs_volume_id] = instance_data[:ebs_volume_id]
|
157
|
-
# end
|
153
|
+
@os_aws.save_worker_keys @save_directory
|
158
154
|
|
159
155
|
@os_aws.launch_server(options[:image_id], options[:instance_type], server_options)
|
160
156
|
end
|
161
157
|
|
162
|
-
#
|
158
|
+
# create workers after the server has been created.
|
163
159
|
#
|
164
|
-
# @
|
165
|
-
|
166
|
-
|
167
|
-
puts ''
|
168
|
-
puts 'Server SSH Command:'
|
169
|
-
puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
|
170
|
-
if @os_aws.workers.size > 0
|
171
|
-
puts ''
|
172
|
-
puts 'Worker SSH Command:'
|
173
|
-
@os_aws.workers.each do |worker|
|
174
|
-
puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{worker.data[:dns]}"
|
175
|
-
end
|
176
|
-
end
|
177
|
-
end
|
178
|
-
|
160
|
+
# @param number_of_instances [Integer] Number of worker instances to create
|
161
|
+
# @param options [Hash]
|
162
|
+
# @option options [String] :instance_type Type of server to start (e.g. m3.medium, m3.xlarge, etc.)
|
179
163
|
def create_workers(number_of_instances, options = {}, user_id = 'unknown_user')
|
180
164
|
defaults = {
|
181
165
|
instance_type: 'm2.4xlarge',
|
@@ -226,13 +210,27 @@ module OpenStudio
|
|
226
210
|
@os_aws.configure_server_and_workers
|
227
211
|
end
|
228
212
|
|
213
|
+
# Write out to the terminal the connection information for the servers and workers
|
214
|
+
#
|
215
|
+
# @return [nil] Only prints to the screen. No return is expected
|
216
|
+
def print_connection_info
|
217
|
+
# Print out some debugging commands (probably work on mac/linux only)
|
218
|
+
puts ''
|
219
|
+
puts 'Server SSH Command:'
|
220
|
+
puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{@os_aws.server.data[:dns]}"
|
221
|
+
if @os_aws.workers.size > 0
|
222
|
+
puts ''
|
223
|
+
puts 'Worker SSH Command:'
|
224
|
+
@os_aws.workers.each do |worker|
|
225
|
+
puts "ssh -i #{@os_aws.private_key_file_name} ubuntu@#{worker.data[:dns]}"
|
226
|
+
end
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
229
230
|
# Return information on the cluster instances as a hash. This includes IP addresses, host names, number of processors, etc.
|
230
231
|
# @return [Hash] Data about the configured cluster
|
231
232
|
def cluster_info
|
232
|
-
|
233
|
-
h[:workers] = @os_aws.to_os_worker_hash[:workers]
|
234
|
-
|
235
|
-
h
|
233
|
+
@os_aws.to_os_hash
|
236
234
|
end
|
237
235
|
|
238
236
|
# Save a JSON with information about the cluster that was configured.
|
@@ -416,6 +414,13 @@ module OpenStudio
|
|
416
414
|
image
|
417
415
|
end
|
418
416
|
|
417
|
+
# Return the Group UUID as defined in the AWS wrapper
|
418
|
+
#
|
419
|
+
# @return [String] UUID
|
420
|
+
def group_uuid
|
421
|
+
@os_aws.group_uuid
|
422
|
+
end
|
423
|
+
|
419
424
|
private
|
420
425
|
|
421
426
|
def os_aws_file_location
|
@@ -65,7 +65,7 @@ class OpenStudioAwsWrapper
|
|
65
65
|
work_dir = options[:save_directory] || '.'
|
66
66
|
if File.exist?(File.join(work_dir, 'ec2_worker_key.pem')) && File.exist?(File.join(work_dir, 'ec2_worker_key.pub'))
|
67
67
|
logger.info "Worker keys already exist, loading from #{work_dir}"
|
68
|
-
|
68
|
+
load_worker_key(File.join(work_dir, 'ec2_worker_key.pem'))
|
69
69
|
else
|
70
70
|
logger.info 'Generating new worker keys'
|
71
71
|
@worker_keys = SSHKey.generate
|
@@ -338,13 +338,24 @@ class OpenStudioAwsWrapper
|
|
338
338
|
@private_key = File.read(filename)
|
339
339
|
end
|
340
340
|
|
341
|
+
# Load the worker key for communicating between the server and worker instances on AWS. The public key
|
342
|
+
# will be automatically created when loading the private key
|
343
|
+
#
|
344
|
+
# @param private_key_filename [String] Fully qualified path to the worker private key
|
345
|
+
def load_worker_key(private_key_filename)
|
346
|
+
logger.info "Loading worker keys from #{private_key_filename}"
|
347
|
+
@worker_keys_filename = private_key_filename
|
348
|
+
@worker_keys = SSHKey.new(File.read(@worker_keys_filename))
|
349
|
+
end
|
350
|
+
|
351
|
+
# Save the private key to disk
|
341
352
|
def save_private_key(directory = '.', filename = 'ec2_server_key.pem')
|
342
353
|
if @private_key
|
343
354
|
@private_key_file_name = File.expand_path "#{directory}/#{filename}"
|
344
355
|
logger.info "Saving server private key in #{@private_key_file_name}"
|
345
356
|
File.open(@private_key_file_name, 'w') { |f| f << @private_key }
|
346
357
|
logger.info 'Setting permissions of server private key to 0600'
|
347
|
-
|
358
|
+
File.chmod(0600, @private_key_file_name)
|
348
359
|
else
|
349
360
|
fail "No private key found in which to persist with filename #{filename}"
|
350
361
|
end
|
@@ -352,11 +363,11 @@ class OpenStudioAwsWrapper
|
|
352
363
|
|
353
364
|
# save off the worker public/private keys that were created
|
354
365
|
def save_worker_keys(directory = '.')
|
355
|
-
|
356
|
-
logger.info "Saving worker private key in #{
|
357
|
-
File.open(
|
366
|
+
@worker_keys_filename = "#{directory}/ec2_worker_key.pem"
|
367
|
+
logger.info "Saving worker private key in #{@worker_keys_filename}"
|
368
|
+
File.open(@worker_keys_filename, 'w') { |f| f << @worker_keys.private_key }
|
358
369
|
logger.info 'Setting permissions of worker private key to 0600'
|
359
|
-
File.chmod(0600,
|
370
|
+
File.chmod(0600, @worker_keys_filename)
|
360
371
|
|
361
372
|
wk = "#{directory}/ec2_worker_key.pub"
|
362
373
|
logger.info "Saving worker public key in #{wk}"
|
@@ -403,7 +414,6 @@ class OpenStudioAwsWrapper
|
|
403
414
|
logger.info("worker user_data #{user_data.inspect}")
|
404
415
|
|
405
416
|
# thread the launching of the workers
|
406
|
-
|
407
417
|
num.times do
|
408
418
|
@workers << OpenStudioAwsInstance.new(@aws, :worker, @key_pair_name, @security_groups, @group_uuid,
|
409
419
|
@private_key, @private_key_file_name, @proxy)
|
@@ -458,14 +468,19 @@ class OpenStudioAwsWrapper
|
|
458
468
|
end
|
459
469
|
|
460
470
|
# method to query the amazon api to find the server (if it exists), based on the group id
|
461
|
-
# if it is found, then it will set the @server
|
471
|
+
# if it is found, then it will set the @server instance variable. The security groups are assigned from the
|
472
|
+
# server node information on AWS if the security groups have not been initialized yet.
|
473
|
+
#
|
462
474
|
# Note that the information around keys and security groups is pulled from the instance information.
|
475
|
+
# @param server_data_hash [Hash] Server data
|
476
|
+
# @option server_data_hash [String] :group_id Group ID of the analysis
|
477
|
+
# @option server_data_hash [String] :server.private_key_file_name Name of the private key to communicate to the server
|
463
478
|
def find_server(server_data_hash)
|
464
479
|
@group_uuid = server_data_hash[:group_id] || @group_uuid
|
465
480
|
load_private_key(server_data_hash[:server][:private_key_file_name])
|
466
481
|
|
467
482
|
logger.info "Finding the server for GroupUUID of #{group_uuid}"
|
468
|
-
fail 'no GroupUUID defined either in member variable or method argument' if group_uuid.nil?
|
483
|
+
fail 'no GroupUUID defined either in member variable or method argument' if @group_uuid.nil?
|
469
484
|
|
470
485
|
# This should really just be a single call to describe running instances
|
471
486
|
@server = nil
|
@@ -476,7 +491,17 @@ class OpenStudioAwsWrapper
|
|
476
491
|
if !@server
|
477
492
|
if resp
|
478
493
|
logger.info "Server found and loading data into object [instance id is #{resp[:instance_id]}]"
|
479
|
-
|
494
|
+
|
495
|
+
sg = resp[:security_groups].map { |s| s[:group_id] }
|
496
|
+
# Set the security groups of the object if these groups haven't been assigned yet.
|
497
|
+
@security_groups = sg if @security_groups.empty?
|
498
|
+
logger.info "The security groups in aws wrapper are #{@security_groups}"
|
499
|
+
|
500
|
+
# set the key name from AWS if it isn't yet assigned
|
501
|
+
logger.info 'Setting the keyname in the aws wrapper'
|
502
|
+
@key_pair_name = resp[:key_name] unless @key_pair_name
|
503
|
+
|
504
|
+
@server = OpenStudioAwsInstance.new(@aws, :server, @key_pair_name, sg, @group_uuid, @private_key, @private_key_file_name, @proxy)
|
480
505
|
|
481
506
|
@server.load_instance_data(resp)
|
482
507
|
end
|
@@ -487,12 +512,12 @@ class OpenStudioAwsWrapper
|
|
487
512
|
logger.info 'could not find a running server instance'
|
488
513
|
end
|
489
514
|
|
490
|
-
#
|
515
|
+
# Find the worker instances.
|
491
516
|
if @workers.size == 0
|
492
517
|
resp = describe_running_instances(group_uuid, :worker)
|
493
518
|
if resp
|
494
519
|
resp.each do |r|
|
495
|
-
@workers << OpenStudioAwsInstance.new(@aws, :worker, r[:key_name], r[:security_groups].
|
520
|
+
@workers << OpenStudioAwsInstance.new(@aws, :worker, r[:key_name], r[:security_groups].map { |s| s[:group_id] }, @group_uuid, @private_key, @private_key_file_name, @proxy)
|
496
521
|
@workers.last.load_instance_data(r)
|
497
522
|
end
|
498
523
|
end
|
@@ -500,6 +525,10 @@ class OpenStudioAwsWrapper
|
|
500
525
|
logger.info 'Worker nodes are already defined'
|
501
526
|
end
|
502
527
|
|
528
|
+
# set the private key from the hash
|
529
|
+
load_private_key server_data_hash[:server][:private_key_file_name]
|
530
|
+
load_worker_key server_data_hash[:server][:worker_private_key_file_name]
|
531
|
+
|
503
532
|
# Really don't need to return anything because this sets the class instance variable
|
504
533
|
@server
|
505
534
|
end
|
@@ -558,23 +587,23 @@ class OpenStudioAwsWrapper
|
|
558
587
|
amis
|
559
588
|
end
|
560
589
|
|
561
|
-
|
562
|
-
|
563
|
-
@
|
564
|
-
|
590
|
+
# save off the instance configuration and instance information into a JSON file for later use
|
591
|
+
def to_os_hash
|
592
|
+
h = @server.to_os_hash
|
593
|
+
|
594
|
+
h[:server][:worker_private_key_file_name] = @worker_keys_filename
|
595
|
+
h[:workers] = @workers.map do |worker|
|
596
|
+
{
|
565
597
|
id: worker.data.id,
|
566
598
|
ip: "http://#{worker.data.ip}",
|
567
599
|
dns: worker.data.dns,
|
568
600
|
procs: worker.data.procs,
|
569
601
|
private_key_file_name: worker.private_key_file_name,
|
570
602
|
private_ip_address: worker.private_ip_address
|
571
|
-
|
603
|
+
}
|
572
604
|
end
|
573
605
|
|
574
|
-
|
575
|
-
logger.info out
|
576
|
-
|
577
|
-
out
|
606
|
+
h
|
578
607
|
end
|
579
608
|
|
580
609
|
# take the base version and increment the patch until
|
@@ -290,4 +290,58 @@ describe OpenStudio::Aws::Aws do
|
|
290
290
|
expect(@aws_2.os_aws.worker_keys.public_key).not_to be_nil
|
291
291
|
end
|
292
292
|
end
|
293
|
+
|
294
|
+
context 'stateful creation of server and worker' do
|
295
|
+
before(:all) do
|
296
|
+
@config = OpenStudio::Aws::Config.new
|
297
|
+
@aws = OpenStudio::Aws::Aws.new
|
298
|
+
end
|
299
|
+
|
300
|
+
it 'should create the server and save the state' do
|
301
|
+
options = {
|
302
|
+
instance_type: 'm3.medium',
|
303
|
+
image_id: SERVER_AMI,
|
304
|
+
tags: [
|
305
|
+
'ci_tests=true',
|
306
|
+
'ServerOnly=true'
|
307
|
+
]
|
308
|
+
}
|
309
|
+
|
310
|
+
test_pem_file = 'ec2_server_key.pem'
|
311
|
+
FileUtils.rm_f test_pem_file if File.exist? test_pem_file
|
312
|
+
FileUtils.rm_f 'server_data.json' if File.exist? 'server_data.json'
|
313
|
+
|
314
|
+
@aws.create_server(options)
|
315
|
+
@aws.save_cluster_info 'server_data.json'
|
316
|
+
|
317
|
+
h = @aws.os_aws.server.to_os_hash
|
318
|
+
expect(h[:group_id]).to be_a String
|
319
|
+
expect(h[:group_id]).to match /^[\d\S]{32}$/
|
320
|
+
expect(h[:location]).to eq 'AWS'
|
321
|
+
end
|
322
|
+
|
323
|
+
it 'should load server information from json and launch worker' do
|
324
|
+
aws2 = OpenStudio::Aws::Aws.new
|
325
|
+
aws2.load_instance_info_from_file('server_data.json')
|
326
|
+
|
327
|
+
options = {
|
328
|
+
instance_type: 'm3.medium',
|
329
|
+
image_id: WORKER_AMI
|
330
|
+
}
|
331
|
+
|
332
|
+
aws2.create_workers(1, options)
|
333
|
+
aws2.save_cluster_info 'server_data.json'
|
334
|
+
|
335
|
+
expect(File.exist?('server_data.json')).to eq true
|
336
|
+
# check if file exists
|
337
|
+
|
338
|
+
h = aws2.cluster_info # to make sure that the settings are correct
|
339
|
+
expect(h[:server][:worker_private_key_file_name]).to match /.*ec2_worker_key.pem/
|
340
|
+
expect(h[:workers].size).to eq 1
|
341
|
+
end
|
342
|
+
|
343
|
+
after :all do
|
344
|
+
@aws.terminate
|
345
|
+
end
|
346
|
+
end
|
293
347
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: openstudio-aws
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.0.
|
4
|
+
version: 0.4.0.alpha3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicholas Long
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-07-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: net-scp
|