stemcell 0.12.2 → 0.13.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 42199bc80153e575d437ce9c618b662e6861af3e
4
- data.tar.gz: 53e93477347e08429b03e74b4580d4cc714c424d
2
+ SHA256:
3
+ metadata.gz: 440002a8d4c58fd1329664656e61589e6d174e7330acfa73af957e8d2e690d7c
4
+ data.tar.gz: 88a3fe213bfc823772d72296a130b37fca10032c8519597277d377e1abdfd798
5
5
  SHA512:
6
- metadata.gz: 67d1742a36a3f003a849651d9f7b899414a738ce2ea2a684efaf77bee1399fc65d824c174a28abdf981c2aa739e137d973589d29b2a15ef8e94dfcc94e9f1b3d
7
- data.tar.gz: 0e916fa0c134eb26ff01cba67c6b1e90ae921fa932940f3596a5d09b0607a52abae2b99099d3569e3cf4b67ac4e2a91226666e32af9a250af0a33f081bbd4f71
6
+ metadata.gz: 80f1c8e2e9f3fee310ff779dfcb9b1b4e7118ed27da2045441747b0783060fb9dc39022a6c23740167f76c9d4655999015a6858d68cdc2c906c73bfb2af99d4e
7
+ data.tar.gz: 07ba99d048effdccd90f9a1185449e13e9829cb047d2c16c027c91834accd2610b053b460bc0f5eba8983ca49312a3238d817d9d5946cd81bd57ff4ce021f4b3
data/.travis.yml CHANGED
@@ -5,5 +5,6 @@ rvm:
5
5
  - 2.1.2
6
6
  - 2.2.4
7
7
  - 2.3.1
8
+ - 2.6.8
8
9
  before_install:
9
10
  - gem install bundler -v 1.16.3
data/CHANGELOG.md CHANGED
@@ -1,3 +1,14 @@
1
+ # 0.13.1
2
+ - Support for specifying --min-count and --max-count
3
+
4
+ # 0.13.1
5
+ - Support for specifying --cpu-options
6
+
7
+ # 0.13.0
8
+ - Migrate to AWS SDK to v3
9
+ - Drop support for ClassicLink
10
+ - Removed `necrosis` script
11
+
1
12
  # 0.12.2
2
13
  - Support for using a custom EC2 endpoint
3
14
 
data/README.md CHANGED
@@ -77,10 +77,10 @@ $ stemcell $your_chef_role --tail
77
77
 
78
78
  ### Terminating:
79
79
 
80
- To terminate, use the necrosis command and pass a space separated list of instance ids:
80
+ To terminate, use the AWS CLI and pass a space separated list of instance ids:
81
81
 
82
82
  ```bash
83
- $ necrosis i-12345678 i-12345679 i-12345670
83
+ $ aws ec2 terminate-instances --instance-ids i-12345678 i-12345679 i-12345670
84
84
  ```
85
85
 
86
86
  ## Automation ##
@@ -132,6 +132,7 @@ module Stemcell
132
132
  # comma-separated list when presented as defaults.
133
133
  pd['security_groups'] &&= pd['security_groups'].join(',')
134
134
  pd['tags'] &&= pd['tags'].to_a.map { |p| p.join('=') }.join(',')
135
+ pd['cpu_options'] &&= pd['cpu_options'].to_a.map { |p| p.join('=') }.join(',')
135
136
  pd['chef_cookbook_attributes'] &&= pd['chef_cookbook_attributes'].join(',')
136
137
  end
137
138
 
@@ -1,4 +1,5 @@
1
- require 'aws-sdk-v1'
1
+ require 'aws-sdk-ec2'
2
+ require 'base64'
2
3
  require 'logger'
3
4
  require 'erb'
4
5
  require 'set'
@@ -30,6 +31,7 @@ module Stemcell
30
31
  'chef_environment',
31
32
  'chef_data_bag_secret',
32
33
  'chef_data_bag_secret_path',
34
+ 'cpu_options',
33
35
  'git_branch',
34
36
  'git_key',
35
37
  'git_origin',
@@ -45,10 +47,11 @@ module Stemcell
45
47
  'dedicated_tenancy',
46
48
  'associate_public_ip_address',
47
49
  'count',
50
+ 'min_count',
51
+ 'max_count',
48
52
  'security_groups',
49
53
  'security_group_ids',
50
54
  'tags',
51
- 'classic_link',
52
55
  'iam_role',
53
56
  'ebs_optimized',
54
57
  'termination_protection',
@@ -90,7 +93,7 @@ module Stemcell
90
93
  opts['git_key'] = try_file(opts['git_key'])
91
94
  opts['chef_data_bag_secret'] = try_file(opts['chef_data_bag_secret'])
92
95
 
93
- # generate tags and merge in any that were specefied as inputs
96
+ # generate tags and merge in any that were specified as inputs
94
97
  tags = {
95
98
  'Name' => "#{opts['chef_role']}-#{opts['chef_environment']}",
96
99
  'Group' => "#{opts['chef_role']}-#{opts['chef_environment']}",
@@ -101,36 +104,46 @@ module Stemcell
101
104
  tags['Name'] = opts['chef_role'] if opts['chef_environment'] == 'production'
102
105
  tags.merge!(opts['tags']) if opts['tags']
103
106
 
107
+ # Min/max number of instances to launch
108
+ min_count = opts['min_count'] || opts['count']
109
+ max_count = opts['max_count'] || opts['count']
110
+
104
111
  # generate launch options
105
112
  launch_options = {
106
113
  :image_id => opts['image_id'],
107
114
  :instance_type => opts['instance_type'],
108
115
  :key_name => opts['key_name'],
109
- :count => opts['count'],
116
+ :min_count => min_count,
117
+ :max_count => max_count,
118
+ }
119
+
120
+ # Associate Public IP can only bet set on network_interfaces, and if present
121
+ # security groups and subnet should be set on the interface. VPC-only.
122
+ # Primary network interface
123
+ network_interface = {
124
+ device_index: 0,
110
125
  }
126
+ launch_options[:network_interfaces] = [network_interface]
111
127
 
112
128
  if opts['security_group_ids'] && !opts['security_group_ids'].empty?
113
- launch_options[:security_group_ids] = opts['security_group_ids']
129
+ network_interface[:groups] = opts['security_group_ids']
114
130
  end
115
131
 
116
132
  if opts['security_groups'] && !opts['security_groups'].empty?
117
- if @vpc_id
118
- # convert sg names to sg ids as VPC only accepts ids
119
- security_group_ids = get_vpc_security_group_ids(@vpc_id, opts['security_groups'])
120
- launch_options[:security_group_ids] ||= []
121
- launch_options[:security_group_ids].concat(security_group_ids)
122
- else
123
- launch_options[:security_groups] = opts['security_groups']
124
- end
133
+ # convert sg names to sg ids as VPC only accepts ids
134
+ security_group_ids = get_vpc_security_group_ids(@vpc_id, opts['security_groups'])
135
+ network_interface[:groups] ||= []
136
+ network_interface[:groups].concat(security_group_ids)
125
137
  end
126
138
 
139
+ launch_options[:placement] = placement = {}
127
140
  # specify availability zone (optional)
128
141
  if opts['availability_zone']
129
- launch_options[:availability_zone] = opts['availability_zone']
142
+ placement[:availability_zone] = opts['availability_zone']
130
143
  end
131
144
 
132
145
  if opts['subnet']
133
- launch_options[:subnet] = opts['subnet']
146
+ network_interface[:subnet_id] = opts['subnet']
134
147
  end
135
148
 
136
149
  if opts['private_ip_address']
@@ -138,23 +151,23 @@ module Stemcell
138
151
  end
139
152
 
140
153
  if opts['dedicated_tenancy']
141
- launch_options[:dedicated_tenancy] = opts['dedicated_tenancy']
154
+ placement[:tenancy] = 'dedicated'
142
155
  end
143
156
 
144
157
  if opts['associate_public_ip_address']
145
- launch_options[:associate_public_ip_address] = opts['associate_public_ip_address']
158
+ network_interface[:associate_public_ip_address] = opts['associate_public_ip_address']
146
159
  end
147
160
 
148
161
  # specify IAM role (optional)
149
162
  if opts['iam_role']
150
- launch_options[:iam_instance_profile] = opts['iam_role']
163
+ launch_options[:iam_instance_profile] = {
164
+ name: opts['iam_role']
165
+ }
151
166
  end
152
167
 
153
168
  # specify placement group (optional)
154
169
  if opts['placement_group']
155
- launch_options[:placement] = {
156
- :group_name => opts['placement_group'],
157
- }
170
+ placement[:group_name] = opts['placement_group']
158
171
  end
159
172
 
160
173
  # specify an EBS-optimized instance (optional)
@@ -171,6 +184,11 @@ module Stemcell
171
184
  launch_options[:block_device_mappings] = opts['block_device_mappings']
172
185
  end
173
186
 
187
+ # specify cpu options (optional)
188
+ if opts['cpu_options']
189
+ launch_options[:cpu_options] = opts['cpu_options']
190
+ end
191
+
174
192
  # specify ephemeral block device mappings (optional)
175
193
  if opts['ephemeral_devices']
176
194
  launch_options[:block_device_mappings] ||= []
@@ -182,35 +200,31 @@ module Stemcell
182
200
  end
183
201
  end
184
202
 
203
+ if opts['termination_protection']
204
+ launch_options[:disable_api_termination] = true
205
+ end
206
+
185
207
  # generate user data script to bootstrap instance, include in launch
186
208
  # options UNLESS we have manually set the user-data (ie. for ec2admin)
187
- launch_options[:user_data] = opts.fetch('user_data', render_template(opts))
209
+ launch_options[:user_data] = Base64.encode64(opts.fetch('user_data', render_template(opts)))
210
+
211
+ # add tags to launch options so we don't need to make a separate CreateTags call
212
+ launch_options[:tag_specifications] = [{
213
+ resource_type: 'instance',
214
+ tags: tags.map { |k, v| { key: k, value: v } }
215
+ }]
188
216
 
189
217
  # launch instances
190
218
  instances = do_launch(launch_options)
191
219
 
192
220
  # everything from here on out must succeed, or we kill the instances we just launched
193
221
  begin
194
- # set tags on all instances launched
195
- set_tags(instances, tags)
196
- @log.info "sent ec2 api tag requests successfully"
197
-
198
- # link to classiclink
199
- unless @vpc_id
200
- set_classic_link(instances, opts['classic_link'])
201
- @log.info "successfully applied classic link settings (if any)"
202
- end
203
-
204
- # turn on termination protection
205
- # we do this now to make sure all other settings worked
206
- if opts['termination_protection']
207
- enable_termination_protection(instances)
208
- @log.info "successfully enabled termination protection"
209
- end
210
-
211
222
  # wait for aws to report instance stats
212
223
  if opts.fetch('wait', true)
213
- wait(instances)
224
+ instance_ids = instances.map(&:instance_id)
225
+ @log.info "Waiting up to #{MAX_RUNNING_STATE_WAIT_TIME} seconds for #{instances.count} " \
226
+ "instance(s): (#{instance_ids})"
227
+ instances = wait(instance_ids)
214
228
  print_run_info(instances)
215
229
  @log.info "launched instances successfully"
216
230
  end
@@ -227,19 +241,18 @@ module Stemcell
227
241
  return instances
228
242
  end
229
243
 
230
- def kill(instances, opts={})
231
- return if !instances || instances.empty?
244
+ def kill(instance_ids, opts={})
245
+ return if !instance_ids || instance_ids.empty?
232
246
 
233
- errors = run_batch_operation(instances) do |instance|
234
- begin
235
- @log.warn "Terminating instance #{instance.id}"
236
- instance.terminate
237
- nil # nil == success
238
- rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
239
- opts[:ignore_not_found] ? nil : e
240
- end
241
- end
242
- check_errors(:kill, instances.map(&:id), errors)
247
+ @log.warn "Terminating instances #{instance_ids}"
248
+ ec2.terminate_instances(instance_ids: instance_ids)
249
+ nil # nil == success
250
+ rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
251
+ raise unless opts[:ignore_not_found]
252
+
253
+ invalid_ids = e.message.scan(/i-[a-z0-9]+/)
254
+ instance_ids -= invalid_ids
255
+ retry unless instance_ids.empty? || invalid_ids.empty? # don't retry if we couldn't find any instance ids
243
256
  end
244
257
 
245
258
  # this is made public for ec2admin usage
@@ -249,7 +262,7 @@ module Stemcell
249
262
  erb_template = ERB.new(template_file)
250
263
  last_bootstrap_line = LAST_BOOTSTRAP_LINE
251
264
  generated_template = erb_template.result(binding)
252
- @log.debug "genereated template is #{generated_template}"
265
+ @log.debug "generated template is #{generated_template}"
253
266
  return generated_template
254
267
  end
255
268
 
@@ -266,16 +279,16 @@ module Stemcell
266
279
  puts "install logs will be in /var/log/init and /var/log/init.err"
267
280
  end
268
281
 
269
- def wait(instances)
270
- @log.info "Waiting up to #{MAX_RUNNING_STATE_WAIT_TIME} seconds for #{instances.count} " \
271
- "instance(s): (#{instances.inspect})"
272
-
273
- times_out_at = Time.now + MAX_RUNNING_STATE_WAIT_TIME
274
- until instances.all?{ |i| i.status == :running }
275
- wait_time_expire_or_sleep(times_out_at)
282
+ def wait(instance_ids)
283
+ started_at = Time.now
284
+ result = ec2.wait_until(:instance_running, instance_ids: instance_ids) do |w|
285
+ w.max_attempts = nil
286
+ w.delay = RUNNING_STATE_WAIT_SLEEP_TIME
287
+ w.before_wait do |attempts, response|
288
+ throw :failure if Time.now - started_at > MAX_RUNNING_STATE_WAIT_TIME
289
+ end
276
290
  end
277
-
278
- @log.info "all instances in running state"
291
+ result.map { |page| page.reservations.map(&:instances) }.flatten
279
292
  end
280
293
 
281
294
  def verify_required_options(params, required_options)
@@ -291,35 +304,22 @@ module Stemcell
291
304
  def do_launch(opts={})
292
305
  @log.debug "about to launch instance(s) with options #{opts}"
293
306
  @log.info "launching instances"
294
- instances = ec2.instances.create(opts)
295
- instances = [instances] unless Array === instances
307
+ instances = ec2.run_instances(opts).instances
296
308
  instances.each do |instance|
297
309
  @log.info "launched instance #{instance.instance_id}"
298
310
  end
299
311
  return instances
300
312
  end
301
313
 
302
- def set_tags(instances=[], tags)
303
- @log.info "setting tags on instance(s)"
304
- errors = run_batch_operation(instances) do |instance|
305
- begin
306
- instance.tags.set(tags)
307
- nil # nil == success
308
- rescue AWS::EC2::Errors::InvalidInstanceID::NotFound => e
309
- e
310
- end
311
- end
312
- check_errors(:set_tags, instances.map(&:id), errors)
313
- end
314
-
315
314
  # Resolve security group names to their ids in the given VPC
316
315
  def get_vpc_security_group_ids(vpc_id, group_names)
317
316
  group_map = {}
318
317
  @log.info "resolving security groups #{group_names} in #{vpc_id}"
319
- vpc = AWS::EC2::VPC.new(vpc_id)
320
- vpc.security_groups.each do |sg|
321
- next if sg.vpc_id != vpc_id
322
- group_map[sg.name] = sg.group_id
318
+ ec2.describe_security_groups(filters: [{ name: 'vpc-id', values: [vpc_id] }]).
319
+ each do |response|
320
+ response.security_groups.each do |sg|
321
+ group_map[sg.group_name] = sg.group_id
322
+ end
323
323
  end
324
324
  group_ids = []
325
325
  group_names.each do |sg_name|
@@ -329,135 +329,21 @@ module Stemcell
329
329
  group_ids
330
330
  end
331
331
 
332
- def set_classic_link(left_to_process, classic_link)
333
- return unless classic_link
334
- return unless classic_link['vpc_id']
335
-
336
- security_group_ids = classic_link['security_group_ids'] || []
337
- security_group_names = classic_link['security_groups'] || []
338
- return if security_group_ids.empty? && security_group_names.empty?
339
-
340
- if !security_group_names.empty?
341
- extra_group_ids = get_vpc_security_group_ids(classic_link['vpc_id'], security_group_names)
342
- security_group_ids = security_group_ids + extra_group_ids
343
- end
344
-
345
- @log.info "applying classic link settings on #{left_to_process.count} instance(s)"
346
-
347
- errors = []
348
- processed = []
349
- times_out_at = Time.now + MAX_RUNNING_STATE_WAIT_TIME
350
- until left_to_process.empty?
351
- wait_time_expire_or_sleep(times_out_at)
352
-
353
- # we can only apply classic link when instances are in the running state
354
- # lets apply classiclink as instances become available so we don't wait longer than necessary
355
- recently_running = left_to_process.select{ |inst| inst.status == :running }
356
- left_to_process = left_to_process.reject{ |inst| recently_running.include?(inst) }
357
-
358
- processed += recently_running
359
- errors += run_batch_operation(recently_running) do |instance|
360
- begin
361
- result = ec2.client.attach_classic_link_vpc({
362
- :instance_id => instance.id,
363
- :vpc_id => classic_link['vpc_id'],
364
- :groups => security_group_ids,
365
- })
366
- result.error
367
- rescue StandardError => e
368
- e
369
- end
370
- end
371
- end
372
-
373
- check_errors(:set_classic_link, processed.map(&:id), errors)
374
- end
375
-
376
- def enable_termination_protection(instances)
377
- @log.info "enabling termination protection on instance(s)"
378
- errors = run_batch_operation(instances) do |instance|
379
- begin
380
- resp = ec2.client.modify_instance_attribute({
381
- :instance_id => instance.id,
382
- :disable_api_termination => {
383
- :value => true
384
- }
385
- })
386
- resp.error # returns nil (success) unless there was an error
387
- rescue StandardError => e
388
- e
389
- end
390
- end
391
- check_errors(:enable_termination_protection, instances.map(&:id), errors)
392
- end
393
-
394
332
  # attempt to accept keys as file paths
395
333
  def try_file(opt="")
396
334
  File.read(File.expand_path(opt)) rescue opt
397
335
  end
398
336
 
399
- INITIAL_RETRY_SEC = 1
400
-
401
- # Return a Hash of instance => error. Empty hash indicates "no error"
402
- # for code block:
403
- # - if block returns nil, success
404
- # - if block returns non-nil value (e.g., exception), retry 3 times w/ backoff
405
- # - if block raises exception, fail
406
- def run_batch_operation(instances)
407
- instances.map do |instance|
408
- begin
409
- attempt = 0
410
- result = nil
411
- while attempt < @max_attempts
412
- # sleep idempotently except for the first attempt
413
- sleep(INITIAL_RETRY_SEC * 2 ** attempt) if attempt != 0
414
- result = yield(instance)
415
- break if result.nil? # nil indicates success
416
- attempt += 1
417
- end
418
- result # result for this instance is nil or returned exception
419
- rescue => e
420
- e # result for this instance is caught exception
421
- end
422
- end
423
- end
424
-
425
- def check_errors(operation, instance_ids, errors)
426
- return if errors.all?(&:nil?)
427
- raise IncompleteOperation.new(
428
- operation,
429
- instance_ids,
430
- instance_ids.zip(errors).reject { |i, e| e.nil? }
431
- )
432
- end
433
-
434
337
  def ec2
435
- return @ec2 if @ec2
436
-
437
- if @vpc_id
438
- @ec2 = AWS::EC2::VPC.new(@vpc_id)
439
- else
440
- @ec2 = AWS::EC2.new
441
- end
442
-
443
- @ec2
444
- end
445
-
446
- def wait_time_expire_or_sleep(times_out_at)
447
- now = Time.now
448
- if now >= times_out_at
449
- raise TimeoutError, "exceded timeout of #{MAX_RUNNING_STATE_WAIT_TIME} seconds"
450
- else
451
- sleep [RUNNING_STATE_WAIT_SLEEP_TIME, times_out_at - now].min
338
+ @ec2 ||= begin
339
+ opts = @ec2_endpoint ? { endpoint: @ec2_endpoint } : {}
340
+ Aws::EC2::Client.new(opts)
452
341
  end
453
342
  end
454
343
 
455
344
  def configure_aws_creds_and_region
456
345
  # configure AWS with creds/region
457
346
  aws_configs = {:region => @region}
458
- aws_configs.merge!({
459
- :ec2_endpoint => @ec2_endpoint
460
- }) if @ec2_endpoint
461
347
  aws_configs.merge!({
462
348
  :access_key_id => @aws_access_key,
463
349
  :secret_access_key => @aws_secret_key
@@ -465,7 +351,7 @@ module Stemcell
465
351
  aws_configs.merge!({
466
352
  :session_token => @aws_session_token,
467
353
  }) if @aws_session_token
468
- AWS.config(aws_configs)
354
+ Aws.config.update(aws_configs)
469
355
  end
470
356
  end
471
357
  end
@@ -96,24 +96,6 @@ module Stemcell
96
96
  :type => String,
97
97
  :env => 'VPC_ID'
98
98
  },
99
- {
100
- :name => 'classic_link_vpc_id',
101
- :desc => 'VPC ID to which this instance will be classic-linked',
102
- :type => String,
103
- :env => 'CLASSIC_LINK_VPC_ID',
104
- },
105
- {
106
- :name => 'classic_link_security_group_ids',
107
- :desc => 'comma-separated list of security group IDs to link into ClassicLink; not used unless classic_link_vpc_id is set',
108
- :type => String,
109
- :env => 'CLASSIC_LINK_SECURITY_GROUP_IDS',
110
- },
111
- {
112
- :name => 'classic_link_security_groups',
113
- :desc => 'comma-separated list of security groups to link into ClassicLink; not used unless classic_link_vpc_id is set',
114
- :type => String,
115
- :env => 'CLASSIC_LINK_SECURITY_GROUPS',
116
- },
117
99
  {
118
100
  :name => 'subnet',
119
101
  :desc => "VPC subnet for which to launch this instance",
@@ -205,6 +187,12 @@ module Stemcell
205
187
  :type => String,
206
188
  :env => 'CHEF_DATA_BAG_SECRET_PATH'
207
189
  },
190
+ {
191
+ :name => 'cpu_options',
192
+ :desc => "comma-separated list of cpu option key=value pairs",
193
+ :type => String,
194
+ :env => 'CPU_OPTIONS'
195
+ },
208
196
  {
209
197
  :name => 'chef_role',
210
198
  :desc => "chef role of instance to be launched",
@@ -271,6 +259,18 @@ module Stemcell
271
259
  :type => Integer,
272
260
  :env => 'COUNT'
273
261
  },
262
+ {
263
+ :name => 'min_count',
264
+ :desc => "minimum number of instances to launch",
265
+ :type => Integer,
266
+ :env => 'MIN_COUNT'
267
+ },
268
+ {
269
+ :name => 'max_count',
270
+ :desc => "maximum number of instances to launch",
271
+ :type => Integer,
272
+ :env => 'MAX_COUNT'
273
+ },
274
274
  {
275
275
  :name => 'tail',
276
276
  :desc => "interactively tail the initial converge",
@@ -363,6 +363,16 @@ module Stemcell
363
363
  options['tags'] = tags
364
364
  end
365
365
 
366
+ # convert cpu_options from comma separated string to ruby hash
367
+ if options['cpu_options']
368
+ cpu_options = {}
369
+ options['cpu_options'].split(',').each do |cpu_opt|
370
+ key, value = cpu_opt.split('=')
371
+ cpu_options[key] = value
372
+ end
373
+ options['cpu_options'] = cpu_options
374
+ end
375
+
366
376
  # parse block_device_mappings to convert it from the standard CLI format
367
377
  # to the EC2 Ruby API format.
368
378
  # All of this is a bit hard to find so here are some docs links to
@@ -430,17 +440,6 @@ module Stemcell
430
440
  # convert chef_cookbook_attributes from comma separated string to ruby array
431
441
  options['chef_cookbook_attributes'] &&= options['chef_cookbook_attributes'].split(',')
432
442
 
433
- # format the classic link options
434
- if options['classic_link_vpc_id']
435
- options['classic_link']['vpc_id'] = options['classic_link_vpc_id']
436
- end
437
- if options['classic_link_security_group_ids']
438
- options['classic_link']['security_group_ids'] = options['classic_link_security_group_ids'].split(',')
439
- end
440
- if options['classic_link_security_groups']
441
- options['classic_link']['security_groups'] = options['classic_link_security_groups'].split(',')
442
- end
443
-
444
443
  options
445
444
  end
446
445
 
@@ -1,3 +1,3 @@
1
1
  module Stemcell
2
- VERSION = "0.12.2"
2
+ VERSION = "0.13.2"
3
3
  end
@@ -1,45 +1,56 @@
1
1
  require 'spec_helper'
2
+ require 'base64'
2
3
 
3
- class MockInstance
4
- def initialize(id)
5
- @id = id
6
- end
7
-
8
- def id
9
- @id
10
- end
11
-
12
- def status
13
- :running
14
- end
15
- end
16
-
17
- class MockSecurityGroup
18
- attr_reader :group_id, :name, :vpc_id
19
- def initialize(id, name, vpc_id)
20
- @group_id = id
21
- @name = name
22
- @vpc_id = vpc_id
4
+ describe Stemcell::Launcher do
5
+ before do
6
+ Aws.config[:stub_responses] = true
23
7
  end
24
- end
25
8
 
26
- class MockException < StandardError
27
- end
28
-
29
- describe Stemcell::Launcher do
30
9
  let(:launcher) {
31
10
  opts = {'region' => 'region'}
32
11
  launcher = Stemcell::Launcher.new(opts)
33
12
  launcher
34
13
  }
35
14
  let(:operation) { 'op' }
36
- let(:instances) { (1..4).map { |id| MockInstance.new(id) } }
15
+ let(:instances) do
16
+ ('1'..'4').map do |id|
17
+ Aws::EC2::Types::Instance.new(
18
+ instance_id: id,
19
+ private_ip_address: "10.10.10.#{id}",
20
+ state: Aws::EC2::Types::InstanceState.new(name: 'pending')
21
+ )
22
+ end
23
+ end
37
24
  let(:instance_ids) { instances.map(&:id) }
38
25
 
39
26
  describe '#launch' do
40
- let(:ec2) { instance_double(AWS::EC2) }
41
- let(:client) { double(AWS::EC2::Client) }
42
- let(:response) { instance_double(AWS::Core::Response) }
27
+ let(:ec2) do
28
+ ec2 = Aws::EC2::Client.new
29
+ ec2.stub_responses(
30
+ :describe_security_groups,
31
+ security_groups: [
32
+ {group_id: 'sg-1', group_name: 'sg_name1', vpc_id:'vpc-1'},
33
+ {group_id: 'sg-2', group_name: 'sg_name2', vpc_id:'vpc-1'},
34
+ ],
35
+ )
36
+ ec2.stub_responses(
37
+ :describe_instances,
38
+ reservations: [{
39
+ instances: ('1'..'4').map do |id|
40
+ {
41
+ instance_id: id,
42
+ private_ip_address: "10.10.10.#{id}",
43
+ public_ip_address: "24.10.10.#{id}",
44
+ state: {
45
+ name: 'running'
46
+ }
47
+ }
48
+ end
49
+ }]
50
+ )
51
+ ec2
52
+ end
53
+ let(:response) { instance_double(Seahorse::Client::Response) }
43
54
  let(:launcher) {
44
55
  opts = {'region' => 'region', 'vpc_id' => 'vpc-1'}
45
56
  launcher = Stemcell::Launcher.new(opts)
@@ -59,7 +70,9 @@ describe Stemcell::Launcher do
59
70
  'availability_zone' => 'us-east-1a',
60
71
  'count' => 2,
61
72
  'security_groups' => ['sg_name1', 'sg_name2'],
62
- 'wait' => false
73
+ 'user' => 'some_user',
74
+ 'wait' => true,
75
+ 'cpu_options' => 'core_count=1,threads_per_core=1'
63
76
  }
64
77
  }
65
78
 
@@ -67,146 +80,119 @@ describe Stemcell::Launcher do
67
80
  allow(launcher).to receive(:try_file).and_return('secret')
68
81
  allow(launcher).to receive(:render_template).and_return('template')
69
82
  allow(launcher).to receive(:ec2).and_return(ec2)
70
- allow(ec2).to receive(:client).and_return(client)
71
83
  allow(response).to receive(:error).and_return(nil)
72
84
  end
73
85
 
74
86
  it 'launches all of the instances' do
75
87
  expect(launcher).to receive(:get_vpc_security_group_ids).
76
88
  with('vpc-1', ['sg_name1', 'sg_name2']).and_call_original
77
- expect_any_instance_of(AWS::EC2::VPC).to receive(:security_groups).
78
- and_return([1,2].map { |i| MockSecurityGroup.new("sg-#{i}", "sg_name#{i}", 'vpc-1')})
89
+ expect(ec2).to receive(:describe_security_groups).and_call_original
79
90
  expect(launcher).to receive(:do_launch).with(a_hash_including(
80
91
  :image_id => 'ami-d9d6a6b0',
81
92
  :instance_type => 'c1.xlarge',
82
93
  :key_name => 'key',
83
- :count => 2,
84
- :security_group_ids => ['sg-1', 'sg-2'],
85
- :availability_zone => 'us-east-1a',
86
- :user_data => 'template'
94
+ :min_count => 2,
95
+ :max_count => 2,
96
+ :placement => { :availability_zone => 'us-east-1a' },
97
+ :network_interfaces => [{
98
+ :device_index => 0,
99
+ :groups => ['sg-1', 'sg-2' ]
100
+ }],
101
+ :tag_specifications => [
102
+ {
103
+ :resource_type => 'instance',
104
+ :tags => [
105
+ { :key => "Name", :value => "role-environment" },
106
+ { :key => "Group", :value => "role-environment" },
107
+ { :key => "created_by", :value => "some_user" },
108
+ { :key => "stemcell", :value => Stemcell::VERSION },
109
+ ]},
110
+ ],
111
+ :user_data => Base64.encode64('template'),
112
+ :cpu_options => 'core_count=1,threads_per_core=1'
87
113
  )).and_return(instances)
88
- expect(launcher).to receive(:set_tags).with(kind_of(Array), kind_of(Hash)).and_return(nil)
89
- # set_classic_link should not be set on vpc hosts.
90
- expect(launcher).not_to receive(:set_classic_link)
91
-
92
- launcher.send(:launch, launch_options)
114
+ launched_instances = launcher.send(:launch, launch_options)
115
+ expect(launched_instances.map(&:public_ip_address)).to all(be_truthy)
93
116
  end
94
117
 
95
- it 'calls set_classic_link for non vpc instances' do
96
- launcher = Stemcell::Launcher.new({'region' => 'region', 'vpc_id' => false})
97
- expect(launcher).to receive(:set_classic_link)
98
- expect(launcher).to receive(:set_tags).with(kind_of(Array), kind_of(Hash)).and_return(nil)
99
- expect(launcher).to receive(:do_launch).and_return(instances)
100
- launcher.send(:launch, launch_options)
118
+ it 'launches with min/max count' do
119
+ expect(launcher).to receive(:do_launch).with(a_hash_including(
120
+ :min_count => 3,
121
+ :max_count => 6,
122
+ )).and_return(instances)
123
+ options = launch_options.merge(
124
+ 'min_count' => 3,
125
+ 'max_count' => 6,
126
+ )
127
+ launched_instances = launcher.send(:launch,
128
+ launch_options.merge('min_count' => 3, 'max_count' => 6))
101
129
  end
102
130
  end
103
131
 
104
- describe '#set_classic_link' do
105
- let(:ec2) { instance_double(AWS::EC2) }
106
- let(:client) { double(AWS::EC2::Client) }
107
- let(:response) { instance_double(AWS::Core::Response) }
108
- before do
109
- allow(launcher).to receive(:ec2).and_return(ec2)
110
- allow(ec2).to receive(:client).and_return(client)
111
- allow(response).to receive(:error).and_return(nil)
132
+ describe '#kill' do
133
+ let(:ec2) do
134
+ ec2 = Aws::EC2::Client.new
135
+ ec2.stub_responses(
136
+ :terminate_instances, -> (context) {
137
+ instance_ids = context.params[:instance_ids]
138
+ if instance_ids.include? 'i-3'
139
+ Aws::EC2::Errors::InvalidInstanceIDNotFound.new(nil, "The instance ID 'i-3' do not exist")
140
+ else
141
+ {} # success
142
+ end
143
+ })
144
+ ec2
112
145
  end
113
146
 
114
- let(:classic_link) {
115
- {
116
- 'vpc_id' => 'vpc-1',
117
- 'security_group_ids' => ['sg-1', 'sg-2'],
118
- 'security_groups' => ['sg_name']
119
- }
120
- }
121
-
122
- it 'invokes classic link on all of the instances' do
123
- expect(launcher).to receive(:get_vpc_security_group_ids).with('vpc-1', ['sg_name']).
124
- and_call_original
125
- expect_any_instance_of(AWS::EC2::VPC).to receive(:security_groups).
126
- and_return([MockSecurityGroup.new('sg-3', 'sg_name', 'vpc-1')])
127
- instances.each do |instance|
128
- expect(client).to receive(:attach_classic_link_vpc).ordered.with(a_hash_including(
129
- :instance_id => instance.id,
130
- :vpc_id => classic_link['vpc_id'],
131
- :groups => ['sg-1', 'sg-2', 'sg-3'],
132
- )).and_return(response)
133
- end
147
+ let(:instance_ids) { ('i-1'..'i-4').to_a }
134
148
 
135
- launcher.send(:set_classic_link, instances, classic_link)
136
- end
137
- end
138
-
139
- describe '#run_batch_operation' do
140
- it "raises no exception when no internal error occur" do
141
- errors = launcher.send(:run_batch_operation, instances) {}
142
- expect(errors.all?(&:nil?)).to be true
149
+ before do
150
+ allow(launcher).to receive(:ec2).and_return(ec2)
143
151
  end
144
152
 
145
- it "runs full batch even when there are two error" do
146
- errors = launcher.send(:run_batch_operation,
147
- instances) do |instance, error|
148
- raise "error-#{instance.id}" if instance.id % 2 == 0
153
+ context 'when ignore_not_found is true' do
154
+ it 'terminates valid instances even if an invalid instance id is provided' do
155
+ launcher.kill(instance_ids, ignore_not_found: true)
149
156
  end
150
- expect(errors.count(&:nil?)).to be_eql(2)
151
- expect(errors.reject(&:nil?).map { |e| e.message }).to \
152
- be_eql([2, 4].map { |id| "error-#{id}" })
153
- end
154
157
 
155
- it "retries after an intermittent error" do
156
- count = 0
157
- errors = launcher.send(:run_batch_operation,
158
- instances) do |instance|
159
- if instance.id == 3
160
- count += 1
161
- count < 3 ?
162
- AWS::EC2::Errors::InvalidInstanceID::NotFound.new("error-#{instance.id}"):
163
- nil
164
- end
158
+ it 'finishes without error even if no instance ids are valid' do
159
+ launcher.kill(['i-3'], ignore_not_found: true)
165
160
  end
166
- expect(errors.all?(&:nil?)).to be true
167
161
  end
168
162
 
169
- it "retries up to max_attempts option per instance" do
170
- max_attempts = 6
171
- opts = {'region' => 'region', 'max_attempts' => max_attempts}
172
- launcher = Stemcell::Launcher.new(opts)
173
- allow(launcher).to receive(:sleep).and_return(0)
174
- tags = double("Tags")
175
- instances = (1..2).map do |id|
176
- inst = MockInstance.new(id)
177
- allow(inst).to receive(:tags).and_return(tags)
178
- inst
163
+ context 'when ignore_not_found is false' do
164
+ it 'raises an error' do
165
+ expect { launcher.kill(instance_ids) }.to raise_error(Aws::EC2::Errors::InvalidInstanceIDNotFound)
179
166
  end
180
- expect(tags).to receive(:set).with({'a' => 'b'}).exactly(12).times.
181
- and_raise(AWS::EC2::Errors::InvalidInstanceID::NotFound.new("error"))
182
- expect do
183
- launcher.send(:set_tags, instances, {'a' => 'b'})
184
- end.to raise_error(Stemcell::IncompleteOperation)
185
167
  end
186
168
  end
187
169
 
188
170
  describe '#configure_aws_creds_and_region' do
189
- it 'AWS region is configured after launcher is instanciated' do
190
- expect(AWS.config.region).to be_eql('region')
171
+ it 'AWS region is configured after launcher is instantiated' do
172
+ expect(Aws.config[:region]).to be_eql('region')
191
173
  end
192
174
 
193
175
  it 'AWS region configuration changed' do
194
176
  mock_launcher = Stemcell::Launcher.new('region' => 'ap-northeast-1')
195
- expect(AWS.config.region).to be_eql('ap-northeast-1')
177
+ expect(Aws.config[:region]).to be_eql('ap-northeast-1')
196
178
  end
197
179
  end
198
180
 
199
181
  describe '#ec2' do
182
+
200
183
  it 'can return a client with regional endpoint' do
201
- launcher = Stemcell::Launcher.new({'region' => 'region1', 'ec2_endpoint' => nil})
184
+ launcher = Stemcell::Launcher.new({'region' => 'us-east-1', 'ec2_endpoint' => nil})
202
185
  client = launcher.send(:ec2)
203
- expect(client.config.ec2_endpoint).to be_eql('ec2.region1.amazonaws.com')
186
+ expect(client.config[:endpoint].to_s).to be_eql('https://ec2.us-east-1.amazonaws.com')
204
187
  end
205
188
 
206
189
  it 'can return a client with custom endpoint' do
207
- launcher = Stemcell::Launcher.new({'region' => 'region1', 'ec2_endpoint' => 'endpoint1'})
190
+ launcher = Stemcell::Launcher.new({
191
+ 'region' => 'region1',
192
+ 'ec2_endpoint' => 'https://endpoint1',
193
+ })
208
194
  client = launcher.send(:ec2)
209
- expect(client.config.ec2_endpoint).to be_eql('endpoint1')
195
+ expect(client.config[:endpoint].to_s).to be_eql('https://endpoint1')
210
196
  end
211
197
  end
212
198
  end
data/stemcell.gemspec CHANGED
@@ -17,8 +17,11 @@ Gem::Specification.new do |s|
17
17
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
18
  s.require_paths = ["lib"]
19
19
 
20
- s.add_runtime_dependency 'aws-sdk-v1', '~> 1.63'
21
- s.add_runtime_dependency 'net-ssh', '~> 2.9'
20
+ # pins several aws sdk transitive dependencies to maintain compatibility with Ruby < 2.3
21
+ s.add_runtime_dependency 'aws-eventstream', '~> 1.1.1'
22
+ s.add_runtime_dependency 'aws-sdk-ec2', '~> 1'
23
+ s.add_runtime_dependency 'aws-sigv4', '~> 1.2.4'
24
+ s.add_runtime_dependency 'net-ssh', '~> 2.9'
22
25
  if RUBY_VERSION >= '2.0'
23
26
  s.add_runtime_dependency 'chef', '>= 11.4.0'
24
27
  else
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: stemcell
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.12.2
4
+ version: 0.13.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Martin Rhoads
@@ -11,22 +11,50 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2019-06-18 00:00:00.000000000 Z
14
+ date: 2022-02-23 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
- name: aws-sdk-v1
17
+ name: aws-eventstream
18
18
  requirement: !ruby/object:Gem::Requirement
19
19
  requirements:
20
20
  - - "~>"
21
21
  - !ruby/object:Gem::Version
22
- version: '1.63'
22
+ version: 1.1.1
23
23
  type: :runtime
24
24
  prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
27
  - - "~>"
28
28
  - !ruby/object:Gem::Version
29
- version: '1.63'
29
+ version: 1.1.1
30
+ - !ruby/object:Gem::Dependency
31
+ name: aws-sdk-ec2
32
+ requirement: !ruby/object:Gem::Requirement
33
+ requirements:
34
+ - - "~>"
35
+ - !ruby/object:Gem::Version
36
+ version: '1'
37
+ type: :runtime
38
+ prerelease: false
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - "~>"
42
+ - !ruby/object:Gem::Version
43
+ version: '1'
44
+ - !ruby/object:Gem::Dependency
45
+ name: aws-sigv4
46
+ requirement: !ruby/object:Gem::Requirement
47
+ requirements:
48
+ - - "~>"
49
+ - !ruby/object:Gem::Version
50
+ version: 1.2.4
51
+ type: :runtime
52
+ prerelease: false
53
+ version_requirements: !ruby/object:Gem::Requirement
54
+ requirements:
55
+ - - "~>"
56
+ - !ruby/object:Gem::Version
57
+ version: 1.2.4
30
58
  - !ruby/object:Gem::Dependency
31
59
  name: net-ssh
32
60
  requirement: !ruby/object:Gem::Requirement
@@ -157,7 +185,6 @@ description: A tool for launching and bootstrapping EC2 instances
157
185
  email:
158
186
  - igor.serebryany@airbnb.com
159
187
  executables:
160
- - necrosis
161
188
  - stemcell
162
189
  extensions: []
163
190
  extra_rdoc_files: []
@@ -169,7 +196,6 @@ files:
169
196
  - LICENSE.txt
170
197
  - README.md
171
198
  - Rakefile
172
- - bin/necrosis
173
199
  - bin/stemcell
174
200
  - examples/stemcell.json
175
201
  - examples/stemcellrc
@@ -241,8 +267,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
241
267
  - !ruby/object:Gem::Version
242
268
  version: '0'
243
269
  requirements: []
244
- rubyforge_project:
245
- rubygems_version: 2.5.2.3
270
+ rubygems_version: 3.0.3.1
246
271
  signing_key:
247
272
  specification_version: 4
248
273
  summary: no summary
data/bin/necrosis DELETED
@@ -1,114 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # -*- mode: shell -*-
4
-
5
- require 'aws-sdk-v1'
6
- require 'trollop'
7
-
8
- options = Trollop::options do
9
- version "Necrosis 0.1.0 (c) Airbnb, Inc."
10
- banner "Necrosis: the killing script"
11
-
12
- opt('aws_access_key',
13
- "aws access key",
14
- :type => String,
15
- :default => ENV['AWS_ACCESS_KEY']
16
- )
17
-
18
- opt('aws_secret_key',
19
- "aws secret key",
20
- :type => String,
21
- :default => ENV['AWS_SECRET_KEY']
22
- )
23
-
24
- opt('aws_regions',
25
- "comma-separated list of aws regions to search",
26
- :type => String,
27
- :default => ENV['AWS_REGIONS'],
28
- :short => :r
29
- )
30
-
31
- opt :non_interactive, 'Do not ask confirmation', :short => :f
32
- end
33
-
34
- NON_INTERACTIVE = options[:non_interactive]
35
-
36
- required_parameters = %w(aws_access_key aws_secret_key)
37
-
38
- required_parameters.each do |arg|
39
- raise ArgumentError, "--#{arg.gsub('_','-')} needs to be specified on the commandline or set \
40
- by the #{arg.upcase.gsub('-','_')} environment variable" if
41
- options[arg].nil? or ! options[arg]
42
- end
43
-
44
- raise ArgumentError, "you did not provide any instance ids to kill" if ARGV.empty?
45
-
46
- # a hash from instance_id => [ec2 instance objects]
47
- instances = ARGV.inject({}) { |h, n| h[n] = []; h }
48
-
49
- # convert comma-separated list to Array
50
- regions = (options['aws_regions'] || '').split(',')
51
-
52
- def authorized? instance
53
- return true if NON_INTERACTIVE
54
-
55
- puts 'Terminate? (y/N)'
56
- confirm = $stdin.gets
57
- confirmed = confirm && confirm.chomp.downcase == 'y'
58
-
59
- if confirmed
60
- age = Time.now - instance.launch_time
61
- days = age / (24*3600)
62
- if days > 2
63
- puts "Instance is #{days} days old. REALLY terminate? (y/N)"
64
- confirm = $stdin.gets
65
- confirmed = confirm && confirm.chomp.downcase == 'y'
66
- end
67
- end
68
-
69
- return confirmed
70
- end
71
-
72
- def delete_instance id, instance, region
73
- if instance.api_termination_disabled?
74
- puts "Cannot terminate instance #{id} -- termination protection enabled"
75
- return
76
- end
77
-
78
- puts "Instance #{id} (#{instance.status} in #{region.name})"
79
- puts "\tKey name: #{instance.key_name}"
80
- puts "\tLaunched: #{instance.launch_time}"
81
- instance.tags.to_h.each do |k, v|
82
- puts "\t#{k} : #{v}"
83
- end
84
-
85
- if authorized? instance
86
- instance.terminate
87
- puts "Instance #{id} terminated"
88
- end
89
- end
90
-
91
- AWS.memoize do
92
- ec2 = AWS::EC2.new(:access_key_id => options['aws_access_key'], :secret_access_key => options['aws_secret_key'])
93
- ec2.regions.each do |region|
94
- next unless regions.empty? || regions.include?(region.name)
95
- instances.each do |id, objects|
96
- instance = region.instances[id]
97
- objects << [instance, region] if instance.exists?
98
- end
99
- end
100
-
101
- instances.each do |id, objects|
102
- case objects.count
103
- when 0
104
- STDERR.puts "No instance #{id} found"
105
- next
106
- when 1
107
- instance, region = objects.first
108
- delete_instance id, instance, region
109
- else
110
- puts "Found multiple instances named #{id}"
111
- next
112
- end
113
- end
114
- end