aboisvert_aws 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (78) hide show
  1. data/History.txt +329 -0
  2. data/Manifest.txt +61 -0
  3. data/README.txt +163 -0
  4. data/Rakefile +130 -0
  5. data/lib/acf/right_acf_interface.rb +549 -0
  6. data/lib/acf/right_acf_invalidations.rb +144 -0
  7. data/lib/acf/right_acf_origin_access_identities.rb +230 -0
  8. data/lib/acf/right_acf_streaming_interface.rb +229 -0
  9. data/lib/acw/right_acw_interface.rb +248 -0
  10. data/lib/as/right_as_interface.rb +698 -0
  11. data/lib/awsbase/benchmark_fix.rb +39 -0
  12. data/lib/awsbase/right_awsbase.rb +1343 -0
  13. data/lib/awsbase/support.rb +35 -0
  14. data/lib/awsbase/version.rb +9 -0
  15. data/lib/ec2/right_ec2.rb +541 -0
  16. data/lib/ec2/right_ec2_ebs.rb +481 -0
  17. data/lib/ec2/right_ec2_images.rb +444 -0
  18. data/lib/ec2/right_ec2_instances.rb +788 -0
  19. data/lib/ec2/right_ec2_monitoring.rb +70 -0
  20. data/lib/ec2/right_ec2_placement_groups.rb +108 -0
  21. data/lib/ec2/right_ec2_reserved_instances.rb +184 -0
  22. data/lib/ec2/right_ec2_security_groups.rb +491 -0
  23. data/lib/ec2/right_ec2_spot_instances.rb +422 -0
  24. data/lib/ec2/right_ec2_tags.rb +139 -0
  25. data/lib/ec2/right_ec2_vpc.rb +590 -0
  26. data/lib/ec2/right_ec2_vpc2.rb +381 -0
  27. data/lib/ec2/right_ec2_windows_mobility.rb +84 -0
  28. data/lib/elb/right_elb_interface.rb +573 -0
  29. data/lib/emr/right_emr_interface.rb +727 -0
  30. data/lib/iam/right_iam_access_keys.rb +71 -0
  31. data/lib/iam/right_iam_groups.rb +195 -0
  32. data/lib/iam/right_iam_interface.rb +341 -0
  33. data/lib/iam/right_iam_mfa_devices.rb +67 -0
  34. data/lib/iam/right_iam_users.rb +251 -0
  35. data/lib/rds/right_rds_interface.rb +1384 -0
  36. data/lib/right_aws.rb +86 -0
  37. data/lib/route_53/right_route_53_interface.rb +640 -0
  38. data/lib/s3/right_s3.rb +1138 -0
  39. data/lib/s3/right_s3_interface.rb +1278 -0
  40. data/lib/sdb/active_sdb.rb +1107 -0
  41. data/lib/sdb/right_sdb_interface.rb +762 -0
  42. data/lib/sns/right_sns_interface.rb +286 -0
  43. data/lib/sqs/right_sqs.rb +387 -0
  44. data/lib/sqs/right_sqs_gen2.rb +342 -0
  45. data/lib/sqs/right_sqs_gen2_interface.rb +523 -0
  46. data/lib/sqs/right_sqs_interface.rb +593 -0
  47. data/right_aws.gemspec +90 -0
  48. data/test/README.mdown +39 -0
  49. data/test/acf/test_helper.rb +2 -0
  50. data/test/acf/test_right_acf.rb +138 -0
  51. data/test/awsbase/test_helper.rb +2 -0
  52. data/test/awsbase/test_right_awsbase.rb +11 -0
  53. data/test/ec2/test_helper.rb +2 -0
  54. data/test/ec2/test_right_ec2.rb +107 -0
  55. data/test/elb/test_helper.rb +2 -0
  56. data/test/elb/test_right_elb.rb +43 -0
  57. data/test/http_connection.rb +87 -0
  58. data/test/rds/test_helper.rb +2 -0
  59. data/test/rds/test_right_rds.rb +120 -0
  60. data/test/route_53/fixtures/a_record.xml +18 -0
  61. data/test/route_53/fixtures/alias_record.xml +18 -0
  62. data/test/route_53/test_helper.rb +2 -0
  63. data/test/route_53/test_right_route_53.rb +141 -0
  64. data/test/s3/test_helper.rb +2 -0
  65. data/test/s3/test_right_s3.rb +528 -0
  66. data/test/s3/test_right_s3_stubbed.rb +97 -0
  67. data/test/sdb/test_active_sdb.rb +357 -0
  68. data/test/sdb/test_batch_put_attributes.rb +54 -0
  69. data/test/sdb/test_helper.rb +3 -0
  70. data/test/sdb/test_right_sdb.rb +253 -0
  71. data/test/sns/test_helper.rb +2 -0
  72. data/test/sns/test_right_sns.rb +153 -0
  73. data/test/sqs/test_helper.rb +2 -0
  74. data/test/sqs/test_right_sqs.rb +285 -0
  75. data/test/sqs/test_right_sqs_gen2.rb +264 -0
  76. data/test/test_credentials.rb +37 -0
  77. data/test/ts_right_aws.rb +15 -0
  78. metadata +257 -0
@@ -0,0 +1,727 @@
1
+ #
2
+ # Copyright (c) 2011 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+
24
+ module RightAws
25
+
26
+ # = RightAWS::EmrInterface -- RightScale Amazon Elastic Map Reduce interface
27
+ #
28
+ # The RightAws::EmrInterface class provides a complete interface to Amazon
29
+ # Elastic Map Reduce service.
30
+ #
31
+ # For explanations of the semantics of each call, please refer to Amazon's
32
+ # documentation at
33
+ # http://aws.amazon.com/documentation/elasticmapreduce/
34
+ #
35
+ # Create an interface handle:
36
+ #
37
+ # emr = RightAws::EmrInterface.new(aws_access_key_id, aws_secret_access_key)
38
+ #
39
+ # Create a job flow:
40
+ #
41
+ # emr.run_job_flow(
42
+ # :name => 'job flow 1',
43
+ # :master_instance_type => 'm1.large',
44
+ # :slave_instance_type => 'm1.large',
45
+ # :instance_count => 5,
46
+ # :log_uri => 's3n://bucket/path/to/logs',
47
+ # :steps => [{
48
+ # :name => 'step 1',
49
+ # :jar => 's3n://bucket/path/to/code.jar',
50
+ # :main_class => 'com.foobar.emr.Step1',
51
+ # :args => ['arg', 'arg'],
52
+ # }]) #=> "j-9K18HM82Q0AE7"
53
+ #
54
+ # Describe a job flow:
55
+ #
56
+ # emr.describe_job_flows('j-9K18HM82Q0AE7') #=> {...}
57
+ #
58
+ # Terminate a job flow:
59
+ #
60
+ # emr.terminate_job_flows('j-9K18HM82Q0AE7') #=> true
61
+ #
62
+ class EmrInterface < RightAwsBase
63
+ include RightAwsBaseInterface
64
+
65
+ # Amazon EMR API version being used
66
+ API_VERSION = '2009-03-31'
67
+ DEFAULT_HOST = 'elasticmapreduce.amazonaws.com'
68
+ DEFAULT_PATH = '/'
69
+ DEFAULT_PROTOCOL = 'https'
70
+ DEFAULT_PORT = 443
71
+
72
+ @@bench = AwsBenchmarkingBlock.new
73
+ def self.bench_xml
74
+ @@bench.xml
75
+ end
76
+ def self.bench_service
77
+ @@bench.service
78
+ end
79
+
80
+ # Create a new handle to a EMR service.
81
+ #
82
+ # All handles share the same per process or per thread HTTP connection
83
+ # to EMR. Each handle is for a specific account. The params have
84
+ # the following options:
85
+ #
86
+ # * <tt>:endpoint_url</tt> a fully qualified url to Amazon API endpoint
87
+ # (this overwrites: :server, :port, :service, :protocol). Example:
88
+ # 'https://elasticmapreduce.amazonaws.com'
89
+ # * <tt>:server</tt>: EMR service host, default: DEFAULT_HOST
90
+ # * <tt>:port</tt>: EMR service port, default: DEFAULT_PORT
91
+ # * <tt>:protocol</tt>: 'http' or 'https', default: DEFAULT_PROTOCOL
92
+ # * <tt>:logger</tt>: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT
93
+ #
94
+ # emr = RightAws::EmrInterface.new('xxxxxxxxxxxxxxxxxxxxx','xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
95
+ # {:logger => Logger.new('/tmp/x.log')}) #=> #<RightAws::EmrInterface::0xb7b3c30c>
96
+ #
97
+ def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={})
98
+ init({ :name => 'EMR',
99
+ :default_host => ENV['EMR_URL'] ? URI.parse(ENV['EMR_URL']).host : DEFAULT_HOST,
100
+ :default_port => ENV['EMR_URL'] ? URI.parse(ENV['EMR_URL']).port : DEFAULT_PORT,
101
+ :default_service => ENV['EMR_URL'] ? URI.parse(ENV['EMR_URL']).path : DEFAULT_PATH,
102
+ :default_protocol => ENV['EMR_URL'] ? URI.parse(ENV['EMR_URL']).scheme : DEFAULT_PROTOCOL,
103
+ :default_api_version => ENV['EMR_API_VERSION'] || API_VERSION },
104
+ aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'] ,
105
+ aws_secret_access_key|| ENV['AWS_SECRET_ACCESS_KEY'],
106
+ params)
107
+ end
108
+
109
+ def generate_request(action, params={}) #:nodoc:
110
+ generate_request_impl(:get, action, params )
111
+ end
112
+
113
+ # Sends request to Amazon and parses the response
114
+ # Raises AwsError if any banana happened
115
+ def request_info(request, parser) #:nodoc:
116
+ request_info_impl(:emr_connection, @@bench, request, parser)
117
+ end
118
+
119
+ #-----------------------------------------------------------------
120
+ # Job Flows
121
+ #-----------------------------------------------------------------
122
+
123
+ EMR_INSTANCES_KEY_MAPPING = { # :nodoc:
124
+ :additional_info => 'AdditionalInfo',
125
+ :log_uri => 'LogUri',
126
+ :name => 'Name',
127
+ # JobFlowInstancesConfig
128
+ :ec2_key_name => 'Instances.Ec2KeyName',
129
+ :hadoop_version => 'Instances.HadoopVersion',
130
+ :instance_count => 'Instances.InstanceCount',
131
+ :keep_job_flow_alive_when_no_steps => 'Instances.KeepJobFlowAliveWhenNoSteps',
132
+ :master_instance_type => 'Instances.MasterInstanceType',
133
+ :slave_instance_type => 'Instances.SlaveInstanceType',
134
+ :termination_protected => 'Instances.TerminationProtected',
135
+ # PlacementType
136
+ :availability_zone => 'Instances.Placement.AvailabilityZone',
137
+ }
138
+
139
+ BOOTSTRAP_ACTION_KEY_MAPPING = { # :nodoc:
140
+ :name => 'Name',
141
+ # ScriptBootstrapActionConfig
142
+ :args => 'ScriptBootstrapAction.Args',
143
+ :path => 'ScriptBootstrapAction.Path',
144
+ }
145
+
146
+ INSTANCE_GROUP_KEY_MAPPING = { # :nodoc:
147
+ :bid_price => 'BidPrice',
148
+ :instance_count => 'InstanceCount',
149
+ :instance_role => 'InstanceRole',
150
+ :instance_type => 'InstanceType',
151
+ :market => 'Market',
152
+ :name => 'Name',
153
+ }
154
+
155
+ STEP_CONFIG_KEY_MAPPING = { # :nodoc:
156
+ :action_on_failure => 'ActionOnFailure',
157
+ :name => 'Name',
158
+ # HadoopJarStepConfig
159
+ :args => 'HadoopJarStep.Args',
160
+ :jar => 'HadoopJarStep.Jar',
161
+ :main_class => 'HadoopJarStep.MainClass',
162
+ :properties => 'HadoopJarStep.Properties',
163
+ }
164
+
165
+ KEY_VALUE_KEY_MAPPINGS = {
166
+ :key => 'Key',
167
+ :value => 'Value',
168
+ }
169
+
170
+ # Creates and starts running a new job flow.
171
+ #
172
+ # The job flow will run the steps specified and terminate (unless
173
+ # keep alive option is set).
174
+ #
175
+ # A maximum of 256 steps are allowed in a job flow.
176
+ #
177
+ # At least the name, instance types, instance count and one step
178
+ # must be specified.
179
+ #
180
+ # # simple usage:
181
+ # emr.run_job_flow(
182
+ # :name => 'job flow 1',
183
+ # :master_instance_type => 'm1.large',
184
+ # :slave_instance_type => 'm1.large',
185
+ # :instance_count => 5,
186
+ # :log_uri => 's3n://bucket/path/to/logs',
187
+ # :steps => [{
188
+ # :name => 'step 1',
189
+ # :jar => 's3n://bucket/path/to/code.jar',
190
+ # :main_class => 'com.foobar.emr.Step1',
191
+ # :args => ['arg', 'arg'],
192
+ # }]) #=> "j-9K18HM82Q0AE7"
193
+ #
194
+ # # advanced usage:
195
+ # emr.run_job_flow(
196
+ # :name => 'job flow 1',
197
+ # :ec2_key_name => 'gsg-keypair',
198
+ # :hadoop_version => '0.20',
199
+ # :instance_groups => [{
200
+ # :bid_price => '0.1',
201
+ # :instance_count => '1',
202
+ # :instance_role => 'MASTER',
203
+ # :instance_type => 'm1.small',
204
+ # :market => 'SPOT',
205
+ # :name => 'master group',
206
+ # }, {
207
+ # :bid_price => '0.1',
208
+ # :instance_count => '2',
209
+ # :instance_role => 'CORE',
210
+ # :instance_type => 'm1.small',
211
+ # :market => 'SPOT',
212
+ # :name => 'core group',
213
+ # }, {
214
+ # :bid_price => '0.1',
215
+ # :instance_count => '2',
216
+ # :instance_role => 'TASK',
217
+ # :instance_type => 'm1.small',
218
+ # :market => 'SPOT',
219
+ # :name => 'task group',
220
+ # }],
221
+ # :keep_job_flow_alive_when_no_steps => true,
222
+ # :availability_zone => 'us-east-1a',
223
+ # :termination_protected => true,
224
+ # :log_uri => 's3n://bucket/path/to/logs',
225
+ # :steps => [{
226
+ # :name => 'step 1',
227
+ # :jar => 's3n://bucket/path/to/code.jar',
228
+ # :main_class => 'com.foobar.emr.Step1',
229
+ # :args => ['arg', 'arg'],
230
+ # :properties => {
231
+ # 'property' => 'value',
232
+ # },
233
+ # :action_on_failure => 'TERMINATE_JOB_FLOW',
234
+ # }],
235
+ # :additional_info => '',
236
+ # :bootstrap_actions => [{
237
+ # :name => 'bootstrap action 1',
238
+ # :path => 's3n://bucket/path/to/bootstrap',
239
+ # :args => ['hello', 'world'],
240
+ # }],
241
+ # ) #=> "j-9K18HM82Q0AE7"
242
+ #
243
+ def run_job_flow(options={})
244
+ request_hash = amazonize_run_job_flow(options)
245
+ request_hash.update(amazonize_bootstrap_actions(options[:bootstrap_actions]))
246
+ request_hash.update(amazonize_instance_groups(options[:instance_groups]))
247
+ request_hash.update(amazonize_steps(options[:steps]))
248
+ link = generate_request("RunJobFlow", request_hash)
249
+ request_info(link, RunJobFlowParser.new(:logger => @logger))
250
+ rescue
251
+ on_exception
252
+ end
253
+
254
+ # Returns a list of job flows that match all of supplied parameters.
255
+ #
256
+ # Without parameters, returns job flows started in the last two weeks
257
+ # or running job flows started in the last two months.
258
+ #
259
+ # Regardless of parameters, only jobs started in the last two months
260
+ # are returned.
261
+ #
262
+ # # default list:
263
+ # emr.describe_job_flows #=> [
264
+ # {:keep_job_flow_alive_when_no_steps=>false,
265
+ # :log_uri=>"s3n://bucket/path/to/logs",
266
+ # :master_instance_type=>"m1.small",
267
+ # :availability_zone=>"us-east-1d",
268
+ # :last_state_change_reason=>"Steps completed",
269
+ # :termination_protected=>false,
270
+ # :master_instance_id=>"i-1fe51278",
271
+ # :instance_count=>1,
272
+ # :ready_date_time=>"2011-08-31T18:58:58Z",
273
+ # :bootstrap_actions=>[],
274
+ # :master_public_dns_name=>"ec2-184-78-29-127.compute-1.amazonaws.com",
275
+ # :instance_groups=>
276
+ # [{:instance_request_count=>1,
277
+ # :last_state_change_reason=>"Job flow terminated",
278
+ # :instance_role=>"MASTER",
279
+ # :ready_date_time=>"2011-08-31T18:58:56Z",
280
+ # :instance_running_count=>0,
281
+ # :start_date_time=>"2011-08-31T18:58:19Z",
282
+ # :market=>"ON_DEMAND",
283
+ # :creation_date_time=>"2011-08-31T18:55:36Z",
284
+ # :name=>"master",
285
+ # :instance_group_id=>"ig-1D91GQR7A9H2K",
286
+ # :state=>"ENDED",
287
+ # :instance_type=>"m1.small",
288
+ # :end_date_time=>"2011-08-31T19:01:09Z"}],
289
+ # :start_date_time=>"2011-08-31T18:58:58Z",
290
+ # :steps=>
291
+ # [{:jar=>"s3n://bucket/path/to/code.jar",
292
+ # :main_class=>"com.foobar.emr.Step1",
293
+ # :start_date_time=>"2011-08-31T18:58:58Z",
294
+ # :properties=>{},
295
+ # :args=>[],
296
+ # :creation_date_time=>"2011-08-31T18:55:36Z",
297
+ # :action_on_failure=>"TERMINATE_JOB_FLOW",
298
+ # :name=>"step 1",
299
+ # :state=>"COMPLETED",
300
+ # :end_date_time=>"2011-08-31T19:00:34Z"}],
301
+ # :normalized_instance_hours=>1,
302
+ # :ami_version=>"1.0",
303
+ # :creation_date_time=>"2011-08-31T18:55:36Z",
304
+ # :name=>"jobflow 1",
305
+ # :hadoop_version=>"0.18",
306
+ # :job_flow_id=>"j-9K18HM82Q0AE7",
307
+ # :state=>"COMPLETED",
308
+ # :end_date_time=>"2011-08-31T19:01:09Z"}]
309
+ #
310
+ # # describe specific job flows:
311
+ # emr.describe_job_flows('j-9K18HM82Q0AE7', 'j-2QE0KHA1LP4GS') #=> [...]
312
+ #
313
+ # # specify parameters:
314
+ # emr.describe_job_flows(
315
+ # :created_after => Time.now - 86400,
316
+ # :created_before => Time.now - 3600,
317
+ # :job_flow_ids => ['j-9K18HM82Q0AE7', 'j-2QE0KHA1LP4GS'],
318
+ # :job_flow_states => ['RUNNING']
319
+ # ) #=> [...]
320
+ #
321
+ # # combined job flow list and parameters syntax:
322
+ # emr.describe_job_flows('j-9K18HM82Q0AE7', 'j-2QE0KHA1LP4GS',
323
+ # :job_flow_states => ['RUNNING']
324
+ # ) #=> [...]
325
+ #
326
+ def describe_job_flows(*job_flow_ids_and_options)
327
+ job_flow_ids, options = AwsUtils::split_items_and_params(job_flow_ids_and_options)
328
+ # merge job flow ids passed in as arguments and in options
329
+ unless job_flow_ids.empty?
330
+ # do not modify passed in options
331
+ options = options.dup
332
+ if job_flow_ids_in_options = options[:job_flow_ids]
333
+ # allow the same ids to be passed in either location;
334
+ # remove duplicates
335
+ options[:job_flow_ids] = (job_flow_ids_in_options + job_flow_ids).uniq
336
+ else
337
+ options[:job_flow_ids] = job_flow_ids
338
+ end
339
+ end
340
+ request_hash = {}
341
+ unless (job_flow_ids = options[:job_flow_ids]).right_blank?
342
+ request_hash.update(amazonize_list("JobFlowIds.member", job_flow_ids))
343
+ end
344
+ unless (job_flow_states = options[:job_flow_states]).right_blank?
345
+ request_hash = amazonize_list("JobFlowStates.member", job_flow_states)
346
+ end
347
+ request_hash['CreatedAfter'] = AwsUtils::utc_iso8601(options[:created_after]) unless options[:created_after].right_blank?
348
+ request_hash['CreatedBefore'] = AwsUtils::utc_iso8601(options[:created_before]) unless options[:created_before].right_blank?
349
+ link = generate_request("DescribeJobFlows", request_hash)
350
+ request_cache_or_info(:describe_job_flows, link, DescribeJobFlowsParser, @@bench, nil)
351
+ rescue
352
+ on_exception
353
+ end
354
+
355
+ # Terminates specified job flows.
356
+ #
357
+ # emr.terminate_job_flows('j-9K18HM82Q0AE7') #=> true
358
+ #
359
+ def terminate_job_flows(*job_flow_ids)
360
+ link = generate_request("TerminateJobFlows", amazonize_list('JobFlowIds.member', job_flow_ids))
361
+ request_info(link, RightHttp2xxParser.new(:logger => @logger))
362
+ rescue
363
+ on_exception
364
+ end
365
+
366
+ # Locks a job flow so the EC2 instances in the cluster cannot be
367
+ # terminated by user intervention, an API call, or in the event of a
368
+ # job flow error. Cluster will still terminate upon successful completion
369
+ # of the job flow.
370
+ #
371
+ # emr.set_termination_protection(
372
+ # 'j-9K18HM82Q0AE7', 'j-2QE0KHA1LP4GS', :termination_protected => true
373
+ # ) #=> true
374
+ #
375
+ # Protection can be enabled using the shortcut syntax:
376
+ #
377
+ # emr.set_termination_protection('j-9K18HM82Q0AE7') #=> true
378
+ #
379
+ def set_termination_protection(*job_flow_ids_and_options)
380
+ job_flow_ids, options = AwsUtils::split_items_and_params(job_flow_ids_and_options)
381
+ request_hash = amazonize_list('JobFlowIds.member', job_flow_ids)
382
+ request_hash['TerminationProtected'] = case value = options[:termination_protected]
383
+ when true
384
+ 'true'
385
+ when false
386
+ 'false'
387
+ when nil
388
+ # if :termination_protected => nil was given, then unprotect;
389
+ # if no :termination_protected option was given, protect
390
+ if options.has_key?(:termination_protected)
391
+ 'false'
392
+ else
393
+ 'true'
394
+ end
395
+ else
396
+ # pass value through
397
+ value
398
+ end
399
+ link = generate_request("SetTerminationProtection", request_hash)
400
+ request_info(link, RightHttp2xxParser.new(:logger => @logger))
401
+ rescue
402
+ on_exception
403
+ end
404
+
405
+ #-----------------------------------------------------------------
406
+ # Steps
407
+ #-----------------------------------------------------------------
408
+
409
+ # Adds steps to a running job flow.
410
+ #
411
+ # A maximum of 256 steps are allowed in a job flow. Steps can only be
412
+ # added to job flows that are starting, bootstrapping, running or waiting.
413
+ #
414
+ # Step configuration options are the same as the ones accepted by
415
+ # run_job_flow.
416
+ #
417
+ # emr.add_job_flow_steps('j-2QE0KHA1LP4GS', {
418
+ # :name => 'step 1',
419
+ # :jar => 's3n://bucket/path/to/code.jar',
420
+ # :main_class => 'com.foobar.emr.Step1',
421
+ # :args => ['arg', 'arg'],
422
+ # :properties => {
423
+ # 'property' => 'value',
424
+ # },
425
+ # :action_on_failure => 'TERMINATE_JOB_FLOW',
426
+ # }) #=> true
427
+ #
428
+ def add_job_flow_steps(job_flow_id, *steps)
429
+ request_hash = amazonize_steps(steps)
430
+ request_hash['JobFlowId'] = job_flow_id
431
+ link = generate_request("AddJobFlowSteps", request_hash)
432
+ request_info(link, RightHttp2xxParser.new(:logger => @logger))
433
+ rescue
434
+ on_exception
435
+ end
436
+
437
+ #-----------------------------------------------------------------
438
+ # Instance Groups
439
+ #-----------------------------------------------------------------
440
+
441
+ # Adds instance groups to a running job flow.
442
+ #
443
+ # Instance group configuration options are the same as the ones accepted
444
+ # by run_job_flow.
445
+ #
446
+ # Only task instance groups may be added at runtime.
447
+ # Instance groups cannot be added to job flows that have only a master
448
+ # instance (i.e. 1 instance in total).
449
+ #
450
+ # emr.add_instance_groups('j-2QE0KHA1LP4GS', {
451
+ # :bid_price => '0.1',
452
+ # :instance_count => '2',
453
+ # :instance_role => 'TASK',
454
+ # :instance_type => 'm1.small',
455
+ # :market => 'SPOT',
456
+ # :name => 'core group',
457
+ # }) #=> true
458
+ #
459
+ def add_instance_groups(job_flow_id, *instance_groups)
460
+ request_hash = amazonize_instance_groups(instance_groups, 'InstanceGroups')
461
+ request_hash['JobFlowId'] = job_flow_id
462
+ link = generate_request("AddInstanceGroups", request_hash)
463
+ request_info(link, AddInstanceGroupsParser.new(:logger => @logger))
464
+ rescue
465
+ on_exception
466
+ end
467
+
468
+ MODIFY_INSTANCE_GROUP_KEY_MAPPINGS = {
469
+ :instance_group_id => 'InstanceGroupId',
470
+ :instance_count => 'InstanceCount',
471
+ }
472
+
473
+ # Modifies instance groups.
474
+ #
475
+ # The only modifiable parameter is instance count.
476
+ #
477
+ # An instance group may only be modified when the job flow is running
478
+ # or waiting. Additionally, hadoop 0.20 is required to resize job flows.
479
+ #
480
+ # # general syntax
481
+ # emr.modify_instance_groups(
482
+ # {:instance_group_id => 'ig-P2OPM2L9ZQ4P', :instance_count => 5},
483
+ # {:instance_group_id => 'ig-J82ML0M94A7E', :instance_count => 1}
484
+ # ) #=> true
485
+ #
486
+ # # shortcut syntax
487
+ # emr.modify_instance_groups('ig-P2OPM2L9ZQ4P', 5) #=> true
488
+ #
489
+ # Shortcut syntax supports modifying only one instance group at a time.
490
+ #
491
+ def modify_instance_groups(*args)
492
+ unless args.first.is_a?(Hash)
493
+ if args.length != 2
494
+ raise ArgumentError, "Must be given two arguments if arguments are not hashes"
495
+ end
496
+ args = [{:instance_group_id => args.first, :instance_count => args.last}]
497
+ end
498
+ request_hash = amazonize_list_with_key_mapping('InstanceGroups.member', MODIFY_INSTANCE_GROUP_KEY_MAPPINGS, args)
499
+ link = generate_request("ModifyInstanceGroups", request_hash)
500
+ request_info(link, RightHttp2xxParser.new(:logger => @logger))
501
+ rescue
502
+ on_exception
503
+ end
504
+
505
+ private
506
+
507
+ def amazonize_run_job_flow(options) # :nodoc:
508
+ result = {}
509
+ unless options.right_blank?
510
+ EMR_INSTANCES_KEY_MAPPING.each do |local_name, remote_name|
511
+ value = options[local_name]
512
+ result[remote_name] = value unless value.nil?
513
+ end
514
+ end
515
+ result
516
+ end
517
+
518
+ def amazonize_bootstrap_actions(bootstrap_actions, key = 'BootstrapActions.member') # :nodoc:
519
+ result = {}
520
+ unless bootstrap_actions.right_blank?
521
+ bootstrap_actions.each_with_index do |item, index|
522
+ BOOTSTRAP_ACTION_KEY_MAPPING.each do |local_name, remote_name|
523
+ value = item[local_name]
524
+ case local_name
525
+ when :args
526
+ result.update(amazonize_list("#{key}.#{index+1}.#{remote_name}.member", value))
527
+ else
528
+ next if value.nil?
529
+ result["#{key}.#{index+1}.#{remote_name}"] = value
530
+ end
531
+ end
532
+ end
533
+ end
534
+ result
535
+ end
536
+
537
+ def amazonize_instance_groups(instance_groups, key = 'Instances.InstanceGroups') # :nodoc:
538
+ result = {}
539
+ unless instance_groups.right_blank?
540
+ instance_groups.each_with_index do |item, index|
541
+ INSTANCE_GROUP_KEY_MAPPING.each do |local_name, remote_name|
542
+ value = item[local_name]
543
+ case local_name
544
+ when :instance_groups
545
+ result.update(amazonize_list_with_key_mapping("#{key}.member.#{index+1}.#{remote_name}", INSTANCE_GROUP_KEY_MAPPING, value))
546
+ else
547
+ next if value.nil?
548
+ result["#{key}.member.#{index+1}.#{remote_name}"] = value
549
+ end
550
+ end
551
+ end
552
+ end
553
+ result
554
+ end
555
+
556
+ def amazonize_steps(steps, key = 'Steps.member') # :nodoc:
557
+ result = {}
558
+ unless steps.right_blank?
559
+ steps.each_with_index do |item, index|
560
+ STEP_CONFIG_KEY_MAPPING.each do |local_name, remote_name|
561
+ value = item[local_name]
562
+ case local_name
563
+ when :args
564
+ result.update(amazonize_list("#{key}.#{index+1}.#{remote_name}.member", value))
565
+ when :properties
566
+ next if value.right_blank?
567
+ list = value.inject([]) do |l, (k, v)|
568
+ l << {:key => k, :value => v}
569
+ end
570
+ result.update(amazonize_list_with_key_mapping("#{key}.#{index+1}.#{remote_name}.member", KEY_VALUE_KEY_MAPPINGS, list))
571
+ else
572
+ next if value.nil?
573
+ result["#{key}.#{index+1}.#{remote_name}"] = value
574
+ end
575
+ end
576
+ end
577
+ end
578
+ result
579
+ end
580
+
581
+ #-----------------------------------------------------------------
582
+ # PARSERS: Run Job Flow
583
+ #-----------------------------------------------------------------
584
+
585
+ class RunJobFlowParser < RightAWSParser #:nodoc:
586
+ def tagend(name)
587
+ case name
588
+ when 'JobFlowId' then @result = @text
589
+ end
590
+ end
591
+ def reset
592
+ @result = nil
593
+ end
594
+ end
595
+
596
+ #-----------------------------------------------------------------
597
+ # PARSERS: Describe Job Flows
598
+ #-----------------------------------------------------------------
599
+
600
+ class DescribeJobFlowsParser < RightAWSParser #:nodoc:
601
+ def tagstart(name, attributes)
602
+ case full_tag_name
603
+ when %r{/JobFlows/member$}
604
+ @item = { :instance_groups => [],
605
+ :steps => [],
606
+ :bootstrap_actions => [] }
607
+ when %r{/BootstrapActionConfig$}
608
+ @bootstrap_action = {}
609
+ when %r{/InstanceGroups/member$}
610
+ @instance_group = {}
611
+ when %r{/Steps/member$}
612
+ @step = { :args => [],
613
+ :properties => {} }
614
+ end
615
+ end
616
+ def tagend(name)
617
+ case full_tag_name
618
+ when %r{/BootstrapActionConfig} # no trailing $
619
+ case name
620
+ when 'Name'
621
+ @bootstrap_action[:name] = @text
622
+ when 'ScriptBootstrapAction'
623
+ @bootstrap_action[:script_bootstrap_action] = @text
624
+ when 'BootstrapActionConfig'
625
+ @step[:bootstrap_actions] << @bootstrap_action
626
+ end
627
+ when %r{/InstanceGroups/member} # no trailing $
628
+ case name
629
+ when 'BidPrice' then @instance_group[:bid_price] = @text
630
+ when 'CreationDateTime' then @instance_group[:creation_date_time] = @text
631
+ when 'EndDateTime' then @instance_group[:end_date_time] = @text
632
+ when 'InstanceGroupId' then @instance_group[:instance_group_id] = @text
633
+ when 'InstanceRequestCount' then @instance_group[:instance_request_count] = @text.to_i
634
+ when 'InstanceRole' then @instance_group[:instance_role] = @text
635
+ when 'InstanceRunningCount' then @instance_group[:instance_running_count] = @text.to_i
636
+ when 'InstanceType' then @instance_group[:instance_type] = @text
637
+ when 'LastStateChangeReason' then @instance_group[:last_state_change_reason] = @text
638
+ when 'Market' then @instance_group[:market] = @text
639
+ when 'Name' then @instance_group[:name] = @text
640
+ when 'ReadyDateTime' then @instance_group[:ready_date_time] = @text
641
+ when 'StartDateTime' then @instance_group[:start_date_time] = @text
642
+ when 'State' then @instance_group[:state] = @text
643
+ when 'member' then @item[:instance_groups] << @instance_group
644
+ end
645
+ when %r{/Steps/member/StepConfig/HadoopJarStep/Args/member}
646
+ @step[:args] << @text
647
+ when %r{/Steps/member/StepConfig/HadoopJarStep/Properties$}
648
+ @step[:properties][@key] = @value
649
+ when %r{/Steps/member/StepConfig/HadoopJarStep/Properties}
650
+ case name
651
+ when 'Key'
652
+ @key = @text
653
+ when 'Value'
654
+ @value = @text
655
+ end
656
+ when %r{/Steps/member$}
657
+ @item[:steps] << @step
658
+ when %r{/Steps/member} # no trailing $
659
+ case name
660
+ # ExecutionStatusDetail
661
+ when 'CreationDateTime' then @step[:creation_date_time] = @text
662
+ when 'EndDateTime' then @step[:end_date_time] = @text
663
+ when 'LastStateChangeReason' then @step[:last_state_change_reason] = @text
664
+ when 'StartDateTime' then @step[:start_date_time] = @text
665
+ when 'State' then @step[:state] = @text
666
+ # StepConfig
667
+ when 'ActionOnFailure' then @step[:action_on_failure] = @text
668
+ when 'Name' then @step[:name] = @text
669
+ # HadoopJarStepConfig
670
+ when 'Jar' then @step[:jar] = @text
671
+ when 'MainClass' then @step[:main_class] = @text
672
+ end
673
+ when %r{/JobFlows/member$}
674
+ @result << @item
675
+ else
676
+ case name
677
+ when 'AmiVersion' then @item[:ami_version] = @text
678
+ when 'JobFlowId' then @item[:job_flow_id] = @text
679
+ when 'LogUri' then @item[:log_uri] = @text
680
+ when 'Name' then @item[:name] = @text
681
+
682
+ # JobFlowExecutionStatusDetail
683
+ when 'CreationDateTime' then @item[:creation_date_time] = @text
684
+ when 'EndDateTime' then @item[:end_date_time] = @text
685
+ when 'LastStateChangeReason' then @item[:last_state_change_reason] = @text
686
+ when 'ReadyDateTime' then @item[:ready_date_time] = @text
687
+ when 'StartDateTime' then @item[:start_date_time] = @text
688
+ when 'State' then @item[:state] = @text
689
+
690
+ # JobFlowInstancesDetail
691
+ when 'Ec2KeyName' then @item[:ec2_key_name] = @text
692
+ when 'HadoopVersion' then @item[:hadoop_version] = @text
693
+ when 'InstanceCount' then @item[:instance_count] = @text.to_i
694
+ when 'KeepJobFlowAliveWhenNoSteps' then @item[:keep_job_flow_alive_when_no_steps] = case @text when 'true' then true when 'false' then false else @text end
695
+ when 'MasterInstanceId' then @item[:master_instance_id] = @text
696
+ when 'MasterInstanceType' then @item[:master_instance_type] = @text
697
+ when 'MasterPublicDnsName' then @item[:master_public_dns_name] = @text
698
+ when 'NormalizedInstanceHours' then @item[:normalized_instance_hours] = @text.to_i
699
+ # Placement
700
+ when 'AvailabilityZone' then @item[:availability_zone] = @text
701
+ when 'SlaveInstanceType' then @item[:slave_instance_type] = @text
702
+ when 'TerminationProtected' then @item[:termination_protected] = case @text when 'true' then true when 'false' then false else @text end
703
+ end
704
+ end
705
+ end
706
+ def reset
707
+ @result = []
708
+ end
709
+ end
710
+
711
+ #-----------------------------------------------------------------
712
+ # PARSERS: Add Instance Groups
713
+ #-----------------------------------------------------------------
714
+
715
+ class AddInstanceGroupsParser < RightAWSParser #:nodoc:
716
+ def tagend(name)
717
+ case name
718
+ when 'InstanceGroupIds' then @result << @text.strip
719
+ end
720
+ end
721
+ def reset
722
+ @result = []
723
+ end
724
+ end
725
+ end
726
+
727
+ end