elasticity 5.0.3 → 6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.md +26 -0
- data/README.md +35 -28
- data/elasticity.gemspec +2 -2
- data/lib/elasticity.rb +5 -3
- data/lib/elasticity/aws_request_v4.rb +15 -3
- data/lib/elasticity/aws_session.rb +4 -23
- data/lib/elasticity/aws_utils.rb +0 -29
- data/lib/elasticity/cluster_status.rb +38 -0
- data/lib/elasticity/cluster_step_status.rb +51 -0
- data/lib/elasticity/emr.rb +208 -78
- data/lib/elasticity/job_flow.rb +16 -17
- data/lib/elasticity/version.rb +1 -1
- data/spec/factories/cluster_status_factory.rb +12 -0
- data/spec/factories/cluster_step_status_factory.rb +17 -0
- data/spec/lib/elasticity/aws_request_v4_spec.rb +54 -4
- data/spec/lib/elasticity/aws_session_spec.rb +22 -88
- data/spec/lib/elasticity/aws_utils_spec.rb +0 -46
- data/spec/lib/elasticity/bootstrap_action_spec.rb +7 -3
- data/spec/lib/elasticity/cluster_status_spec.rb +98 -0
- data/spec/lib/elasticity/cluster_step_status_spec.rb +80 -0
- data/spec/lib/elasticity/custom_jar_step_spec.rb +10 -7
- data/spec/lib/elasticity/emr_spec.rb +422 -132
- data/spec/lib/elasticity/ganglia_bootstrap_action_spec.rb +8 -3
- data/spec/lib/elasticity/hadoop_bootstrap_action_spec.rb +8 -3
- data/spec/lib/elasticity/hadoop_file_bootstrap_action_spec.rb +7 -3
- data/spec/lib/elasticity/hive_step_spec.rb +21 -17
- data/spec/lib/elasticity/instance_group_spec.rb +9 -5
- data/spec/lib/elasticity/job_flow_integration_spec.rb +4 -4
- data/spec/lib/elasticity/job_flow_spec.rb +102 -76
- data/spec/lib/elasticity/job_flow_step_spec.rb +1 -1
- data/spec/lib/elasticity/looper_spec.rb +1 -1
- data/spec/lib/elasticity/pig_step_spec.rb +13 -9
- data/spec/lib/elasticity/s3distcp_step_spec.rb +7 -5
- data/spec/lib/elasticity/script_step_spec.rb +11 -6
- data/spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb +9 -5
- data/spec/lib/elasticity/streaming_step_spec.rb +13 -9
- data/spec/spec_helper.rb +8 -0
- data/spec/support/factory_girl.rb +8 -0
- metadata +24 -21
- data/lib/elasticity/aws_request_v2.rb +0 -42
- data/lib/elasticity/job_flow_status.rb +0 -91
- data/lib/elasticity/job_flow_status_step.rb +0 -38
- data/spec/lib/elasticity/aws_request_v2_spec.rb +0 -38
- data/spec/lib/elasticity/job_flow_status_spec.rb +0 -265
- data/spec/lib/elasticity/job_flow_status_step_spec.rb +0 -80
data/lib/elasticity/emr.rb
CHANGED
@@ -4,49 +4,8 @@ module Elasticity
|
|
4
4
|
|
5
5
|
attr_reader :aws_request
|
6
6
|
|
7
|
-
def initialize(
|
8
|
-
@aws_request = Elasticity::AwsSession.new(
|
9
|
-
end
|
10
|
-
|
11
|
-
# Describe a specific jobflow.
|
12
|
-
#
|
13
|
-
# describe_jobflow("j-3UN6WX5RRO2AG")
|
14
|
-
#
|
15
|
-
# Raises ArgumentError if the specified jobflow does not exist.
|
16
|
-
def describe_jobflow(jobflow_id)
|
17
|
-
aws_result = @aws_request.submit({
|
18
|
-
:operation => 'DescribeJobFlows',
|
19
|
-
:job_flow_ids => [jobflow_id]
|
20
|
-
})
|
21
|
-
xml_doc = Nokogiri::XML(aws_result)
|
22
|
-
xml_doc.remove_namespaces!
|
23
|
-
yield aws_result if block_given?
|
24
|
-
JobFlowStatus.from_members_nodeset(xml_doc.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member')).first
|
25
|
-
end
|
26
|
-
|
27
|
-
# This is primarily for debugging purposes, providing insight into how
|
28
|
-
# Amazon internally represents jobs. It's used to reverse-engineer
|
29
|
-
# how API calls construct jobflows.
|
30
|
-
def describe_jobflow_xml(jobflow_id)
|
31
|
-
describe_jobflow(jobflow_id) do |xml|
|
32
|
-
return xml
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
# Lists all jobflows in all states.
|
37
|
-
#
|
38
|
-
# To override this behaviour, pass additional filters as specified in the AWS
|
39
|
-
# documentation - http://docs.amazonwebservices.com/ElasticMapReduce/latest/API/index.html?API_DescribeJobFlows.html.
|
40
|
-
#
|
41
|
-
# describe_jobflows(:CreatedBefore => "2011-10-04")
|
42
|
-
def describe_jobflows(params = {})
|
43
|
-
aws_result = @aws_request.submit(
|
44
|
-
params.merge({:operation => 'DescribeJobFlows'})
|
45
|
-
)
|
46
|
-
xml_doc = Nokogiri::XML(aws_result)
|
47
|
-
xml_doc.remove_namespaces!
|
48
|
-
yield aws_result if block_given?
|
49
|
-
JobFlowStatus.from_members_nodeset(xml_doc.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member'))
|
7
|
+
def initialize(options = {})
|
8
|
+
@aws_request = Elasticity::AwsSession.new(options)
|
50
9
|
end
|
51
10
|
|
52
11
|
# Adds a new group of instances to the specified jobflow. Elasticity maps a
|
@@ -56,11 +15,11 @@ module Elasticity
|
|
56
15
|
#
|
57
16
|
# instance_group_config = {
|
58
17
|
# :bid_price => 5,
|
18
|
+
# :market => "SPOT",
|
19
|
+
# :name => "Go Canucks Go!",
|
59
20
|
# :instance_count => 1,
|
60
21
|
# :instance_role => "TASK",
|
61
|
-
# :
|
62
|
-
# :name => "Go Canucks Go!"
|
63
|
-
# :type => "m1.small",
|
22
|
+
# :instance_type => "m1.small"
|
64
23
|
# }
|
65
24
|
#
|
66
25
|
# add_instance_groups takes an array of {}. Returns an array of the instance IDs
|
@@ -74,42 +33,179 @@ module Elasticity
|
|
74
33
|
:instance_groups => instance_group_configs
|
75
34
|
}
|
76
35
|
aws_result = @aws_request.submit(params)
|
77
|
-
xml_doc = Nokogiri::XML(aws_result)
|
78
|
-
xml_doc.remove_namespaces!
|
79
|
-
instance_group_ids = []
|
80
|
-
xml_doc.xpath('/AddInstanceGroupsResponse/AddInstanceGroupsResult/InstanceGroupIds/member').each do |member|
|
81
|
-
instance_group_ids << member.text
|
82
|
-
end
|
83
36
|
yield aws_result if block_given?
|
84
|
-
|
37
|
+
JSON.parse(aws_result)['InstanceGroupIds']
|
85
38
|
end
|
86
39
|
|
87
40
|
# Add a step (or steps) to the specified job flow.
|
88
41
|
#
|
89
|
-
# emr.add_jobflow_step("j-123",
|
90
|
-
#
|
91
|
-
#
|
92
|
-
#
|
93
|
-
# :
|
94
|
-
#
|
95
|
-
# "
|
96
|
-
#
|
97
|
-
#
|
98
|
-
#
|
99
|
-
#
|
100
|
-
#
|
101
|
-
#
|
102
|
-
#
|
103
|
-
#
|
104
|
-
# ]
|
105
|
-
# })
|
42
|
+
# emr.add_jobflow_step("j-123", [
|
43
|
+
# {
|
44
|
+
# :action_on_failure => "TERMINATE_JOB_FLOW",
|
45
|
+
# :hadoop_jar_step => {
|
46
|
+
# :args => [
|
47
|
+
# "s3://elasticmapreduce/libs/pig/pig-script",
|
48
|
+
# "--base-path",
|
49
|
+
# "s3://elasticmapreduce/libs/pig/",
|
50
|
+
# "--install-pig"
|
51
|
+
# ],
|
52
|
+
# :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar"
|
53
|
+
# },
|
54
|
+
# :name => "Setup Pig"
|
55
|
+
# }
|
56
|
+
# ])
|
106
57
|
def add_jobflow_steps(jobflow_id, steps_config)
|
107
58
|
params = {
|
108
59
|
:operation => 'AddJobFlowSteps',
|
109
|
-
:job_flow_id => jobflow_id
|
110
|
-
|
60
|
+
:job_flow_id => jobflow_id,
|
61
|
+
:steps => steps_config
|
62
|
+
}
|
63
|
+
aws_result = @aws_request.submit(params)
|
64
|
+
yield aws_result if block_given?
|
65
|
+
JSON.parse(aws_result)
|
66
|
+
end
|
67
|
+
|
68
|
+
# Sets the specified tags on all instances in the specified jobflow
|
69
|
+
#
|
70
|
+
# emr.add_tags('j-123', [{:key => 'key1', :value => 'value1'}, {:key => 'key_only2'}])
|
71
|
+
#
|
72
|
+
# See http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_AddTags.html
|
73
|
+
def add_tags(jobflow_id, tags)
|
74
|
+
params = {
|
75
|
+
:operation => 'AddTags',
|
76
|
+
:resource_id => jobflow_id,
|
77
|
+
:tags => tags
|
78
|
+
}
|
79
|
+
aws_result = @aws_request.submit(params)
|
80
|
+
yield aws_result if block_given?
|
81
|
+
end
|
82
|
+
|
83
|
+
# Provides details about the specified jobflow
|
84
|
+
#
|
85
|
+
# emr.describe_cluster('j-123')
|
86
|
+
#
|
87
|
+
# http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_DescribeCluster.html
|
88
|
+
def describe_cluster(jobflow_id)
|
89
|
+
params = {
|
90
|
+
:operation => 'DescribeCluster',
|
91
|
+
:cluster_id => jobflow_id,
|
92
|
+
}
|
93
|
+
aws_result = @aws_request.submit(params)
|
94
|
+
yield aws_result if block_given?
|
95
|
+
JSON.parse(aws_result)
|
96
|
+
end
|
97
|
+
|
98
|
+
# Provides details about the specified step within an existing jobflow
|
99
|
+
#
|
100
|
+
# emr.describe_step('j-123', 'step-456')
|
101
|
+
#
|
102
|
+
# http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_DescribeStep.html
|
103
|
+
def describe_step(jobflow_id, step_id)
|
104
|
+
params = {
|
105
|
+
:operation => 'DescribeStep',
|
106
|
+
:cluster_id => jobflow_id,
|
107
|
+
:step_id => step_id
|
108
|
+
}
|
109
|
+
aws_result = @aws_request.submit(params)
|
110
|
+
yield aws_result if block_given?
|
111
|
+
JSON.parse(aws_result)
|
112
|
+
end
|
113
|
+
|
114
|
+
# List the clusters given specified filtering
|
115
|
+
#
|
116
|
+
# emr.list_clusters({
|
117
|
+
# :states => ['status1', 'status2', ...],
|
118
|
+
# :created_before => Time, # Amazon times are in UTC
|
119
|
+
# :created_after => Time, # Amazon times are in UTC
|
120
|
+
# :marker => 'marker' # Retrieve from a prior call to list_clusters
|
121
|
+
# })
|
122
|
+
#
|
123
|
+
# http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListClusters.html
|
124
|
+
def list_clusters(options={})
|
125
|
+
params = {
|
126
|
+
:operation => 'ListClusters'
|
127
|
+
}
|
128
|
+
params.merge!(:cluster_states => options[:states]) if options[:states]
|
129
|
+
params.merge!(:created_before => options[:created_before].to_i) if options[:created_before]
|
130
|
+
params.merge!(:created_after => options[:created_after].to_i) if options[:created_after]
|
131
|
+
params.merge!(:marker => options[:marker]) if options[:marker]
|
132
|
+
aws_result = @aws_request.submit(params)
|
133
|
+
yield aws_result if block_given?
|
134
|
+
JSON.parse(aws_result)
|
135
|
+
end
|
136
|
+
|
137
|
+
# List the instances in a cluster given specified filtering
|
138
|
+
#
|
139
|
+
# emr.list_instances('j-123', {
|
140
|
+
# :types => ['MASTER', 'CORE', 'TASK'],
|
141
|
+
# :marker => 'marker' # Retrieve from a prior call to list_clusters
|
142
|
+
# })
|
143
|
+
#
|
144
|
+
# http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListInstances.html
|
145
|
+
def list_instances(jobflow_id, options={})
|
146
|
+
params = {
|
147
|
+
:operation => 'ListInstances',
|
148
|
+
:cluster_id => jobflow_id
|
149
|
+
}
|
150
|
+
params.merge!(:instance_group_id => options[:instance_group_id]) if options[:instance_group_id]
|
151
|
+
params.merge!(:instance_group_types => options[:instance_group_types]) if options[:instance_group_types]
|
152
|
+
params.merge!(:marker => options[:marker]) if options[:marker]
|
153
|
+
aws_result = @aws_request.submit(params)
|
154
|
+
yield aws_result if block_given?
|
155
|
+
JSON.parse(aws_result)
|
156
|
+
end
|
157
|
+
|
158
|
+
# List the instance groups in the specified jobflow
|
159
|
+
#
|
160
|
+
# emr.list_instance_groups('j-123')
|
161
|
+
#
|
162
|
+
# http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListInstanceGroups.html
|
163
|
+
def list_instance_groups(jobflow_id)
|
164
|
+
params = {
|
165
|
+
:operation => 'ListInstanceGroups',
|
166
|
+
:cluster_id => jobflow_id,
|
167
|
+
}
|
168
|
+
aws_result = @aws_request.submit(params)
|
169
|
+
yield aws_result if block_given?
|
170
|
+
JSON.parse(aws_result)
|
171
|
+
end
|
172
|
+
|
173
|
+
# List the bootstrap actions in the specified jobflow
|
174
|
+
#
|
175
|
+
# emr.list_bootstrap_actions('j-123')
|
176
|
+
#
|
177
|
+
# http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListBootstrapActions.html
|
178
|
+
def list_bootstrap_actions(jobflow_id)
|
179
|
+
params = {
|
180
|
+
:operation => 'ListBootstrapActions',
|
181
|
+
:cluster_id => jobflow_id,
|
182
|
+
}
|
183
|
+
aws_result = @aws_request.submit(params)
|
184
|
+
yield aws_result if block_given?
|
185
|
+
JSON.parse(aws_result)
|
186
|
+
end
|
187
|
+
|
188
|
+
# List the steps in a job flow given specified filtering
|
189
|
+
#
|
190
|
+
# emr.list_steps('j-123', {
|
191
|
+
# :types => ['MASTER', 'CORE', 'TASK'],
|
192
|
+
# :step_ids => ['ID-1', 'ID-2']
|
193
|
+
# :step_states => ['PENDING', 'RUNNING', ...]
|
194
|
+
# :marker => 'marker' # Retrieve from a prior call to list_steps
|
195
|
+
# })
|
196
|
+
#
|
197
|
+
# http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListSteps.html
|
198
|
+
def list_steps(jobflow_id, options={})
|
199
|
+
params = {
|
200
|
+
:operation => 'ListSteps',
|
201
|
+
:cluster_id => jobflow_id,
|
202
|
+
}
|
203
|
+
params.merge!(:step_ids => options[:step_ids]) if options[:step_ids]
|
204
|
+
params.merge!(:step_states => options[:step_states]) if options[:step_states]
|
205
|
+
params.merge!(:marker => options[:marker]) if options[:marker]
|
111
206
|
aws_result = @aws_request.submit(params)
|
112
207
|
yield aws_result if block_given?
|
208
|
+
JSON.parse(aws_result)
|
113
209
|
end
|
114
210
|
|
115
211
|
# Set the number of instances in the specified instance groups to the
|
@@ -129,6 +225,21 @@ module Elasticity
|
|
129
225
|
yield aws_result if block_given?
|
130
226
|
end
|
131
227
|
|
228
|
+
# Remove the specified tags on all instances in the specified jobflow
|
229
|
+
#
|
230
|
+
# emr.remove_tags('j-123', ['key1','key_only2'])
|
231
|
+
#
|
232
|
+
# See http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_RemoveTags.html
|
233
|
+
def remove_tags(jobflow_id, keys)
|
234
|
+
params = {
|
235
|
+
:operation => 'RemoveTags',
|
236
|
+
:resource_id => jobflow_id,
|
237
|
+
:tag_keys => keys
|
238
|
+
}
|
239
|
+
aws_result = @aws_request.submit(params)
|
240
|
+
yield aws_result if block_given?
|
241
|
+
end
|
242
|
+
|
132
243
|
# Start a job flow with the specified configuration. This is a very thin
|
133
244
|
# wrapper around the AWS API, so in order to use it directly you'll need
|
134
245
|
# to have the PDF API reference handy, which can be found here:
|
@@ -211,14 +322,33 @@ module Elasticity
|
|
211
322
|
yield aws_result if block_given?
|
212
323
|
end
|
213
324
|
|
214
|
-
#
|
215
|
-
#
|
216
|
-
#
|
217
|
-
#
|
218
|
-
|
325
|
+
# Whether or not all IAM users in this account can access the job flows.
|
326
|
+
#
|
327
|
+
# Takes an [] of job flow IDs.
|
328
|
+
#
|
329
|
+
# ["j-1B4D1XP0C0A35", "j-1YG2MYL0HVYS5", ...]
|
330
|
+
#
|
331
|
+
# http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_SetVisibleToAllUsers.html
|
332
|
+
def set_visible_to_all_users(jobflow_ids, visible=true)
|
333
|
+
params = {
|
334
|
+
:operation => 'SetVisibleToAllUsers',
|
335
|
+
:visible_to_all_users => visible,
|
336
|
+
:job_flow_ids => jobflow_ids
|
337
|
+
}
|
338
|
+
aws_result = @aws_request.submit(params)
|
339
|
+
yield aws_result if block_given?
|
340
|
+
end
|
341
|
+
|
342
|
+
# Terminate the specified jobflows. Amazon does not define a return value
|
343
|
+
# for this operation, so you'll need to poll to see the state of the jobflow.
|
344
|
+
#
|
345
|
+
# Takes an [] of job flow IDs.
|
346
|
+
#
|
347
|
+
# ["j-1B4D1XP0C0A35", "j-1YG2MYL0HVYS5", ...]
|
348
|
+
def terminate_jobflows(jobflow_ids)
|
219
349
|
params = {
|
220
350
|
:operation => 'TerminateJobFlows',
|
221
|
-
:job_flow_ids =>
|
351
|
+
:job_flow_ids => jobflow_ids
|
222
352
|
}
|
223
353
|
aws_result = @aws_request.submit(params)
|
224
354
|
yield aws_result if block_given?
|
data/lib/elasticity/job_flow.rb
CHANGED
@@ -25,10 +25,7 @@ module Elasticity
|
|
25
25
|
attr_accessor :job_flow_role
|
26
26
|
attr_accessor :service_role
|
27
27
|
|
28
|
-
|
29
|
-
attr_reader :secret_key
|
30
|
-
|
31
|
-
def initialize(access=nil, secret=nil)
|
28
|
+
def initialize
|
32
29
|
@action_on_failure = 'TERMINATE_JOB_FLOW'
|
33
30
|
@name = 'Elasticity Job Flow'
|
34
31
|
@ami_version = 'latest'
|
@@ -36,8 +33,6 @@ module Elasticity
|
|
36
33
|
self.placement = 'us-east-1a'
|
37
34
|
@enable_debugging = false
|
38
35
|
|
39
|
-
@access_key = access
|
40
|
-
@secret_key = secret
|
41
36
|
@visible_to_all_users = false
|
42
37
|
|
43
38
|
@bootstrap_actions = []
|
@@ -50,16 +45,13 @@ module Elasticity
|
|
50
45
|
@instance_count = 2
|
51
46
|
@master_instance_type = 'm1.small'
|
52
47
|
@slave_instance_type = 'm1.small'
|
53
|
-
|
54
|
-
@access_key = access
|
55
|
-
@secret_key = secret
|
56
48
|
end
|
57
49
|
|
58
|
-
def self.from_jobflow_id(
|
59
|
-
JobFlow.new
|
50
|
+
def self.from_jobflow_id(jobflow_id, region = 'us-east-1')
|
51
|
+
JobFlow.new.tap do |j|
|
60
52
|
j.instance_variable_set(:@region, region)
|
61
53
|
j.instance_variable_set(:@jobflow_id, jobflow_id)
|
62
|
-
j.instance_variable_set(:@installed_steps, j.
|
54
|
+
j.instance_variable_set(:@installed_steps, ClusterStepStatus.installed_steps(j.cluster_step_status))
|
63
55
|
end
|
64
56
|
end
|
65
57
|
|
@@ -154,11 +146,18 @@ module Elasticity
|
|
154
146
|
emr.terminate_jobflows(@jobflow_id)
|
155
147
|
end
|
156
148
|
|
157
|
-
def
|
149
|
+
def cluster_status
|
150
|
+
if !is_jobflow_running?
|
151
|
+
raise JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
|
152
|
+
end
|
153
|
+
ClusterStatus.from_aws_data(emr.describe_cluster(@jobflow_id))
|
154
|
+
end
|
155
|
+
|
156
|
+
def cluster_step_status
|
158
157
|
if !is_jobflow_running?
|
159
158
|
raise JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
|
160
159
|
end
|
161
|
-
emr.
|
160
|
+
ClusterStepStatus.from_aws_list_data(emr.list_steps(@jobflow_id))
|
162
161
|
end
|
163
162
|
|
164
163
|
def wait_for_completion(&on_wait)
|
@@ -169,12 +168,12 @@ module Elasticity
|
|
169
168
|
private
|
170
169
|
|
171
170
|
def retry_check
|
172
|
-
jf_status =
|
173
|
-
return
|
171
|
+
jf_status = cluster_status
|
172
|
+
return cluster_status.active?, jf_status
|
174
173
|
end
|
175
174
|
|
176
175
|
def emr
|
177
|
-
@emr ||= Elasticity::EMR.new(
|
176
|
+
@emr ||= Elasticity::EMR.new(:region => @region)
|
178
177
|
end
|
179
178
|
|
180
179
|
def is_jobflow_running?
|
data/lib/elasticity/version.rb
CHANGED
@@ -0,0 +1,12 @@
|
|
1
|
+
FactoryGirl.define do
|
2
|
+
factory :cluster_status, class: Elasticity::ClusterStatus do
|
3
|
+
cluster_id 'CLUSTER_ID'
|
4
|
+
state 'TERMINATED'
|
5
|
+
created_at Time.at(1436788464.415)
|
6
|
+
ready_at Time.at(1436788842.195)
|
7
|
+
ended_at Time.at(1436791032.097)
|
8
|
+
last_state_change_reason 'ALL_STEPS_COMPLETED'
|
9
|
+
master_public_dns_name 'ec2-54-81-173-103.compute-1.amazonaws.com'
|
10
|
+
normalized_instance_hours 999
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
FactoryGirl.define do
|
2
|
+
factory :cluster_step_status, class: Elasticity::ClusterStepStatus do
|
3
|
+
action_on_failure 'TERMINATE_CLUSTER'
|
4
|
+
args ['36', '3', '0',]
|
5
|
+
jar 's3n://elasticmapreduce/samples/cloudburst/cloudburst.jar'
|
6
|
+
main_class 'MAIN_CLASS'
|
7
|
+
step_id 's-OYPPAC4XPPUC'
|
8
|
+
properties 'Key1' => 'Value1', 'Key2' => 'Value2'
|
9
|
+
name 'Elasticity Custom Jar Step'
|
10
|
+
state 'COMPLETED'
|
11
|
+
state_change_reason 'ALL_STEPS_COMPLETED'
|
12
|
+
state_change_reason_message 'Steps completed'
|
13
|
+
created_at Time.at(1436788464.416)
|
14
|
+
started_at Time.at(1436788841.237)
|
15
|
+
ended_at Time.at(1436790944.162)
|
16
|
+
end
|
17
|
+
end
|
@@ -2,6 +2,11 @@ describe Elasticity::AwsRequestV4 do
|
|
2
2
|
|
3
3
|
before do
|
4
4
|
Timecop.freeze(Time.at(1315611360))
|
5
|
+
|
6
|
+
Elasticity.configure do |c|
|
7
|
+
c.access_key = 'access'
|
8
|
+
c.secret_key = 'secret'
|
9
|
+
end
|
5
10
|
end
|
6
11
|
|
7
12
|
after do
|
@@ -10,11 +15,41 @@ describe Elasticity::AwsRequestV4 do
|
|
10
15
|
|
11
16
|
subject do
|
12
17
|
Elasticity::AwsRequestV4.new(
|
13
|
-
Elasticity::AwsSession.new
|
18
|
+
Elasticity::AwsSession.new,
|
14
19
|
{:operation => 'DescribeJobFlows', :job_flow_ids => ['TEST_JOBFLOW_ID']}
|
15
20
|
)
|
16
21
|
end
|
17
22
|
|
23
|
+
describe '.initialize' do
|
24
|
+
|
25
|
+
describe 'access key' do
|
26
|
+
context 'when not provided' do
|
27
|
+
it 'should be an error' do
|
28
|
+
Elasticity.configure do |c|
|
29
|
+
c.access_key = nil
|
30
|
+
end
|
31
|
+
expect {
|
32
|
+
Elasticity::AwsRequestV4.new(nil, {})
|
33
|
+
}.to raise_error(ArgumentError, '.access_key must be set in the configuration block')
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
describe 'secret key' do
|
39
|
+
context 'when not provided' do
|
40
|
+
it 'should be an error' do
|
41
|
+
Elasticity.configure do |c|
|
42
|
+
c.secret_key = nil
|
43
|
+
end
|
44
|
+
expect {
|
45
|
+
Elasticity::AwsRequestV4.new(nil, {})
|
46
|
+
}.to raise_error(ArgumentError, '.secret_key must be set in the configuration block')
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
18
53
|
describe '#url' do
|
19
54
|
it 'should construct a proper endpoint' do
|
20
55
|
subject.url.should == 'https://elasticmapreduce.us-east-1.amazonaws.com'
|
@@ -22,16 +57,31 @@ describe Elasticity::AwsRequestV4 do
|
|
22
57
|
end
|
23
58
|
|
24
59
|
describe '#headers' do
|
25
|
-
|
26
|
-
|
60
|
+
|
61
|
+
let(:base_headers) {
|
62
|
+
{
|
27
63
|
'Authorization' => "AWS4-HMAC-SHA256 Credential=access/20110909/us-east-1/elasticmapreduce/aws4_request, SignedHeaders=content-type;host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-target, Signature=#{subject.send(:aws_v4_signature)}",
|
28
64
|
'Content-Type' => 'application/x-amz-json-1.1',
|
29
65
|
'Host' => 'elasticmapreduce.us-east-1.amazonaws.com',
|
30
66
|
'User-Agent' => "elasticity/#{Elasticity::VERSION}",
|
31
67
|
'X-Amz-Content-SHA256' => Digest::SHA256.hexdigest(subject.payload),
|
32
68
|
'X-Amz-Date' => '20110909T233600Z',
|
33
|
-
'X-Amz-Target' => 'ElasticMapReduce.DescribeJobFlows'
|
69
|
+
'X-Amz-Target' => 'ElasticMapReduce.DescribeJobFlows'
|
34
70
|
}
|
71
|
+
}
|
72
|
+
|
73
|
+
context 'when a security token is specified' do
|
74
|
+
it 'should create the proper headers' do
|
75
|
+
Elasticity.configure {|c| c.security_token = 'SECURITY_TOKEN' }
|
76
|
+
subject.headers.should == base_headers.merge('X-Amz-Security-Token' => 'SECURITY_TOKEN')
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
context 'when a security token is not specified' do
|
81
|
+
it 'should create the proper headers' do
|
82
|
+
Elasticity.configure {|c| c.security_token = nil }
|
83
|
+
subject.headers.should == base_headers
|
84
|
+
end
|
35
85
|
end
|
36
86
|
end
|
37
87
|
|