elasticity 1.1 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/HISTORY.mediawiki +6 -0
- data/lib/elasticity/hive_job.rb +9 -3
- data/lib/elasticity/version.rb +1 -1
- data/spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml +35 -0
- data/spec/lib/elasticity/emr_spec.rb +0 -3
- data/spec/lib/elasticity/hive_job_spec.rb +23 -4
- data/spec/spec_helper.rb +2 -0
- metadata +6 -3
data/HISTORY.mediawiki
CHANGED
@@ -10,3 +10,9 @@
|
|
10
10
|
|
11
11
|
* Added HiveJob, a simplified way to launch basic Hive job flows.
|
12
12
|
* Added HISTORY.
|
13
|
+
|
14
|
+
=== 1.1.1 ===
|
15
|
+
|
16
|
+
* HiveJob critical bug fixed, now it works :)
|
17
|
+
* Added log_uri and action_on_failure as options to HiveJob.
|
18
|
+
* Added integration tests to HiveJob.
|
data/lib/elasticity/hive_job.rb
CHANGED
@@ -4,16 +4,19 @@ module Elasticity
|
|
4
4
|
# having to understand the entirety of the EMR API.
|
5
5
|
class HiveJob
|
6
6
|
|
7
|
+
attr_accessor :action_on_failure
|
7
8
|
attr_accessor :aws_access_key_id
|
8
9
|
attr_accessor :aws_secret_access_key
|
9
10
|
attr_accessor :ec2_key_name
|
10
11
|
attr_accessor :name
|
11
12
|
attr_accessor :hadoop_version
|
12
13
|
attr_accessor :instance_count
|
14
|
+
attr_accessor :log_uri
|
13
15
|
attr_accessor :master_instance_type
|
14
16
|
attr_accessor :slave_instance_type
|
15
17
|
|
16
18
|
def initialize(aws_access_key_id, aws_secret_access_key)
|
19
|
+
@action_on_failure = "TERMINATE_JOB_FLOW"
|
17
20
|
@aws_access_key_id = aws_access_key_id
|
18
21
|
@aws_secret_access_key = aws_secret_access_key
|
19
22
|
@ec2_key_name = "default"
|
@@ -23,7 +26,7 @@ module Elasticity
|
|
23
26
|
@name = "Elasticity Hive Job"
|
24
27
|
@slave_instance_type = "m1.small"
|
25
28
|
|
26
|
-
@
|
29
|
+
@emr = Elasticity::EMR.new(aws_access_key_id, aws_secret_access_key)
|
27
30
|
end
|
28
31
|
|
29
32
|
# Run the specified Hive script with the specified variables.
|
@@ -68,7 +71,7 @@ module Elasticity
|
|
68
71
|
:name => "Setup Hive"
|
69
72
|
},
|
70
73
|
{
|
71
|
-
:action_on_failure =>
|
74
|
+
:action_on_failure => @action_on_failure,
|
72
75
|
:hadoop_jar_step => {
|
73
76
|
:jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
|
74
77
|
:args => script_arguments,
|
@@ -77,7 +80,10 @@ module Elasticity
|
|
77
80
|
}
|
78
81
|
]
|
79
82
|
}
|
80
|
-
|
83
|
+
|
84
|
+
jobflow_config.merge!(:log_uri => @log_uri) if @log_uri
|
85
|
+
|
86
|
+
@emr.run_job_flow(jobflow_config)
|
81
87
|
end
|
82
88
|
|
83
89
|
end
|
data/lib/elasticity/version.rb
CHANGED
@@ -0,0 +1,35 @@
|
|
1
|
+
---
|
2
|
+
- !ruby/struct:VCR::HTTPInteraction
|
3
|
+
request: !ruby/struct:VCR::Request
|
4
|
+
method: :get
|
5
|
+
uri: !ruby/regexp /^http:\/\/elasticmapreduce.amazonaws.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Instances\.Ec2KeyName=sharethrough_dev&Instances\.HadoopVersion=0\.20&Instances\.InstanceCount=2&Instances\.MasterInstanceType=m1\.small&Instances\.SlaveInstanceType=m1\.small&Name=Elasticity%20Hive%20Job&Operation=RunJobFlow&.*&Steps\.member\.1\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.1\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/hive\/hive-script&Steps\.member\.1\.HadoopJarStep\.Args\.member\.2=--base-path&Steps\.member\.1\.HadoopJarStep\.Args\.member\.3=s3:\/\/elasticmapreduce\/libs\/hive\/&Steps\.member\.1\.HadoopJarStep\.Args\.member\.4=--install-hive&Steps\.member\.1\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.1\.Name=Setup%20Hive&Steps\.member\.2\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.2\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/hive\/hive-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.10=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.11=LIBS=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/libs&Steps\.member\.2\.HadoopJarStep\.Args\.member\.2=--run-hive-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.3=--args&Steps\.member\.2\.HadoopJarStep\.Args\.member\.4=-f&Steps\.member\.2\.HadoopJarStep\.Args\.member\.5=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/libs\/model-build\.q&Steps\.member\.2\.HadoopJarStep\.Args\.member\.6=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.7=INPUT=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/tables&Steps\.member\.2\.HadoopJarStep\.Args\.member\.8=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.9=OUTPUT=s3n:\/\/slif-elasticity\/hive-ads\/output\/2011-04-19&Steps\.member\.2\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.2\.Name=Run%20Hive%20Script/
|
6
|
+
body:
|
7
|
+
headers:
|
8
|
+
accept:
|
9
|
+
- "*/*; q=0.5, application/xml"
|
10
|
+
accept-encoding:
|
11
|
+
- gzip, deflate
|
12
|
+
response: !ruby/struct:VCR::Response
|
13
|
+
status: !ruby/struct:VCR::ResponseStatus
|
14
|
+
code: 200
|
15
|
+
message: OK
|
16
|
+
headers:
|
17
|
+
x-amzn-requestid:
|
18
|
+
- 87d43cfd-6fb2-11e0-bc88-bf902ad8b20b
|
19
|
+
content-type:
|
20
|
+
- text/xml
|
21
|
+
date:
|
22
|
+
- Tue, 26 Apr 2011 03:08:58 GMT
|
23
|
+
content-length:
|
24
|
+
- "297"
|
25
|
+
body: |
|
26
|
+
<RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
27
|
+
<RunJobFlowResult>
|
28
|
+
<JobFlowId>j-1UUVYMHBLKEGN</JobFlowId>
|
29
|
+
</RunJobFlowResult>
|
30
|
+
<ResponseMetadata>
|
31
|
+
<RequestId>87d43cfd-6fb2-11e0-bc88-bf902ad8b20b</RequestId>
|
32
|
+
</ResponseMetadata>
|
33
|
+
</RunJobFlowResponse>
|
34
|
+
|
35
|
+
http_version: "1.1"
|
@@ -14,16 +14,19 @@ describe Elasticity::HiveJob do
|
|
14
14
|
hive.master_instance_type.should == "m1.small"
|
15
15
|
hive.name.should == "Elasticity Hive Job"
|
16
16
|
hive.slave_instance_type.should == "m1.small"
|
17
|
+
hive.action_on_failure.should == "TERMINATE_JOB_FLOW"
|
18
|
+
hive.log_uri.should == nil
|
17
19
|
end
|
18
20
|
|
19
21
|
end
|
20
22
|
|
21
23
|
describe "#run" do
|
22
24
|
|
23
|
-
it "run the script with the specified variables and return the jobflow_id" do
|
24
|
-
aws = Elasticity::
|
25
|
+
it "should run the script with the specified variables and return the jobflow_id" do
|
26
|
+
aws = Elasticity::EMR.new("", "")
|
25
27
|
aws.should_receive(:run_job_flow).with({
|
26
28
|
:name => "Elasticity Hive Job",
|
29
|
+
:log_uri => "s3n://slif-test/output/logs",
|
27
30
|
:instances => {
|
28
31
|
:ec2_key_name => "default",
|
29
32
|
:hadoop_version => "0.20",
|
@@ -46,7 +49,7 @@ describe Elasticity::HiveJob do
|
|
46
49
|
:name => "Setup Hive"
|
47
50
|
},
|
48
51
|
{
|
49
|
-
:action_on_failure => "
|
52
|
+
:action_on_failure => "CONTINUE",
|
50
53
|
:hadoop_jar_step => {
|
51
54
|
:jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
|
52
55
|
:args => [
|
@@ -62,9 +65,11 @@ describe Elasticity::HiveJob do
|
|
62
65
|
}
|
63
66
|
]
|
64
67
|
}).and_return("new_jobflow_id")
|
65
|
-
Elasticity::
|
68
|
+
Elasticity::EMR.should_receive(:new).with("access", "secret").and_return(aws)
|
66
69
|
|
67
70
|
hive = Elasticity::HiveJob.new("access", "secret")
|
71
|
+
hive.log_uri = "s3n://slif-test/output/logs"
|
72
|
+
hive.action_on_failure = "CONTINUE"
|
68
73
|
jobflow_id = hive.run('s3n://slif-hive/test.q', {
|
69
74
|
'OUTPUT' => 's3n://slif-test/output',
|
70
75
|
'XREFS' => 's3n://slif-test/xrefs'
|
@@ -74,4 +79,18 @@ describe Elasticity::HiveJob do
|
|
74
79
|
|
75
80
|
end
|
76
81
|
|
82
|
+
describe "integration happy path" do
|
83
|
+
use_vcr_cassette "hive_job/hive_ads", :record => :none
|
84
|
+
it "should kick off the sample Amazion EMR Hive application" do
|
85
|
+
hive = Elasticity::HiveJob.new(AWS_ACCESS_KEY_ID, AWS_SECRET_KEY)
|
86
|
+
hive.ec2_key_name = "sharethrough_dev"
|
87
|
+
jobflow_id = hive.run("s3n://elasticmapreduce/samples/hive-ads/libs/model-build.q", {
|
88
|
+
"LIBS" => "s3n://elasticmapreduce/samples/hive-ads/libs",
|
89
|
+
"INPUT" => "s3n://elasticmapreduce/samples/hive-ads/tables",
|
90
|
+
"OUTPUT" => "s3n://slif-elasticity/hive-ads/output/2011-04-19"
|
91
|
+
})
|
92
|
+
jobflow_id.should == "j-1UUVYMHBLKEGN"
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
77
96
|
end
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticity
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 17
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 1
|
8
8
|
- 1
|
9
|
-
|
9
|
+
- 1
|
10
|
+
version: 1.1.1
|
10
11
|
platform: ruby
|
11
12
|
authors:
|
12
13
|
- Robert Slifka
|
@@ -14,7 +15,7 @@ autorequire:
|
|
14
15
|
bindir: bin
|
15
16
|
cert_chain: []
|
16
17
|
|
17
|
-
date: 2011-04-
|
18
|
+
date: 2011-04-25 00:00:00 -07:00
|
18
19
|
default_executable:
|
19
20
|
dependencies:
|
20
21
|
- !ruby/object:Gem::Dependency
|
@@ -180,6 +181,7 @@ files:
|
|
180
181
|
- spec/fixtures/vcr_cassettes/add_jobflow_steps/add_multiple_steps.yml
|
181
182
|
- spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
|
182
183
|
- spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
|
184
|
+
- spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
|
183
185
|
- spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
|
184
186
|
- spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
|
185
187
|
- spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml
|
@@ -231,6 +233,7 @@ test_files:
|
|
231
233
|
- spec/fixtures/vcr_cassettes/add_jobflow_steps/add_multiple_steps.yml
|
232
234
|
- spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
|
233
235
|
- spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
|
236
|
+
- spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
|
234
237
|
- spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
|
235
238
|
- spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
|
236
239
|
- spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml
|