wakoopa-elasticity 1.2.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. data/.autotest +2 -0
  2. data/.gitignore +5 -0
  3. data/.rspec +2 -0
  4. data/.rvmrc +1 -0
  5. data/Gemfile +4 -0
  6. data/HISTORY.mediawiki +30 -0
  7. data/LICENSE +202 -0
  8. data/README.mediawiki +332 -0
  9. data/Rakefile +11 -0
  10. data/elasticity.gemspec +29 -0
  11. data/lib/elasticity.rb +16 -0
  12. data/lib/elasticity/aws_request.rb +52 -0
  13. data/lib/elasticity/emr.rb +282 -0
  14. data/lib/elasticity/hive_job.rb +71 -0
  15. data/lib/elasticity/job_flow.rb +53 -0
  16. data/lib/elasticity/job_flow_step.rb +36 -0
  17. data/lib/elasticity/pig_job.rb +112 -0
  18. data/lib/elasticity/simple_job.rb +50 -0
  19. data/lib/elasticity/version.rb +3 -0
  20. data/spec/fixtures/vcr_cassettes/add_instance_groups/one_group_successful.yml +38 -0
  21. data/spec/fixtures/vcr_cassettes/add_instance_groups/one_group_unsuccessful.yml +35 -0
  22. data/spec/fixtures/vcr_cassettes/add_jobflow_steps/add_multiple_steps.yml +252 -0
  23. data/spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml +69 -0
  24. data/spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml +32 -0
  25. data/spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml +35 -0
  26. data/spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml +32 -0
  27. data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml +35 -0
  28. data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml +35 -0
  29. data/spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml +35 -0
  30. data/spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml +35 -0
  31. data/spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml +32 -0
  32. data/spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml +32 -0
  33. data/spec/lib/elasticity/aws_request_spec.rb +62 -0
  34. data/spec/lib/elasticity/emr_spec.rb +794 -0
  35. data/spec/lib/elasticity/hive_job_spec.rb +96 -0
  36. data/spec/lib/elasticity/job_flow_spec.rb +139 -0
  37. data/spec/lib/elasticity/job_flow_step_spec.rb +76 -0
  38. data/spec/lib/elasticity/pig_job_spec.rb +211 -0
  39. data/spec/spec_helper.rb +43 -0
  40. metadata +253 -0
data/spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml ADDED
@@ -0,0 +1,69 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Operation=DescribeJobFlows/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 994deb7a-63ac-11e0-bc41-ababd98a870b
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 10 Apr 2011 19:56:17 GMT
23
+ content-encoding:
24
+ - gzip
25
+ vary:
26
+ - Accept-Encoding
27
+ transfer-encoding:
28
+ - chunked
29
+ body: !binary |
30
+ H4sIAAAAAAAAAO1bW3PaOhB+769gztM5c0bYEld7XGZIgIQQIAXnMnnJyLYC
31
+ 7vGttpw2/fVHtmOwjQkQIKUpM0wSS7urlfTp0+6aSC3iqa6ukAtb6Rj2d29E
32
+ PMe2PFL4YRqW9/mvKaWOyHHEwB7VVRM7LtF8lRSxiX/aFv7uFVXb5DRb5RDP
33
+ C4AvgRL8q/GpUJByLPsGDbpYZ9wYPbIGk5gKceNH1tD+QVSf6rY1ppj6XotQ
34
+ rBvzfiZx6hIcCLQwJbJukgbiIQR8mX1kWBPLUKzAe4lbEEvauGTTCgYgp1Ns
35
+ TciIYM+2GjJxTd1irVpBeS74HnELLvnmE49KXL5G0ibrdekypypiSWBOpWWS
36
+ 2syg9rxCOy2TGZuShtwe9buDptxuhSPRtEzb0hbtCzKsiuzDB0uWlJjvCLdi
37
+ S6QT26YedbHTVAMpj0v0DTCzddv/dzSWuPDvTwmfieOlPMyCYSZ2aluP+iTd
38
+ wbrOsWbbzgV2A5lsb4A27Da8Ug6MOUNXPC7AqUOB61sWcdNPxa/YlbhAf9Fq
39
+ 0514i81z918Zcao/kfAHiEaTuLwpp80BoGCPAAfT6Triq0Zfb0jd8ig2DBCo
40
+ LNeQuPy1kK5c2yEu1UkSC7HOq7sWAWZMqO8UzsPB06iJ9yBE2tDqMCT6bgL5
41
+ DxfDk4fO5fCW+ZaRScOKW4ar1QwUSu2AhUI7W7FGbIGd9NNh/+qynX/0o2nl
42
+ Hf9oGMYAlWXHP1qs1ay8AJJohRPoiMm/qzW+Ati/r8g3t52L0bg3YCdt1jUX
43
+ 7wYYtFSS5og+QzZx4z6moANcrdf5R1WQuIXOpOaVgVViEotmJtd8YnPBim7o
44
+ 9PnetkjD9wBjdwqgykCU7UzNOdfkzPEz1/adzPHI5bgdwikee8RoTLcmp7bP
45
+ nOMlLrd9IyyWRVRdicVVt1jttVss9oGBtz1obYjk1RdZdoWimz1aCZhYoWT7
46
+ MmX52SENExY9k7HkXDdszur0sfsfoY3h4KHV7jcHrQCmYUtWMD/EYEej8MjO
47
+ RoHOopM1opH0VG2DNPrNMWPJxDyDxmUaIXSDwzUBSDjv8L0xX+1enXyZq8cS
48
+ ufxthucwj7vzLpOM0fR5H9guW2P9J9FioXPbd70GhOy8L+tMGoiumxvieuzc
49
+ NPgi4uMrKG5bTi+Zbc7pXNS98hVDV1uWF06eqAgghk9UqgIo1ACCMIicHZ8S
50
+ ANPRdGw/bSAVwamoR57DZm+KXUKnbL0m0weNPDHMzzvzyGgB5wsAl3qEOC9E
51
+ 3DTY3Xs7JdbAjlicuj67i1+TSBD/AnNnd32HYX+QeMiwLvIVEQqHEvbPnKqL
52
+ EG4c9me19xD2Q5mHIiq/cu/vIOzvWgzNmHU+kULT13TCIBEGdQU5XNtjPnDM
53
+ Bz56PrAJPYV2tqKT2MLb8oHEMEh453ygVG9fo7v7L1dC76K8ZT7A16tlDVZr
54
+ O88HlF+eD2wMp53nAzMPaiKsvSkfSKKZ31s+sPqGy67QLvIBA7sT8rHzAbl6
55
+ 1ul1eHR+17s5nHSghN4vG3jZ5DdmA7UKgDwEqFIGAjrmArvPBWpipRKVAQ4o
56
+ F2BOCWKZf2MuMNfedS4wI+OyiEp7zAVm8f/fQRT3zzH4Pwb/hT8k+F+Pj0I7
57
+ W/FHbGHL4J8XYXnnwf/xPOebA4FLYGMlFoB4a0k+rscSVt6kPGw6BonmBbDm
58
+ RbM0bY0YQPF1Qyt+W8sHbR2p7uDqWv68ricUK+x5d6MPr+X58Owei3aEslt/
59
+ PqbtUxapcfPTsrvhL7sn47XnHuzCL2BhlihGtbvxC04/OBUvT742z7tezaNf
60
+ +HYtWm8OTtuXl0tofQ9VGSS3Tm/O2u3zYacrbFmVEaq1qlp63P1b2kOpymwA
61
+ rj1VZZgH7FPaoioThhml+n6qMmvlGtkVOr6lXe8t7eBu3C+1Bmd3/HXncMoy
62
+ 8Hd5R1vhAawBWEEA8r9lUeYRG95hV2VQRUT1Q6vKlKCI0JurMrH2fqoyjIyR
63
+ iGrvUZUZEcdmwaE1ORZmjoWZwh9TmFmHkkI7W1FIbGGbwkw4TFj/ec+3soNe
64
+ 70sZCa16pd7dMvwvE0FDdQQ/4Jc0N0bT3sJ/JLAcZKvwPwDzq985in14a/i/
65
+ 4lLLrtDW4b8KiybRdN/82PE/7JVu7+XhqDk6GQqHE/+/11vZxC6/MQGAtTKA
66
+ SAAQBr+rv2UOsMsXszO6D/slbvm/dUnx/4712X2jYYpfDLycUYY+QShrRKlh
67
+ UC1hla0w4YGiliHAClY0oY7rNV4JKCeWDwdctJrrRCjT+PQ/yLkKP9E2AAA=
68
+
69
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml ADDED
@@ -0,0 +1,32 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1MZ5TVWFJRSKN&Operation=TerminateJobFlows/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 54a92156-63dc-11e0-a11b-4b625b21bf04
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Mon, 11 Apr 2011 01:37:58 GMT
23
+ content-length:
24
+ - "225"
25
+ body: |
26
+ <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <ResponseMetadata>
28
+ <RequestId>54a92156-63dc-11e0-a11b-4b625b21bf04</RequestId>
29
+ </ResponseMetadata>
30
+ </TerminateJobFlowsResponse>
31
+
32
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Instances\.Ec2KeyName=sharethrough_dev&Instances\.HadoopVersion=0\.20&Instances\.InstanceCount=2&Instances\.MasterInstanceType=m1\.small&Instances\.SlaveInstanceType=m1\.small&Name=Elasticity%20Hive%20Job&Operation=RunJobFlow&.*&Steps\.member\.1\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.1\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/hive\/hive-script&Steps\.member\.1\.HadoopJarStep\.Args\.member\.2=--base-path&Steps\.member\.1\.HadoopJarStep\.Args\.member\.3=s3:\/\/elasticmapreduce\/libs\/hive\/&Steps\.member\.1\.HadoopJarStep\.Args\.member\.4=--install-hive&Steps\.member\.1\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.1\.Name=Setup%20Hive&Steps\.member\.2\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.2\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/hive\/hive-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.10=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.11=LIBS=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/libs&Steps\.member\.2\.HadoopJarStep\.Args\.member\.2=--run-hive-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.3=--args&Steps\.member\.2\.HadoopJarStep\.Args\.member\.4=-f&Steps\.member\.2\.HadoopJarStep\.Args\.member\.5=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/libs\/model-build\.q&Steps\.member\.2\.HadoopJarStep\.Args\.member\.6=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.7=INPUT=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/tables&Steps\.member\.2\.HadoopJarStep\.Args\.member\.8=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.9=OUTPUT=s3n:\/\/slif-elasticity\/hive-ads\/output\/2011-04-19&Steps\.member\.2\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.2\.Name=Run%20Hive%20Script/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 87d43cfd-6fb2-11e0-bc88-bf902ad8b20b
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Tue, 26 Apr 2011 03:08:58 GMT
23
+ content-length:
24
+ - "297"
25
+ body: |
26
+ <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <RunJobFlowResult>
28
+ <JobFlowId>j-1UUVYMHBLKEGN</JobFlowId>
29
+ </RunJobFlowResult>
30
+ <ResponseMetadata>
31
+ <RequestId>87d43cfd-6fb2-11e0-bc88-bf902ad8b20b</RequestId>
32
+ </ResponseMetadata>
33
+ </RunJobFlowResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml ADDED
@@ -0,0 +1,32 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&InstanceGroups.member.1.InstanceCount=2&InstanceGroups.member.1.InstanceGroupId=ig-2T1HNUO61BG3O&Operation=ModifyInstanceGroups/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 4ef75373-659c-11e0-bdf6-e3d62a364c28
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Wed, 13 Apr 2011 07:04:43 GMT
23
+ content-length:
24
+ - "231"
25
+ body: |
26
+ <ModifyInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <ResponseMetadata>
28
+ <RequestId>4ef75373-659c-11e0-bdf6-e3d62a364c28</RequestId>
29
+ </ResponseMetadata>
30
+ </ModifyInstanceGroupsResponse>
31
+
32
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Instances.Ec2KeyName=sharethrough_dev&Instances.HadoopVersion=0.20&Instances.InstanceCount=2&Instances.MasterInstanceType=m1.small&Instances.SlaveInstanceType=m1.small&LogUri=s3n:\/\/slif-elasticity\/pig-apache\/logs&Name=Elasticity%20Pig%20Job&Operation=RunJobFlow&.*&Steps.member.1.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.1.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.1.HadoopJarStep.Args.member.2=--base-path&Steps.member.1.HadoopJarStep.Args.member.3=s3:\/\/elasticmapreduce\/libs\/pig\/&Steps.member.1.HadoopJarStep.Args.member.4=--install-pig&Steps.member.1.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.1.Name=Setup%20Pig&Steps.member.2.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.2.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.2.HadoopJarStep.Args.member.10=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/do-reports.pig&Steps.member.2.HadoopJarStep.Args.member.2=--run-pig-script&Steps.member.2.HadoopJarStep.Args.member.3=--args&Steps.member.2.HadoopJarStep.Args.member.4=-p&Steps.member.2.HadoopJarStep.Args.member.5=INPUT=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/input&Steps.member.2.HadoopJarStep.Args.member.6=-p&Steps.member.2.HadoopJarStep.Args.member.7=OUTPUT=s3n:\/\/slif-elasticity\/pig-apache\/output\/2011-05-04&Steps.member.2.HadoopJarStep.Args.member.8=-p&Steps.member.2.HadoopJarStep.Args.member.9=E_PARALLELS=1&Steps.member.2.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.2.Name=Run%20Pig%20Script/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - c00d285d-7935-11e0-a51d-7bf947dae271
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 08 May 2011 05:40:58 GMT
23
+ content-length:
24
+ - "297"
25
+ body: |
26
+ <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <RunJobFlowResult>
28
+ <JobFlowId>j-1HB7A3TBRT3VS</JobFlowId>
29
+ </RunJobFlowResult>
30
+ <ResponseMetadata>
31
+ <RequestId>c00d285d-7935-11e0-a51d-7bf947dae271</RequestId>
32
+ </ResponseMetadata>
33
+ </RunJobFlowResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&BootstrapActions.member.1.Name=Elasticity%20Bootstrap%20Action%20\(Configure%20Hadoop\)&BootstrapActions.member.1.ScriptBootstrapAction.Args.member.1=-m&BootstrapActions.member.1.ScriptBootstrapAction.Args.member.2=mapred.job.reuse.jvm.num.tasks=120&BootstrapActions.member.1.ScriptBootstrapAction.Path=s3n:\/\/elasticmapreduce\/bootstrap-actions\/configure-hadoop&Instances.Ec2KeyName=sharethrough_dev&Instances.HadoopVersion=0.20&Instances.InstanceCount=2&Instances.MasterInstanceType=m1.small&Instances.SlaveInstanceType=m1.small&Name=Elasticity%20Pig%20Job&Operation=RunJobFlow&.*&Steps.member.1.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.1.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.1.HadoopJarStep.Args.member.2=--base-path&Steps.member.1.HadoopJarStep.Args.member.3=s3:\/\/elasticmapreduce\/libs\/pig\/&Steps.member.1.HadoopJarStep.Args.member.4=--install-pig&Steps.member.1.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.1.Name=Setup%20Pig&Steps.member.2.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.2.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.2.HadoopJarStep.Args.member.10=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/do-reports.pig&Steps.member.2.HadoopJarStep.Args.member.2=--run-pig-script&Steps.member.2.HadoopJarStep.Args.member.3=--args&Steps.member.2.HadoopJarStep.Args.member.4=-p&Steps.member.2.HadoopJarStep.Args.member.5=INPUT=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/input&Steps.member.2.HadoopJarStep.Args.member.6=-p&Steps.member.2.HadoopJarStep.Args.member.7=OUTPUT=s3n:\/\/slif-elasticity\/pig-apache\/output\/2011-05-10&Steps.member.2.HadoopJarStep.Args.member.8=-p&Steps.member.2.HadoopJarStep.Args.member.9=E_PARALLELS=1&Steps.member.2.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.2.Name=Run%20Pig%20Script/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 10533247-7ad7-11e0-86ad-3b6ddfd3d4b9
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Tue, 10 May 2011 07:28:12 GMT
23
+ content-length:
24
+ - "297"
25
+ body: |
26
+ <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <RunJobFlowResult>
28
+ <JobFlowId>j-1UK43AWRT3QHD</JobFlowId>
29
+ </RunJobFlowResult>
30
+ <ResponseMetadata>
31
+ <RequestId>10533247-7ad7-11e0-86ad-3b6ddfd3d4b9</RequestId>
32
+ </ResponseMetadata>
33
+ </RunJobFlowResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Instances\.Ec2KeyName=sharethrough-dev&Instances\.HadoopVersion=0\.20&Instances\.InstanceCount=2&Instances\.MasterInstanceType=m1\.small&Instances\.Placement\.AvailabilityZone=us-east-1a&Instances\.SlaveInstanceType=m1\.small&Name=Elasticity%20Test%20Flow%20\(EMR%20Pig%20Script\)&Operation=RunJobFlow&.*&Steps\.member\.1\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.1\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps\.member\.1\.HadoopJarStep\.Args\.member\.2=--base-path&Steps\.member\.1\.HadoopJarStep\.Args\.member\.3=s3:\/\/elasticmapreduce\/libs\/pig\/&Steps\.member\.1\.HadoopJarStep\.Args\.member\.4=--install-pig&Steps\.member\.1\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.1\.Name=Setup%20Pig&Steps\.member\.2\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.2\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.2=--run-pig-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.3=--args&Steps\.member\.2\.HadoopJarStep\.Args\.member\.4=-p&Steps\.member\.2\.HadoopJarStep\.Args\.member\.5=INPUT=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/input&Steps\.member\.2\.HadoopJarStep\.Args\.member\.6=-p&Steps\.member\.2\.HadoopJarStep\.Args\.member\.7=OUTPUT=s3n:\/\/slif-elasticity\/pig-apache\/output\/2011-04-19&Steps\.member\.2\.HadoopJarStep\.Args\.member\.8=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/do-reports\.pig&Steps\.member\.2\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.2\.Name=Run%20Pig%20Script/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - b22f4aea-6a4b-11e0-9ddc-a168e244afdb
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Tue, 19 Apr 2011 06:10:16 GMT
23
+ content-length:
24
+ - "296"
25
+ body: |
26
+ <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <RunJobFlowResult>
28
+ <JobFlowId>j-G6N5HA528AD4</JobFlowId>
29
+ </RunJobFlowResult>
30
+ <ResponseMetadata>
31
+ <RequestId>b22f4aea-6a4b-11e0-9ddc-a168e244afdb</RequestId>
32
+ </ResponseMetadata>
33
+ </RunJobFlowResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1B4D1XP0C0A35&JobFlowIds.member.2=j-2&Operation=SetTerminationProtection.*&TerminationProtected=true/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 400
15
+ message: Bad Request
16
+ headers:
17
+ x-amzn-requestid:
18
+ - dc0ce847-6923-11e0-9da4-8d278034dea6
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 17 Apr 2011 18:52:35 GMT
23
+ content-length:
24
+ - "289"
25
+ body: |
26
+ <ErrorResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <Error>
28
+ <Type>Sender</Type>
29
+ <Code>ValidationError</Code>
30
+ <Message>Specified job flow ID not valid</Message>
31
+ </Error>
32
+ <RequestId>dc0ce847-6923-11e0-9da4-8d278034dea6</RequestId>
33
+ </ErrorResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml ADDED
@@ -0,0 +1,32 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1B4D1XP0C0A35&JobFlowIds.member.2=j-1YG2MYL0HVYS5&Operation=SetTerminationProtection&.*TerminationProtected=true/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 755ebe8a-6923-11e0-a9c2-c126f1bb4493
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 17 Apr 2011 18:49:43 GMT
23
+ content-length:
24
+ - "239"
25
+ body: |
26
+ <SetTerminationProtectionResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <ResponseMetadata>
28
+ <RequestId>755ebe8a-6923-11e0-a9c2-c126f1bb4493</RequestId>
29
+ </ResponseMetadata>
30
+ </SetTerminationProtectionResponse>
31
+
32
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml ADDED
@@ -0,0 +1,32 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1MZ5TVWFJRSKN&Operation=TerminateJobFlows/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 83e0154d-63c6-11e0-bc41-ababd98a870b
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 10 Apr 2011 23:01:48 GMT
23
+ content-length:
24
+ - "225"
25
+ body: |
26
+ <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <ResponseMetadata>
28
+ <RequestId>83e0154d-63c6-11e0-bc41-ababd98a870b</RequestId>
29
+ </ResponseMetadata>
30
+ </TerminateJobFlowsResponse>
31
+
32
+ http_version: "1.1"
data/spec/lib/elasticity/aws_request_spec.rb ADDED
@@ -0,0 +1,62 @@
1
+ require 'spec_helper'
2
+
3
+ describe Elasticity::AwsRequest do
4
+
5
+ describe ".aws_escape" do
6
+ it "should escape according to Amazon's rules" do
7
+ # Don't encode reserved characters
8
+ Elasticity::AwsRequest.aws_escape("foo-_.~bar").should == "foo-_.~bar"
9
+ # Encode as %20, not as +
10
+ Elasticity::AwsRequest.aws_escape("foo bar").should == "foo%20bar"
11
+ # Percent encode all other characters with %XY, where X and Y are hex characters 0-9 and uppercase A-F.
12
+ Elasticity::AwsRequest.aws_escape("foo$&+,/:;=?@bar").should == "foo%24%26%2B%2C%2F%3A%3B%3D%3F%40bar"
13
+ end
14
+ end
15
+
16
+ describe "#sign_params" do
17
+ before do
18
+ Time.should_receive(:now).and_return(Time.at(1302461096))
19
+ end
20
+ it "should sign according to Amazon's rules" do
21
+ request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key")
22
+ signed_params = request.send(:sign_params, {}, "GET", "example.com", "/")
23
+ signed_params.should == "AWSAccessKeyId=aws_access_key_id&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-04-10T18%3A44%3A56.000Z&Signature=jVLfPS056dNmjpCcikBnPmRHJNZ8YGaI7zdmHWUk658%3D"
24
+ end
25
+ end
26
+
27
+ describe "#aws_emr_request" do
28
+ before do
29
+ Time.should_receive(:now).and_return(Time.at(1302461096))
30
+ end
31
+ it "should accept region as option" do
32
+ request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key",
33
+ :region => "eu-west-1")
34
+ RestClient.should_receive(:get).with(/elasticmapreduce\.eu\-west\-1\.amazonaws\.com/)
35
+ request.aws_emr_request({})
36
+ end
37
+ end
38
+
39
+ describe "#aws_emr_request" do
40
+ before do
41
+ Time.should_receive(:now).and_return(Time.at(1302461096))
42
+ end
43
+ it "should accept secure" do
44
+ request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key",
45
+ :secure => false)
46
+ RestClient.should_receive(:get).with(/^http:/)
47
+ request.aws_emr_request({})
48
+ end
49
+ end
50
+
51
+ describe "#aws_emr_request" do
52
+ before do
53
+ Time.should_receive(:now).and_return(Time.at(1302461096))
54
+ end
55
+ it "should default to secure connection" do
56
+ request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key")
57
+ RestClient.should_receive(:get).with(/^https:/)
58
+ request.aws_emr_request({})
59
+ end
60
+ end
61
+
62
+ end