wakoopa-elasticity 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/.autotest +2 -0
  2. data/.gitignore +5 -0
  3. data/.rspec +2 -0
  4. data/.rvmrc +1 -0
  5. data/Gemfile +4 -0
  6. data/HISTORY.mediawiki +30 -0
  7. data/LICENSE +202 -0
  8. data/README.mediawiki +332 -0
  9. data/Rakefile +11 -0
  10. data/elasticity.gemspec +29 -0
  11. data/lib/elasticity.rb +16 -0
  12. data/lib/elasticity/aws_request.rb +52 -0
  13. data/lib/elasticity/emr.rb +282 -0
  14. data/lib/elasticity/hive_job.rb +71 -0
  15. data/lib/elasticity/job_flow.rb +53 -0
  16. data/lib/elasticity/job_flow_step.rb +36 -0
  17. data/lib/elasticity/pig_job.rb +112 -0
  18. data/lib/elasticity/simple_job.rb +50 -0
  19. data/lib/elasticity/version.rb +3 -0
  20. data/spec/fixtures/vcr_cassettes/add_instance_groups/one_group_successful.yml +38 -0
  21. data/spec/fixtures/vcr_cassettes/add_instance_groups/one_group_unsuccessful.yml +35 -0
  22. data/spec/fixtures/vcr_cassettes/add_jobflow_steps/add_multiple_steps.yml +252 -0
  23. data/spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml +69 -0
  24. data/spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml +32 -0
  25. data/spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml +35 -0
  26. data/spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml +32 -0
  27. data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml +35 -0
  28. data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml +35 -0
  29. data/spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml +35 -0
  30. data/spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml +35 -0
  31. data/spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml +32 -0
  32. data/spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml +32 -0
  33. data/spec/lib/elasticity/aws_request_spec.rb +62 -0
  34. data/spec/lib/elasticity/emr_spec.rb +794 -0
  35. data/spec/lib/elasticity/hive_job_spec.rb +96 -0
  36. data/spec/lib/elasticity/job_flow_spec.rb +139 -0
  37. data/spec/lib/elasticity/job_flow_step_spec.rb +76 -0
  38. data/spec/lib/elasticity/pig_job_spec.rb +211 -0
  39. data/spec/spec_helper.rb +43 -0
  40. metadata +253 -0
data/spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml ADDED
@@ -0,0 +1,69 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Operation=DescribeJobFlows/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 994deb7a-63ac-11e0-bc41-ababd98a870b
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 10 Apr 2011 19:56:17 GMT
23
+ content-encoding:
24
+ - gzip
25
+ vary:
26
+ - Accept-Encoding
27
+ transfer-encoding:
28
+ - chunked
29
+ body: !binary |
30
+ H4sIAAAAAAAAAO1bW3PaOhB+769gztM5c0bYEld7XGZIgIQQIAXnMnnJyLYC
31
+ 7vGttpw2/fVHtmOwjQkQIKUpM0wSS7urlfTp0+6aSC3iqa6ukAtb6Rj2d29E
32
+ PMe2PFL4YRqW9/mvKaWOyHHEwB7VVRM7LtF8lRSxiX/aFv7uFVXb5DRb5RDP
33
+ C4AvgRL8q/GpUJByLPsGDbpYZ9wYPbIGk5gKceNH1tD+QVSf6rY1ppj6XotQ
34
+ rBvzfiZx6hIcCLQwJbJukgbiIQR8mX1kWBPLUKzAe4lbEEvauGTTCgYgp1Ns
35
+ TciIYM+2GjJxTd1irVpBeS74HnELLvnmE49KXL5G0ibrdekypypiSWBOpWWS
36
+ 2syg9rxCOy2TGZuShtwe9buDptxuhSPRtEzb0hbtCzKsiuzDB0uWlJjvCLdi
37
+ S6QT26YedbHTVAMpj0v0DTCzddv/dzSWuPDvTwmfieOlPMyCYSZ2aluP+iTd
38
+ wbrOsWbbzgV2A5lsb4A27Da8Ug6MOUNXPC7AqUOB61sWcdNPxa/YlbhAf9Fq
39
+ 0514i81z918Zcao/kfAHiEaTuLwpp80BoGCPAAfT6Triq0Zfb0jd8ig2DBCo
40
+ LNeQuPy1kK5c2yEu1UkSC7HOq7sWAWZMqO8UzsPB06iJ9yBE2tDqMCT6bgL5
41
+ DxfDk4fO5fCW+ZaRScOKW4ar1QwUSu2AhUI7W7FGbIGd9NNh/+qynX/0o2nl
42
+ Hf9oGMYAlWXHP1qs1ay8AJJohRPoiMm/qzW+Ati/r8g3t52L0bg3YCdt1jUX
43
+ 7wYYtFSS5og+QzZx4z6moANcrdf5R1WQuIXOpOaVgVViEotmJtd8YnPBim7o
44
+ 9PnetkjD9wBjdwqgykCU7UzNOdfkzPEz1/adzPHI5bgdwikee8RoTLcmp7bP
45
+ nOMlLrd9IyyWRVRdicVVt1jttVss9oGBtz1obYjk1RdZdoWimz1aCZhYoWT7
46
+ MmX52SENExY9k7HkXDdszur0sfsfoY3h4KHV7jcHrQCmYUtWMD/EYEej8MjO
47
+ RoHOopM1opH0VG2DNPrNMWPJxDyDxmUaIXSDwzUBSDjv8L0xX+1enXyZq8cS
48
+ ufxthucwj7vzLpOM0fR5H9guW2P9J9FioXPbd70GhOy8L+tMGoiumxvieuzc
49
+ NPgi4uMrKG5bTi+Zbc7pXNS98hVDV1uWF06eqAgghk9UqgIo1ACCMIicHZ8S
50
+ ANPRdGw/bSAVwamoR57DZm+KXUKnbL0m0weNPDHMzzvzyGgB5wsAl3qEOC9E
51
+ 3DTY3Xs7JdbAjlicuj67i1+TSBD/AnNnd32HYX+QeMiwLvIVEQqHEvbPnKqL
52
+ EG4c9me19xD2Q5mHIiq/cu/vIOzvWgzNmHU+kULT13TCIBEGdQU5XNtjPnDM
53
+ Bz56PrAJPYV2tqKT2MLb8oHEMEh453ygVG9fo7v7L1dC76K8ZT7A16tlDVZr
54
+ O88HlF+eD2wMp53nAzMPaiKsvSkfSKKZ31s+sPqGy67QLvIBA7sT8rHzAbl6
55
+ 1ul1eHR+17s5nHSghN4vG3jZ5DdmA7UKgDwEqFIGAjrmArvPBWpipRKVAQ4o
56
+ F2BOCWKZf2MuMNfedS4wI+OyiEp7zAVm8f/fQRT3zzH4Pwb/hT8k+F+Pj0I7
57
+ W/FHbGHL4J8XYXnnwf/xPOebA4FLYGMlFoB4a0k+rscSVt6kPGw6BonmBbDm
58
+ RbM0bY0YQPF1Qyt+W8sHbR2p7uDqWv68ricUK+x5d6MPr+X58Owei3aEslt/
59
+ PqbtUxapcfPTsrvhL7sn47XnHuzCL2BhlihGtbvxC04/OBUvT742z7tezaNf
60
+ +HYtWm8OTtuXl0tofQ9VGSS3Tm/O2u3zYacrbFmVEaq1qlp63P1b2kOpymwA
61
+ rj1VZZgH7FPaoioThhml+n6qMmvlGtkVOr6lXe8t7eBu3C+1Bmd3/HXncMoy
62
+ 8Hd5R1vhAawBWEEA8r9lUeYRG95hV2VQRUT1Q6vKlKCI0JurMrH2fqoyjIyR
63
+ iGrvUZUZEcdmwaE1ORZmjoWZwh9TmFmHkkI7W1FIbGGbwkw4TFj/ec+3soNe
64
+ 70sZCa16pd7dMvwvE0FDdQQ/4Jc0N0bT3sJ/JLAcZKvwPwDzq985in14a/i/
65
+ 4lLLrtDW4b8KiybRdN/82PE/7JVu7+XhqDk6GQqHE/+/11vZxC6/MQGAtTKA
66
+ SAAQBr+rv2UOsMsXszO6D/slbvm/dUnx/4712X2jYYpfDLycUYY+QShrRKlh
67
+ UC1hla0w4YGiliHAClY0oY7rNV4JKCeWDwdctJrrRCjT+PQ/yLkKP9E2AAA=
68
+
69
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml ADDED
@@ -0,0 +1,32 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1MZ5TVWFJRSKN&Operation=TerminateJobFlows/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 54a92156-63dc-11e0-a11b-4b625b21bf04
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Mon, 11 Apr 2011 01:37:58 GMT
23
+ content-length:
24
+ - "225"
25
+ body: |
26
+ <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <ResponseMetadata>
28
+ <RequestId>54a92156-63dc-11e0-a11b-4b625b21bf04</RequestId>
29
+ </ResponseMetadata>
30
+ </TerminateJobFlowsResponse>
31
+
32
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Instances\.Ec2KeyName=sharethrough_dev&Instances\.HadoopVersion=0\.20&Instances\.InstanceCount=2&Instances\.MasterInstanceType=m1\.small&Instances\.SlaveInstanceType=m1\.small&Name=Elasticity%20Hive%20Job&Operation=RunJobFlow&.*&Steps\.member\.1\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.1\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/hive\/hive-script&Steps\.member\.1\.HadoopJarStep\.Args\.member\.2=--base-path&Steps\.member\.1\.HadoopJarStep\.Args\.member\.3=s3:\/\/elasticmapreduce\/libs\/hive\/&Steps\.member\.1\.HadoopJarStep\.Args\.member\.4=--install-hive&Steps\.member\.1\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.1\.Name=Setup%20Hive&Steps\.member\.2\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.2\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/hive\/hive-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.10=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.11=LIBS=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/libs&Steps\.member\.2\.HadoopJarStep\.Args\.member\.2=--run-hive-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.3=--args&Steps\.member\.2\.HadoopJarStep\.Args\.member\.4=-f&Steps\.member\.2\.HadoopJarStep\.Args\.member\.5=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/libs\/model-build\.q&Steps\.member\.2\.HadoopJarStep\.Args\.member\.6=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.7=INPUT=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/tables&Steps\.member\.2\.HadoopJarStep\.Args\.member\.8=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.9=OUTPUT=s3n:\/\/slif-elasticity\/hive-ads\/output\/2011-04-19&Steps\.member\.2\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.2\.Name=Run%20Hive%20Script/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 87d43cfd-6fb2-11e0-bc88-bf902ad8b20b
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Tue, 26 Apr 2011 03:08:58 GMT
23
+ content-length:
24
+ - "297"
25
+ body: |
26
+ <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <RunJobFlowResult>
28
+ <JobFlowId>j-1UUVYMHBLKEGN</JobFlowId>
29
+ </RunJobFlowResult>
30
+ <ResponseMetadata>
31
+ <RequestId>87d43cfd-6fb2-11e0-bc88-bf902ad8b20b</RequestId>
32
+ </ResponseMetadata>
33
+ </RunJobFlowResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml ADDED
@@ -0,0 +1,32 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&InstanceGroups.member.1.InstanceCount=2&InstanceGroups.member.1.InstanceGroupId=ig-2T1HNUO61BG3O&Operation=ModifyInstanceGroups/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 4ef75373-659c-11e0-bdf6-e3d62a364c28
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Wed, 13 Apr 2011 07:04:43 GMT
23
+ content-length:
24
+ - "231"
25
+ body: |
26
+ <ModifyInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <ResponseMetadata>
28
+ <RequestId>4ef75373-659c-11e0-bdf6-e3d62a364c28</RequestId>
29
+ </ResponseMetadata>
30
+ </ModifyInstanceGroupsResponse>
31
+
32
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Instances.Ec2KeyName=sharethrough_dev&Instances.HadoopVersion=0.20&Instances.InstanceCount=2&Instances.MasterInstanceType=m1.small&Instances.SlaveInstanceType=m1.small&LogUri=s3n:\/\/slif-elasticity\/pig-apache\/logs&Name=Elasticity%20Pig%20Job&Operation=RunJobFlow&.*&Steps.member.1.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.1.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.1.HadoopJarStep.Args.member.2=--base-path&Steps.member.1.HadoopJarStep.Args.member.3=s3:\/\/elasticmapreduce\/libs\/pig\/&Steps.member.1.HadoopJarStep.Args.member.4=--install-pig&Steps.member.1.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.1.Name=Setup%20Pig&Steps.member.2.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.2.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.2.HadoopJarStep.Args.member.10=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/do-reports.pig&Steps.member.2.HadoopJarStep.Args.member.2=--run-pig-script&Steps.member.2.HadoopJarStep.Args.member.3=--args&Steps.member.2.HadoopJarStep.Args.member.4=-p&Steps.member.2.HadoopJarStep.Args.member.5=INPUT=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/input&Steps.member.2.HadoopJarStep.Args.member.6=-p&Steps.member.2.HadoopJarStep.Args.member.7=OUTPUT=s3n:\/\/slif-elasticity\/pig-apache\/output\/2011-05-04&Steps.member.2.HadoopJarStep.Args.member.8=-p&Steps.member.2.HadoopJarStep.Args.member.9=E_PARALLELS=1&Steps.member.2.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.2.Name=Run%20Pig%20Script/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - c00d285d-7935-11e0-a51d-7bf947dae271
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 08 May 2011 05:40:58 GMT
23
+ content-length:
24
+ - "297"
25
+ body: |
26
+ <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <RunJobFlowResult>
28
+ <JobFlowId>j-1HB7A3TBRT3VS</JobFlowId>
29
+ </RunJobFlowResult>
30
+ <ResponseMetadata>
31
+ <RequestId>c00d285d-7935-11e0-a51d-7bf947dae271</RequestId>
32
+ </ResponseMetadata>
33
+ </RunJobFlowResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&BootstrapActions.member.1.Name=Elasticity%20Bootstrap%20Action%20\(Configure%20Hadoop\)&BootstrapActions.member.1.ScriptBootstrapAction.Args.member.1=-m&BootstrapActions.member.1.ScriptBootstrapAction.Args.member.2=mapred.job.reuse.jvm.num.tasks=120&BootstrapActions.member.1.ScriptBootstrapAction.Path=s3n:\/\/elasticmapreduce\/bootstrap-actions\/configure-hadoop&Instances.Ec2KeyName=sharethrough_dev&Instances.HadoopVersion=0.20&Instances.InstanceCount=2&Instances.MasterInstanceType=m1.small&Instances.SlaveInstanceType=m1.small&Name=Elasticity%20Pig%20Job&Operation=RunJobFlow&.*&Steps.member.1.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.1.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.1.HadoopJarStep.Args.member.2=--base-path&Steps.member.1.HadoopJarStep.Args.member.3=s3:\/\/elasticmapreduce\/libs\/pig\/&Steps.member.1.HadoopJarStep.Args.member.4=--install-pig&Steps.member.1.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.1.Name=Setup%20Pig&Steps.member.2.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.2.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.2.HadoopJarStep.Args.member.10=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/do-reports.pig&Steps.member.2.HadoopJarStep.Args.member.2=--run-pig-script&Steps.member.2.HadoopJarStep.Args.member.3=--args&Steps.member.2.HadoopJarStep.Args.member.4=-p&Steps.member.2.HadoopJarStep.Args.member.5=INPUT=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/input&Steps.member.2.HadoopJarStep.Args.member.6=-p&Steps.member.2.HadoopJarStep.Args.member.7=OUTPUT=s3n:\/\/slif-elasticity\/pig-apache\/output\/2011-05-10&Steps.member.2.HadoopJarStep.Args.member.8=-p&Steps.member.2.HadoopJarStep.Args.member.9=E_PARALLELS=1&Steps.member.2.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.2.Name=Run%20Pig%20Script/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 10533247-7ad7-11e0-86ad-3b6ddfd3d4b9
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Tue, 10 May 2011 07:28:12 GMT
23
+ content-length:
24
+ - "297"
25
+ body: |
26
+ <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <RunJobFlowResult>
28
+ <JobFlowId>j-1UK43AWRT3QHD</JobFlowId>
29
+ </RunJobFlowResult>
30
+ <ResponseMetadata>
31
+ <RequestId>10533247-7ad7-11e0-86ad-3b6ddfd3d4b9</RequestId>
32
+ </ResponseMetadata>
33
+ </RunJobFlowResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Instances\.Ec2KeyName=sharethrough-dev&Instances\.HadoopVersion=0\.20&Instances\.InstanceCount=2&Instances\.MasterInstanceType=m1\.small&Instances\.Placement\.AvailabilityZone=us-east-1a&Instances\.SlaveInstanceType=m1\.small&Name=Elasticity%20Test%20Flow%20\(EMR%20Pig%20Script\)&Operation=RunJobFlow&.*&Steps\.member\.1\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.1\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps\.member\.1\.HadoopJarStep\.Args\.member\.2=--base-path&Steps\.member\.1\.HadoopJarStep\.Args\.member\.3=s3:\/\/elasticmapreduce\/libs\/pig\/&Steps\.member\.1\.HadoopJarStep\.Args\.member\.4=--install-pig&Steps\.member\.1\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.1\.Name=Setup%20Pig&Steps\.member\.2\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.2\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.2=--run-pig-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.3=--args&Steps\.member\.2\.HadoopJarStep\.Args\.member\.4=-p&Steps\.member\.2\.HadoopJarStep\.Args\.member\.5=INPUT=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/input&Steps\.member\.2\.HadoopJarStep\.Args\.member\.6=-p&Steps\.member\.2\.HadoopJarStep\.Args\.member\.7=OUTPUT=s3n:\/\/slif-elasticity\/pig-apache\/output\/2011-04-19&Steps\.member\.2\.HadoopJarStep\.Args\.member\.8=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/do-reports\.pig&Steps\.member\.2\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.2\.Name=Run%20Pig%20Script/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - b22f4aea-6a4b-11e0-9ddc-a168e244afdb
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Tue, 19 Apr 2011 06:10:16 GMT
23
+ content-length:
24
+ - "296"
25
+ body: |
26
+ <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <RunJobFlowResult>
28
+ <JobFlowId>j-G6N5HA528AD4</JobFlowId>
29
+ </RunJobFlowResult>
30
+ <ResponseMetadata>
31
+ <RequestId>b22f4aea-6a4b-11e0-9ddc-a168e244afdb</RequestId>
32
+ </ResponseMetadata>
33
+ </RunJobFlowResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml ADDED
@@ -0,0 +1,35 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1B4D1XP0C0A35&JobFlowIds.member.2=j-2&Operation=SetTerminationProtection.*&TerminationProtected=true/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 400
15
+ message: Bad Request
16
+ headers:
17
+ x-amzn-requestid:
18
+ - dc0ce847-6923-11e0-9da4-8d278034dea6
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 17 Apr 2011 18:52:35 GMT
23
+ content-length:
24
+ - "289"
25
+ body: |
26
+ <ErrorResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <Error>
28
+ <Type>Sender</Type>
29
+ <Code>ValidationError</Code>
30
+ <Message>Specified job flow ID not valid</Message>
31
+ </Error>
32
+ <RequestId>dc0ce847-6923-11e0-9da4-8d278034dea6</RequestId>
33
+ </ErrorResponse>
34
+
35
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml ADDED
@@ -0,0 +1,32 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1B4D1XP0C0A35&JobFlowIds.member.2=j-1YG2MYL0HVYS5&Operation=SetTerminationProtection&.*TerminationProtected=true/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 755ebe8a-6923-11e0-a9c2-c126f1bb4493
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 17 Apr 2011 18:49:43 GMT
23
+ content-length:
24
+ - "239"
25
+ body: |
26
+ <SetTerminationProtectionResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <ResponseMetadata>
28
+ <RequestId>755ebe8a-6923-11e0-a9c2-c126f1bb4493</RequestId>
29
+ </ResponseMetadata>
30
+ </SetTerminationProtectionResponse>
31
+
32
+ http_version: "1.1"
data/spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml ADDED
@@ -0,0 +1,32 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com:443\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1MZ5TVWFJRSKN&Operation=TerminateJobFlows/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 83e0154d-63c6-11e0-bc41-ababd98a870b
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 10 Apr 2011 23:01:48 GMT
23
+ content-length:
24
+ - "225"
25
+ body: |
26
+ <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <ResponseMetadata>
28
+ <RequestId>83e0154d-63c6-11e0-bc41-ababd98a870b</RequestId>
29
+ </ResponseMetadata>
30
+ </TerminateJobFlowsResponse>
31
+
32
+ http_version: "1.1"
data/spec/lib/elasticity/aws_request_spec.rb ADDED
@@ -0,0 +1,62 @@
1
+ require 'spec_helper'
2
+
3
+ describe Elasticity::AwsRequest do
4
+
5
+ describe ".aws_escape" do
6
+ it "should escape according to Amazon's rules" do
7
+ # Don't encode reserved characters
8
+ Elasticity::AwsRequest.aws_escape("foo-_.~bar").should == "foo-_.~bar"
9
+ # Encode as %20, not as +
10
+ Elasticity::AwsRequest.aws_escape("foo bar").should == "foo%20bar"
11
+ # Percent encode all other characters with %XY, where X and Y are hex characters 0-9 and uppercase A-F.
12
+ Elasticity::AwsRequest.aws_escape("foo$&+,/:;=?@bar").should == "foo%24%26%2B%2C%2F%3A%3B%3D%3F%40bar"
13
+ end
14
+ end
15
+
16
+ describe "#sign_params" do
17
+ before do
18
+ Time.should_receive(:now).and_return(Time.at(1302461096))
19
+ end
20
+ it "should sign according to Amazon's rules" do
21
+ request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key")
22
+ signed_params = request.send(:sign_params, {}, "GET", "example.com", "/")
23
+ signed_params.should == "AWSAccessKeyId=aws_access_key_id&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-04-10T18%3A44%3A56.000Z&Signature=jVLfPS056dNmjpCcikBnPmRHJNZ8YGaI7zdmHWUk658%3D"
24
+ end
25
+ end
26
+
27
+ describe "#aws_emr_request" do
28
+ before do
29
+ Time.should_receive(:now).and_return(Time.at(1302461096))
30
+ end
31
+ it "should accept region as option" do
32
+ request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key",
33
+ :region => "eu-west-1")
34
+ RestClient.should_receive(:get).with(/elasticmapreduce\.eu\-west\-1\.amazonaws\.com/)
35
+ request.aws_emr_request({})
36
+ end
37
+ end
38
+
39
+ describe "#aws_emr_request" do
40
+ before do
41
+ Time.should_receive(:now).and_return(Time.at(1302461096))
42
+ end
43
+ it "should accept secure" do
44
+ request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key",
45
+ :secure => false)
46
+ RestClient.should_receive(:get).with(/^http:/)
47
+ request.aws_emr_request({})
48
+ end
49
+ end
50
+
51
+ describe "#aws_emr_request" do
52
+ before do
53
+ Time.should_receive(:now).and_return(Time.at(1302461096))
54
+ end
55
+ it "should default to secure connection" do
56
+ request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key")
57
+ RestClient.should_receive(:get).with(/^https:/)
58
+ request.aws_emr_request({})
59
+ end
60
+ end
61
+
62
+ end