RubyGems - elasticity - Versions diffs - 1.5 → 2.0 - Mend

elasticity 1.5 → 2.0

Files changed (57) hide show

data/.rspec +2 -1
data/.rvmrc +1 -1
data/HISTORY.md +47 -24
data/LICENSE +1 -1
data/README.md +165 -317
data/Rakefile +4 -3
data/elasticity.gemspec +3 -5
data/lib/elasticity.rb +10 -5
data/lib/elasticity/aws_request.rb +81 -20
data/lib/elasticity/custom_jar_step.rb +33 -0
data/lib/elasticity/emr.rb +45 -117
data/lib/elasticity/hadoop_bootstrap_action.rb +27 -0
data/lib/elasticity/hive_step.rb +57 -0
data/lib/elasticity/job_flow.rb +109 -39
data/lib/elasticity/job_flow_status.rb +53 -0
data/lib/elasticity/job_flow_status_step.rb +35 -0
data/lib/elasticity/job_flow_step.rb +17 -25
data/lib/elasticity/pig_step.rb +82 -0
data/lib/elasticity/support/conditional_raise.rb +23 -0
data/lib/elasticity/version.rb +1 -1
data/spec/lib/elasticity/aws_request_spec.rb +159 -51
data/spec/lib/elasticity/custom_jar_step_spec.rb +59 -0
data/spec/lib/elasticity/emr_spec.rb +231 -762
data/spec/lib/elasticity/hadoop_bootstrap_action_spec.rb +26 -0
data/spec/lib/elasticity/hive_step_spec.rb +74 -0
data/spec/lib/elasticity/job_flow_integration_spec.rb +197 -0
data/spec/lib/elasticity/job_flow_spec.rb +369 -138
data/spec/lib/elasticity/job_flow_status_spec.rb +147 -0
data/spec/lib/elasticity/job_flow_status_step_spec.rb +73 -0
data/spec/lib/elasticity/job_flow_step_spec.rb +26 -64
data/spec/lib/elasticity/pig_step_spec.rb +104 -0
data/spec/lib/elasticity/support/conditional_raise_spec.rb +35 -0
data/spec/spec_helper.rb +1 -50
data/spec/support/be_a_hash_including_matcher.rb +35 -0
metadata +101 -119
data/.autotest +0 -2
data/lib/elasticity/custom_jar_job.rb +0 -38
data/lib/elasticity/hive_job.rb +0 -69
data/lib/elasticity/pig_job.rb +0 -109
data/lib/elasticity/simple_job.rb +0 -51
data/spec/fixtures/vcr_cassettes/add_instance_groups/one_group_successful.yml +0 -44
data/spec/fixtures/vcr_cassettes/add_instance_groups/one_group_unsuccessful.yml +0 -41
data/spec/fixtures/vcr_cassettes/add_jobflow_steps/add_multiple_steps.yml +0 -266
data/spec/fixtures/vcr_cassettes/custom_jar_job/cloudburst.yml +0 -41
data/spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml +0 -75
data/spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml +0 -38
data/spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml +0 -41
data/spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml +0 -38
data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml +0 -41
data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml +0 -41
data/spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml +0 -41
data/spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml +0 -41
data/spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml +0 -38
data/spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml +0 -38
data/spec/lib/elasticity/custom_jar_job_spec.rb +0 -118
data/spec/lib/elasticity/hive_job_spec.rb +0 -90
data/spec/lib/elasticity/pig_job_spec.rb +0 -226

data/spec/fixtures/vcr_cassettes/custom_jar_job/cloudburst.yml DELETED

@@ -1,41 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Length:
-      - "297"
-      Date:
-      - Fri, 09 Dec 2011 23:01:29 GMT
-      Content-Type:
-      - text/xml
-      X-Amzn-Requestid:
-      - ba9e82df-22b9-11e1-beee-7d8a92482267
-    body:
-      encoding: US-ASCII
-      string: |
-        <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-          <RunJobFlowResult>
-            <JobFlowId>j-1IU6NM8OUPS9I</JobFlowId>
-          </RunJobFlowResult>
-          <ResponseMetadata>
-            <RequestId>ba9e82df-22b9-11e1-beee-7d8a92482267</RequestId>
-          </ResponseMetadata>
-        </RunJobFlowResponse>
-    http_version: "1.1"
-  recorded_at: Sat, 03 Mar 2012 22:59:53 GMT
-recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml DELETED

@@ -1,75 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Date:
-      - Sun, 10 Apr 2011 19:56:17 GMT
-      Content-Type:
-      - text/xml
-      Content-Encoding:
-      - gzip
-      X-Amzn-Requestid:
-      - 994deb7a-63ac-11e0-bc41-ababd98a870b
-      Vary:
-      - Accept-Encoding
-      Transfer-Encoding:
-      - chunked
-    body:
-      encoding: ASCII-8BIT
-      string: !binary |
-        H4sIAAAAAAAAAO1bW3PaOhB+769gztM5c0bYEld7XGZIgIQQIAXnMnnJyLYC
-        7vGttpw2/fVHtmOwjQkQIKUpM0wSS7urlfTp0+6aSC3iqa6ukAtb6Rj2d29E
-        PMe2PFL4YRqW9/mvKaWOyHHEwB7VVRM7LtF8lRSxiX/aFv7uFVXb5DRb5RDP
-        C4AvgRL8q/GpUJByLPsGDbpYZ9wYPbIGk5gKceNH1tD+QVSf6rY1ppj6XotQ
-        rBvzfiZx6hIcCLQwJbJukgbiIQR8mX1kWBPLUKzAe4lbEEvauGTTCgYgp1Ns
-        TciIYM+2GjJxTd1irVpBeS74HnELLvnmE49KXL5G0ibrdekypypiSWBOpWWS
-        2syg9rxCOy2TGZuShtwe9buDptxuhSPRtEzb0hbtCzKsiuzDB0uWlJjvCLdi
-        S6QT26YedbHTVAMpj0v0DTCzddv/dzSWuPDvTwmfieOlPMyCYSZ2aluP+iTd
-        wbrOsWbbzgV2A5lsb4A27Da8Ug6MOUNXPC7AqUOB61sWcdNPxa/YlbhAf9Fq
-        0514i81z918Zcao/kfAHiEaTuLwpp80BoGCPAAfT6Triq0Zfb0jd8ig2DBCo
-        LNeQuPy1kK5c2yEu1UkSC7HOq7sWAWZMqO8UzsPB06iJ9yBE2tDqMCT6bgL5
-        DxfDk4fO5fCW+ZaRScOKW4ar1QwUSu2AhUI7W7FGbIGd9NNh/+qynX/0o2nl
-        Hf9oGMYAlWXHP1qs1ay8AJJohRPoiMm/qzW+Ati/r8g3t52L0bg3YCdt1jUX
-        7wYYtFSS5og+QzZx4z6moANcrdf5R1WQuIXOpOaVgVViEotmJtd8YnPBim7o
-        9PnetkjD9wBjdwqgykCU7UzNOdfkzPEz1/adzPHI5bgdwikee8RoTLcmp7bP
-        nOMlLrd9IyyWRVRdicVVt1jttVss9oGBtz1obYjk1RdZdoWimz1aCZhYoWT7
-        MmX52SENExY9k7HkXDdszur0sfsfoY3h4KHV7jcHrQCmYUtWMD/EYEej8MjO
-        RoHOopM1opH0VG2DNPrNMWPJxDyDxmUaIXSDwzUBSDjv8L0xX+1enXyZq8cS
-        ufxthucwj7vzLpOM0fR5H9guW2P9J9FioXPbd70GhOy8L+tMGoiumxvieuzc
-        NPgi4uMrKG5bTi+Zbc7pXNS98hVDV1uWF06eqAgghk9UqgIo1ACCMIicHZ8S
-        ANPRdGw/bSAVwamoR57DZm+KXUKnbL0m0weNPDHMzzvzyGgB5wsAl3qEOC9E
-        3DTY3Xs7JdbAjlicuj67i1+TSBD/AnNnd32HYX+QeMiwLvIVEQqHEvbPnKqL
-        EG4c9me19xD2Q5mHIiq/cu/vIOzvWgzNmHU+kULT13TCIBEGdQU5XNtjPnDM
-        Bz56PrAJPYV2tqKT2MLb8oHEMEh453ygVG9fo7v7L1dC76K8ZT7A16tlDVZr
-        O88HlF+eD2wMp53nAzMPaiKsvSkfSKKZ31s+sPqGy67QLvIBA7sT8rHzAbl6
-        1ul1eHR+17s5nHSghN4vG3jZ5DdmA7UKgDwEqFIGAjrmArvPBWpipRKVAQ4o
-        F2BOCWKZf2MuMNfedS4wI+OyiEp7zAVm8f/fQRT3zzH4Pwb/hT8k+F+Pj0I7
-        W/FHbGHL4J8XYXnnwf/xPOebA4FLYGMlFoB4a0k+rscSVt6kPGw6BonmBbDm
-        RbM0bY0YQPF1Qyt+W8sHbR2p7uDqWv68ricUK+x5d6MPr+X58Owei3aEslt/
-        PqbtUxapcfPTsrvhL7sn47XnHuzCL2BhlihGtbvxC04/OBUvT742z7tezaNf
-        +HYtWm8OTtuXl0tofQ9VGSS3Tm/O2u3zYacrbFmVEaq1qlp63P1b2kOpymwA
-        rj1VZZgH7FPaoioThhml+n6qMmvlGtkVOr6lXe8t7eBu3C+1Bmd3/HXncMoy
-        8Hd5R1vhAawBWEEA8r9lUeYRG95hV2VQRUT1Q6vKlKCI0JurMrH2fqoyjIyR
-        iGrvUZUZEcdmwaE1ORZmjoWZwh9TmFmHkkI7W1FIbGGbwkw4TFj/ec+3soNe
-        70sZCa16pd7dMvwvE0FDdQQ/4Jc0N0bT3sJ/JLAcZKvwPwDzq985in14a/i/
-        4lLLrtDW4b8KiybRdN/82PE/7JVu7+XhqDk6GQqHE/+/11vZxC6/MQGAtTKA
-        SAAQBr+rv2UOsMsXszO6D/slbvm/dUnx/4712X2jYYpfDLycUYY+QShrRKlh
-        UC1hla0w4YGiliHAClY0oY7rNV4JKCeWDwdctJrrRCjT+PQ/yLkKP9E2AAA=
-    http_version: "1.1"
-  recorded_at: Sat, 03 Mar 2012 22:59:59 GMT
-recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml DELETED

@@ -1,38 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Length:
-      - "225"
-      Date:
-      - Mon, 11 Apr 2011 01:37:58 GMT
-      Content-Type:
-      - text/xml
-      X-Amzn-Requestid:
-      - 54a92156-63dc-11e0-a11b-4b625b21bf04
-    body:
-      encoding: US-ASCII
-      string: |
-        <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-          <ResponseMetadata>
-            <RequestId>54a92156-63dc-11e0-a11b-4b625b21bf04</RequestId>
-          </ResponseMetadata>
-        </TerminateJobFlowsResponse>
-    http_version: "1.1"
-  recorded_at: Sat, 03 Mar 2012 23:00:04 GMT
-recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml DELETED

@@ -1,41 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Length:
-      - "297"
-      Date:
-      - Wed, 29 Feb 2012 22:00:49 GMT
-      Content-Type:
-      - text/xml
-      X-Amzn-Requestid:
-      - d73ae5f8-6320-11e1-aa97-9959c72600a0
-    body:
-      encoding: US-ASCII
-      string: |
-        <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-          <RunJobFlowResult>
-            <JobFlowId>j-2I4HV6S3SDGD9</JobFlowId>
-          </RunJobFlowResult>
-          <ResponseMetadata>
-            <RequestId>d73ae5f8-6320-11e1-aa97-9959c72600a0</RequestId>
-          </ResponseMetadata>
-        </RunJobFlowResponse>
-    http_version: "1.1"
-  recorded_at: Thu, 01 Mar 2012 09:47:52 GMT
-recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml DELETED

@@ -1,38 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Length:
-      - "231"
-      Date:
-      - Wed, 13 Apr 2011 07:04:43 GMT
-      Content-Type:
-      - text/xml
-      X-Amzn-Requestid:
-      - 4ef75373-659c-11e0-bdf6-e3d62a364c28
-    body:
-      encoding: US-ASCII
-      string: |
-        <ModifyInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-          <ResponseMetadata>
-            <RequestId>4ef75373-659c-11e0-bdf6-e3d62a364c28</RequestId>
-          </ResponseMetadata>
-        </ModifyInstanceGroupsResponse>
-    http_version: "1.1"
-  recorded_at: Sat, 03 Mar 2012 23:00:12 GMT
-recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml DELETED

@@ -1,41 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Length:
-      - "297"
-      Date:
-      - Sun, 08 May 2011 05:40:58 GMT
-      Content-Type:
-      - text/xml
-      X-Amzn-Requestid:
-      - c00d285d-7935-11e0-a51d-7bf947dae271
-    body:
-      encoding: US-ASCII
-      string: |
-        <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-          <RunJobFlowResult>
-            <JobFlowId>j-1HB7A3TBRT3VS</JobFlowId>
-          </RunJobFlowResult>
-          <ResponseMetadata>
-            <RequestId>c00d285d-7935-11e0-a51d-7bf947dae271</RequestId>
-          </ResponseMetadata>
-        </RunJobFlowResponse>
-    http_version: "1.1"
-  recorded_at: Sat, 03 Mar 2012 23:00:18 GMT
-recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml DELETED

@@ -1,41 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Length:
-      - "297"
-      Date:
-      - Tue, 10 May 2011 07:28:12 GMT
-      Content-Type:
-      - text/xml
-      X-Amzn-Requestid:
-      - 10533247-7ad7-11e0-86ad-3b6ddfd3d4b9
-    body:
-      encoding: US-ASCII
-      string: |
-        <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-          <RunJobFlowResult>
-            <JobFlowId>j-1UK43AWRT3QHD</JobFlowId>
-          </RunJobFlowResult>
-          <ResponseMetadata>
-            <RequestId>10533247-7ad7-11e0-86ad-3b6ddfd3d4b9</RequestId>
-          </ResponseMetadata>
-        </RunJobFlowResponse>
-    http_version: "1.1"
-  recorded_at: Sat, 03 Mar 2012 23:00:22 GMT
-recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml DELETED

@@ -1,41 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Length:
-      - "296"
-      Date:
-      - Tue, 19 Apr 2011 06:10:16 GMT
-      Content-Type:
-      - text/xml
-      X-Amzn-Requestid:
-      - b22f4aea-6a4b-11e0-9ddc-a168e244afdb
-    body:
-      encoding: US-ASCII
-      string: |
-        <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-          <RunJobFlowResult>
-            <JobFlowId>j-G6N5HA528AD4</JobFlowId>
-          </RunJobFlowResult>
-          <ResponseMetadata>
-            <RequestId>b22f4aea-6a4b-11e0-9ddc-a168e244afdb</RequestId>
-          </ResponseMetadata>
-        </RunJobFlowResponse>
-    http_version: "1.1"
-  recorded_at: Sat, 03 Mar 2012 23:00:27 GMT
-recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml DELETED

@@ -1,41 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 400
-      message: Bad Request
-    headers:
-      Content-Length:
-      - "289"
-      Date:
-      - Sun, 17 Apr 2011 18:52:35 GMT
-      Content-Type:
-      - text/xml
-      X-Amzn-Requestid:
-      - dc0ce847-6923-11e0-9da4-8d278034dea6
-    body:
-      encoding: US-ASCII
-      string: |
-        <ErrorResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-          <Error>
-            <Type>Sender</Type>
-            <Code>ValidationError</Code>
-            <Message>Specified job flow ID not valid</Message>
-          </Error>
-          <RequestId>dc0ce847-6923-11e0-9da4-8d278034dea6</RequestId>
-        </ErrorResponse>
-    http_version: "1.1"
-  recorded_at: Sat, 03 Mar 2012 23:00:34 GMT
-recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml DELETED

@@ -1,38 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: !ruby/regexp /^https/
-    body:
-      encoding: US-ASCII
-      string: ""
-    headers:
-      Accept:
-      - "*/*; q=0.5, application/xml"
-      Accept-Encoding:
-      - gzip, deflate
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Length:
-      - "239"
-      Date:
-      - Sun, 17 Apr 2011 18:49:43 GMT
-      Content-Type:
-      - text/xml
-      X-Amzn-Requestid:
-      - 755ebe8a-6923-11e0-a9c2-c126f1bb4493
-    body:
-      encoding: US-ASCII
-      string: |
-        <SetTerminationProtectionResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-          <ResponseMetadata>
-            <RequestId>755ebe8a-6923-11e0-a9c2-c126f1bb4493</RequestId>
-          </ResponseMetadata>
-        </SetTerminationProtectionResponse>
-    http_version: "1.1"
-  recorded_at: Sat, 03 Mar 2012 23:00:38 GMT
-recorded_with: VCR 2.0.0