elasticity 6.0.1 → 6.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2fc0007bc333d20b825ba0f5c8ae04ed9475e561
-  data.tar.gz: 96241ed239fc3857cf68c71a68630d5f3fa620dd
+  metadata.gz: affb1ae0de5df89c0061fabfde695f56ac0f4845
+  data.tar.gz: 43c2bdd9e155587f28288c42271c4561b787e3aa
 SHA512:
-  metadata.gz: 06ff88bee6ff7635e3372a262a10bb544345100ca41e6e1ba6492ed69d9e5c65419670390976f9a40c92a569ef839315582642228236733e1a61e0c4ff545e33
-  data.tar.gz: 3af02b989f196cd04d8cf11762884a243500271e3baf46b61b670c69b02f84f8f8bd51f3349d7e6a2d77dbc65001bfec52805a4ddd49d371081813f0fd956d84
+  metadata.gz: 1ec9e99c8fda33defb4dc9bfb318080740920201c25080f87310036b85970831a194e0a0d0b41a2361d9af95240ca327fab4554deb622044123d6abce5af6347
+  data.tar.gz: 4aa6636cfd730be45dd3829ed8db82951167b056f5fa76e7aabac6efa02f1ded0a2f49e70164661aff0119178ea9cb68ad07b600167102cdca649786bd867183

data/HISTORY.md

@@ -1,3 +1,10 @@
+## 6.0.2 - July 20, 2015
+
+- Including PR [#111](https://github.com/rslifka/elasticity/pull/111) - "Handle steps status that hasn't run yet".  Thank you [@robert2d](https://github.com/robert2d)!
+- Including PR [#112](https://github.com/rslifka/elasticity/pull/112) - "Parse Dates when step/cluster still starting".  Thank you [@robert2d](https://github.com/robert2d)!
+- Including PR [#113](https://github.com/rslifka/elasticity/pull/113) - "Add job flow steps remove extra steps key".  Thank you [@robert2d](https://github.com/robert2d)!
+- Including PR [#114](https://github.com/rslifka/elasticity/pull/114) - "Allow tags to be set on runJobFlow".  Thank you [@robert2d](https://github.com/robert2d)!
+
 ## 6.0.1 - July 19, 2015
 
 - Including PR [#110](https://github.com/rslifka/elasticity/pull/110) - "Send bid price as string to aws api".  Thank you [@robert2d](https://github.com/robert2d)!

data/README.md

@@ -119,6 +119,7 @@ jobflow.ami_version                       = 'latest'
 jobflow.log_uri                           = nil
 jobflow.enable_debugging                  = false # Requires a log_uri to enable
 
+jobflow.tags                              = {name: "app-name", department: 'marketing'}
 jobflow.ec2_key_name                      = nil
 jobflow.visible_to_all_users              = false
 jobflow.placement                         = 'us-east-1a'

data/lib/elasticity/cluster_status.rb

@@ -20,8 +20,8 @@ module Elasticity
         c.cluster_id = cluster_data['Id']
         c.state = cluster_data['Status']['State']
         c.created_at = Time.at(cluster_data['Status']['Timeline']['CreationDateTime'])
-        c.ready_at = Time.at(cluster_data['Status']['Timeline']['ReadyDateTime'])
-        c.ended_at = Time.at(cluster_data['Status']['Timeline']['EndDateTime'])
+        c.ready_at = cluster_data['Status']['Timeline']['ReadyDateTime'] ? Time.at(cluster_data['Status']['Timeline']['ReadyDateTime']) : nil
+        c.ended_at = cluster_data['Status']['Timeline']['EndDateTime'] ? Time.at(cluster_data['Status']['Timeline']['EndDateTime']) : nil
         c.last_state_change_reason = cluster_data['Status']['StateChangeReason']['Code']
         c.master_public_dns_name = cluster_data['MasterPublicDnsName']
         c.normalized_instance_hours = cluster_data['NormalizedInstanceHours']
@@ -35,4 +35,4 @@ module Elasticity
 
   end
 
-end
+end

data/lib/elasticity/cluster_step_status.rb

@@ -31,8 +31,8 @@ module Elasticity
           c.state_change_reason = s['Status']['StateChangeReason']['Code']
           c.state_change_reason_message = s['Status']['StateChangeReason']['Message']
           c.created_at = Time.at(s['Status']['Timeline']['CreationDateTime'])
-          c.started_at = Time.at(s['Status']['Timeline']['StartDateTime'])
-          c.ended_at   = Time.at(s['Status']['Timeline']['EndDateTime'])
+          c.started_at = s['Status']['Timeline']['StartDateTime'] ? Time.at(s['Status']['Timeline']['StartDateTime']) : nil
+          c.ended_at   = s['Status']['Timeline']['EndDateTime'] ? Time.at(s['Status']['Timeline']['EndDateTime']) : nil
         end
       end
     end
@@ -48,4 +48,4 @@ module Elasticity
 
   end
 
-end
+end

data/lib/elasticity/job_flow.rb

@@ -13,6 +13,7 @@ module Elasticity
     attr_accessor :name
     attr_accessor :instance_count
     attr_accessor :log_uri
+    attr_accessor :tags
     attr_accessor :master_instance_type
     attr_accessor :slave_instance_type
     attr_accessor :ami_version
@@ -126,7 +127,7 @@ module Elasticity
           jobflow_steps.concat(jobflow_step.aws_installation_steps)
         end
         jobflow_steps << jobflow_step.to_aws_step(self)
-        emr.add_jobflow_steps(@jobflow_id, {:steps => jobflow_steps})
+        emr.add_jobflow_steps(@jobflow_id, jobflow_steps)
       else
         @jobflow_steps << jobflow_step
       end
@@ -186,12 +187,22 @@ module Elasticity
       steps.insert(0, Elasticity::SetupHadoopDebuggingStep.new.to_aws_step(self)) if @enable_debugging
       config[:steps] = steps
       config[:log_uri] = @log_uri if @log_uri
+      config[:tags] = jobflow_tags if @tags
       config[:job_flow_role] = @job_flow_role if @job_flow_role
       config[:service_role] = @service_role if @service_role
       config[:bootstrap_actions] = @bootstrap_actions.map{|a| a.to_aws_bootstrap_action} unless @bootstrap_actions.empty?
       config
     end
 
+    def jobflow_tags
+      @tags.map do |key, value|
+        {
+          key: key.to_s,
+          value: value
+        }
+      end
+    end
+
     def jobflow_preamble
       preamble = {
         :name => @name,

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '6.0.1'
+  VERSION = '6.0.2'
 end

data/spec/lib/elasticity/aws_request_v4_spec.rb

@@ -123,7 +123,7 @@ describe Elasticity::AwsRequestV4 do
 
   describe '.aws_v4_signature' do
     it 'should create the proper signature' do
-      subject.send(:aws_v4_signature).should == '8105dbe78c4198071af447357621c53d8c7b4c984b3540e918fa67c9eb299d1d'
+      subject.send(:aws_v4_signature).should == '05d406786907e37181060b8c744b5aea8451a7da93f20cacb6c6fd846cfcfa5d'
     end
   end
 

data/spec/lib/elasticity/cluster_status_spec.rb

@@ -1,6 +1,16 @@
 describe Elasticity::ClusterStatus do
 
   let(:cluster_state) { 'TERMINATED' }
+  let(:timeline) do
+    <<-JSON
+      {
+        "CreationDateTime": 1436788464.415,
+        "EndDateTime": 1436791032.097,
+        "ReadyDateTime": 1436788842.195
+      }
+    JSON
+  end
+
   let(:aws_cluster_status) do
     <<-JSON
       {
@@ -32,11 +42,7 @@ describe Elasticity::ClusterStatus do
               "Code": "ALL_STEPS_COMPLETED",
               "Message": "Steps completed"
             },
-            "Timeline": {
-              "CreationDateTime": 1436788464.415,
-              "EndDateTime": 1436791032.097,
-              "ReadyDateTime": 1436788842.195
-            }
+            "Timeline": #{timeline}
           },
           "Tags": [
             {
@@ -67,6 +73,21 @@ describe Elasticity::ClusterStatus do
       expect(subject.master_public_dns_name).to eql('ec2-54-81-173-103.compute-1.amazonaws.com')
       expect(subject.normalized_instance_hours).to eql(2)
     end
+
+    context 'when cluster is brand new' do
+      let(:timeline) do
+        <<-JSON
+          {
+            "CreationDateTime": 1436788464.415
+          }
+        JSON
+      end
+
+      it 'defaults ready_at and ended_at correctly' do
+        expect(subject.ready_at).not_to be
+        expect(subject.ended_at).not_to be
+      end
+    end
   end
 
   describe '#active?' do
@@ -95,4 +116,4 @@ describe Elasticity::ClusterStatus do
 
   end
 
-end
+end

data/spec/lib/elasticity/cluster_step_status_spec.rb

@@ -1,5 +1,15 @@
 describe Elasticity::ClusterStepStatus do
 
+  let(:timeline) do
+    <<-JSON
+      {
+        "CreationDateTime": 1436788464.416,
+        "StartDateTime": 1436788841.237,
+        "EndDateTime": 1436790944.162
+      }
+    JSON
+  end
+
   let(:aws_cluster_steps) do
     <<-JSON
       {
@@ -27,11 +37,7 @@ describe Elasticity::ClusterStepStatus do
                         "Code": "ALL_STEPS_COMPLETED",
                         "Message": "Steps completed"
                       },
-                      "Timeline": {
-                          "CreationDateTime": 1436788464.416,
-                          "EndDateTime": 1436790944.162,
-                          "StartDateTime": 1436788841.237
-                      }
+                      "Timeline": #{timeline}
                   }
               }
           ]
@@ -62,6 +68,22 @@ describe Elasticity::ClusterStepStatus do
       expect(status.started_at).to eql(Time.at(1436788841.237))
       expect(status.ended_at).to eql(Time.at(1436790944.162))
     end
+
+    context 'newly created step that hasn\'t started yet' do
+      let(:timeline) do
+        <<-JSON
+          {
+            "CreationDateTime": 1436788464.416
+          }
+        JSON
+      end
+
+      it 'sets started_at and ended_at to nil' do
+        status = cluster_step_statuses[0]
+        expect(status.started_at).not_to be
+        expect(status.ended_at).not_to be
+      end
+    end
   end
 
   describe '.installed_steps' do
@@ -77,4 +99,4 @@ describe Elasticity::ClusterStepStatus do
     end
   end
 
-end
+end

data/spec/lib/elasticity/job_flow_spec.rb

@@ -24,7 +24,7 @@ describe Elasticity::JobFlow do
       expect(subject.service_role).to eql(nil)
     end
   end
-  
+
   describe '#placement=' do
 
     context 'when the placement is set' do
@@ -218,9 +218,7 @@ describe Elasticity::JobFlow do
           let(:additional_step) { Elasticity::PigStep.new('_') }
 
           it 'should submit the step' do
-            emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
-                :steps => [additional_step.to_aws_step(running_jobflow)]
-              })
+            emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', [additional_step.to_aws_step(running_jobflow)])
             running_jobflow.add_step(additional_step)
           end
         end
@@ -229,9 +227,7 @@ describe Elasticity::JobFlow do
           let(:additional_step) { Elasticity::HiveStep.new('_') }
 
           it 'should submit the installation step and the step' do
-            emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
-                :steps => Elasticity::HiveStep.aws_installation_steps << additional_step.to_aws_step(running_jobflow)
-              })
+            emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', Elasticity::HiveStep.aws_installation_steps << additional_step.to_aws_step(running_jobflow))
             running_jobflow.add_step(additional_step)
           end
         end
@@ -243,9 +239,7 @@ describe Elasticity::JobFlow do
         let(:additional_step) { Elasticity::CustomJarStep.new('jar') }
 
         it 'should submit the step' do
-          emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
-              :steps => [additional_step.to_aws_step(running_jobflow)]
-            })
+          emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', [additional_step.to_aws_step(running_jobflow)])
           running_jobflow.add_step(additional_step)
         end
 
@@ -336,6 +330,45 @@ describe Elasticity::JobFlow do
 
     end
 
+    describe 'tags' do
+
+      context 'when a tag is specified' do
+        let(:jobflow_with_tags) do
+          Elasticity::JobFlow.new.tap do |jf|
+            jf.tags = {
+              name: 'app-name',
+              contact: 'admin@example.com'
+            }
+          end
+        end
+        it 'should incorporate it into the jobflow config' do
+          jobflow_with_tags.send(:jobflow_config).should be_a_hash_including(
+            tags: [
+              {
+                key: 'name',
+                value: 'app-name'
+              }, {
+                key: 'contact',
+                value: 'admin@example.com'
+              }
+            ]
+          )
+        end
+      end
+
+      context 'when a tag is not specified' do
+        let(:jobflow_with_no_tags) do
+          Elasticity::JobFlow.new.tap do |jf|
+            jf.tags = nil
+          end
+        end
+        it 'should not make space for it in the jobflow config' do
+          jobflow_with_no_tags.send(:jobflow_config).should_not have_key(:tags)
+        end
+      end
+
+    end
+
     describe 'job flow role' do
 
       context 'when a job flow role is specified' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: 6.0.1
+  version: 6.0.2
 platform: ruby
 authors:
 - Robert Slifka