elasticity 5.0.3 → 6.0

data/spec/lib/elasticity/job_flow_step_spec.rb

@@ -29,7 +29,7 @@ describe Elasticity::JobFlowStep do
 
   describe '.requires_installation?' do
     it 'should be false by default' do
-      FakeStep.requires_installation?.should be_false
+      expect(FakeStep.requires_installation?).to be false
     end
   end
 

data/spec/lib/elasticity/looper_spec.rb

@@ -77,7 +77,7 @@ describe Elasticity::Looper do
 
     it 'does not communicate that waiting is about to occur' do
       client = double(:client, on_retry_check: false, on_wait: nil)
-      client.should_not receive(:on_wait)
+      client.should_not_receive(:on_wait)
       l = Elasticity::Looper.new(client.method(:on_retry_check), client.method(:on_wait))
       l.go
     end

data/spec/lib/elasticity/pig_step_spec.rb

@@ -6,15 +6,19 @@ describe Elasticity::PigStep do
 
   it { should be_a Elasticity::JobFlowStep }
 
-  its(:name) { should == 'Elasticity Pig Step (script.pig)' }
-  its(:script) { should == 'script.pig' }
-  its(:variables) { should == {} }
-  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
+  describe '.initialize' do
+    it 'should set the fields appropriately' do
+      expect(subject.name).to eql('Elasticity Pig Step (script.pig)')
+      expect(subject.script).to eql('script.pig')
+      expect(subject.variables).to eql({})
+      expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
+    end
+  end
 
   describe '#to_aws_step' do
 
     it 'should convert to aws step format' do
-      step = subject.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
+      step = subject.to_aws_step(Elasticity::JobFlow.new)
       step[:name].should == 'Elasticity Pig Step (script.pig)'
       step[:action_on_failure].should == 'TERMINATE_JOB_FLOW'
       step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
@@ -29,7 +33,7 @@ describe Elasticity::PigStep do
 
     describe 'E_PARALLELS' do
       it 'should include the correct value of E_PARALLELS' do
-        job_flow = Elasticity::JobFlow.new('access', 'secret')
+        job_flow = Elasticity::JobFlow.new
         job_flow.instance_count = 8
         {
           '_' => 7,
@@ -49,7 +53,7 @@ describe Elasticity::PigStep do
       let(:ps_with_no_variables) { Elasticity::PigStep.new('script.pig') }
 
       it 'should convert to aws step format' do
-        step = ps_with_no_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
+        step = ps_with_no_variables.to_aws_step(Elasticity::JobFlow.new)
         step[:hadoop_jar_step][:args][5].should == 'script.pig'
       end
     end
@@ -65,7 +69,7 @@ describe Elasticity::PigStep do
       end
 
       it 'should convert to aws step format' do
-        step = ps_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
+        step = ps_with_variables.to_aws_step(Elasticity::JobFlow.new)
         step[:hadoop_jar_step][:args][3..9].should == [
           '-p', 'VAR1=VALUE1',
           '-p', 'VAR2=VALUE2',
@@ -79,7 +83,7 @@ describe Elasticity::PigStep do
 
   describe '.requires_installation?' do
     it 'should require installation' do
-      Elasticity::PigStep.requires_installation?.should be_true
+      expect(Elasticity::PigStep.requires_installation?).to be true
     end
   end
 

data/spec/lib/elasticity/s3distcp_step_spec.rb

@@ -1,10 +1,12 @@
 describe Elasticity::S3DistCpStep do
 
-  it { should be_a Elasticity::CustomJarStep }
+  it 'should be a CustomJarStep' do
+    expect(subject).to be_a(Elasticity::CustomJarStep)
+  end
 
-  its(:name) { should == 'Elasticity S3DistCp Step' }
-  its(:jar) { should == '/home/hadoop/lib/emr-s3distcp-1.0.jar' }
-  its(:arguments) { should == [] }
-  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
+  it 'should set the appropriate default fields' do
+    expect(subject.name).to eql('Elasticity S3DistCp Step')
+    expect(subject.jar).to eql('/home/hadoop/lib/emr-s3distcp-1.0.jar')
+  end
 
 end

data/spec/lib/elasticity/script_step_spec.rb

@@ -1,12 +1,17 @@
 describe Elasticity::ScriptStep do
 
-  it { should be_a Elasticity::CustomJarStep }
+  subject do
+    Elasticity::ScriptStep.new('script_location', 'arg1', 'arg2')
+  end
 
-  subject { Elasticity::ScriptStep.new('script_location', 'arg1', 'arg2') }
+  it { should be_a Elasticity::CustomJarStep }
 
-  its(:name) { should == 'Elasticity Script Step' }
-  its(:jar) { should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar' }
-  its(:arguments) { should == %w(script_location arg1 arg2) }
-  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
+  describe '.initialize' do
+    it 'should set the fields appropriately' do
+      expect(subject.name).to eql('Elasticity Script Step')
+      expect(subject.jar).to eql('s3://elasticmapreduce/libs/script-runner/script-runner.jar')
+      expect(subject.arguments).to eql(%w(script_location arg1 arg2))
+    end
+  end
 
 end

data/spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb

@@ -1,10 +1,14 @@
 describe Elasticity::SetupHadoopDebuggingStep do
 
-  it { should be_a Elasticity::CustomJarStep }
+  it 'should be a CustomJarStep' do
+    expect(subject).to be_a(Elasticity::CustomJarStep)
+  end
 
-  its(:name) { should == 'Elasticity Setup Hadoop Debugging' }
-  its(:jar) { should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar' }
-  its(:arguments) { should == ['s3://elasticmapreduce/libs/state-pusher/0.1/fetch'] }
-  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
+  it 'should set the appropriate fields' do
+    expect(subject.name).to eql('Elasticity Setup Hadoop Debugging')
+    expect(subject.jar).to eql('s3://elasticmapreduce/libs/script-runner/script-runner.jar')
+    expect(subject.arguments).to eql(['s3://elasticmapreduce/libs/state-pusher/0.1/fetch'])
+    expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
+  end
 
 end

data/spec/lib/elasticity/streaming_step_spec.rb

@@ -6,18 +6,22 @@ describe Elasticity::StreamingStep do
 
   it { should be_a Elasticity::JobFlowStep }
 
-  its(:name) { should == 'Elasticity Streaming Step' }
-  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
-  its(:input_bucket) { should == 'INPUT_BUCKET' }
-  its(:output_bucket) { should == 'OUTPUT_BUCKET' }
-  its(:mapper) { should == 'MAPPER' }
-  its(:reducer) { should == 'REDUCER' }
-  its(:arguments) { should == %w(-ARG1 VALUE1) }
+  describe '.initialize' do
+    it 'should set the fields appropriately' do
+      expect(subject.name).to eql('Elasticity Streaming Step')
+      expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
+      expect(subject.input_bucket).to eql('INPUT_BUCKET')
+      expect(subject.output_bucket).to eql('OUTPUT_BUCKET')
+      expect(subject.mapper).to eql('MAPPER')
+      expect(subject.reducer).to eql('REDUCER')
+      expect(subject.arguments).to eql(%w(-ARG1 VALUE1))
+    end
+  end
 
   describe '#to_aws_step' do
 
     it 'should convert to aws step format' do
-      subject.to_aws_step(Elasticity::JobFlow.new('_', '_')).should == {
+      subject.to_aws_step(Elasticity::JobFlow.new).should == {
         :name => 'Elasticity Streaming Step',
         :action_on_failure => 'TERMINATE_JOB_FLOW',
         :hadoop_jar_step => {
@@ -31,7 +35,7 @@ describe Elasticity::StreamingStep do
 
   describe '.requires_installation?' do
     it 'should not require installation' do
-      Elasticity::StreamingStep.requires_installation?.should be_false
+      expect(Elasticity::StreamingStep.requires_installation?).to be false
     end
   end
 

data/spec/spec_helper.rb

@@ -10,4 +10,12 @@ RSpec.configure do |config|
     Elasticity.default_configuration
   end
 
+  config.expect_with :rspec do |c|
+    c.syntax = [:should, :expect]
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.syntax = :should
+  end
+
 end

data/spec/support/factory_girl.rb

@@ -0,0 +1,8 @@
+require 'factory_girl'
+
+FactoryGirl.find_definitions
+
+RSpec.configure do |config|
+  config.include FactoryGirl::Syntax::Methods
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: 5.0.3
+  version: '6.0'
 platform: ruby
 authors:
 - Robert Slifka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-09 00:00:00.000000000 Z
+date: 2015-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: fog
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
@@ -39,33 +39,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: fog
+  name: unf
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: unf
+  name: factory_girl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
+        version: '4.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.14.0
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.14.0
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -142,11 +142,12 @@ files:
 - Rakefile
 - elasticity.gemspec
 - lib/elasticity.rb
-- lib/elasticity/aws_request_v2.rb
 - lib/elasticity/aws_request_v4.rb
 - lib/elasticity/aws_session.rb
 - lib/elasticity/aws_utils.rb
 - lib/elasticity/bootstrap_action.rb
+- lib/elasticity/cluster_status.rb
+- lib/elasticity/cluster_step_status.rb
 - lib/elasticity/custom_jar_step.rb
 - lib/elasticity/emr.rb
 - lib/elasticity/ganglia_bootstrap_action.rb
@@ -155,8 +156,6 @@ files:
 - lib/elasticity/hive_step.rb
 - lib/elasticity/instance_group.rb
 - lib/elasticity/job_flow.rb
-- lib/elasticity/job_flow_status.rb
-- lib/elasticity/job_flow_status_step.rb
 - lib/elasticity/job_flow_step.rb
 - lib/elasticity/looper.rb
 - lib/elasticity/pig_step.rb
@@ -166,11 +165,14 @@ files:
 - lib/elasticity/streaming_step.rb
 - lib/elasticity/sync_to_s3.rb
 - lib/elasticity/version.rb
-- spec/lib/elasticity/aws_request_v2_spec.rb
+- spec/factories/cluster_status_factory.rb
+- spec/factories/cluster_step_status_factory.rb
 - spec/lib/elasticity/aws_request_v4_spec.rb
 - spec/lib/elasticity/aws_session_spec.rb
 - spec/lib/elasticity/aws_utils_spec.rb
 - spec/lib/elasticity/bootstrap_action_spec.rb
+- spec/lib/elasticity/cluster_status_spec.rb
+- spec/lib/elasticity/cluster_step_status_spec.rb
 - spec/lib/elasticity/custom_jar_step_spec.rb
 - spec/lib/elasticity/emr_spec.rb
 - spec/lib/elasticity/ganglia_bootstrap_action_spec.rb
@@ -180,8 +182,6 @@ files:
 - spec/lib/elasticity/instance_group_spec.rb
 - spec/lib/elasticity/job_flow_integration_spec.rb
 - spec/lib/elasticity/job_flow_spec.rb
-- spec/lib/elasticity/job_flow_status_spec.rb
-- spec/lib/elasticity/job_flow_status_step_spec.rb
 - spec/lib/elasticity/job_flow_step_spec.rb
 - spec/lib/elasticity/looper_spec.rb
 - spec/lib/elasticity/pig_step_spec.rb
@@ -192,6 +192,7 @@ files:
 - spec/lib/elasticity/sync_to_s3_spec.rb
 - spec/spec_helper.rb
 - spec/support/be_a_hash_including_matcher.rb
+- spec/support/factory_girl.rb
 homepage: http://www.github.com/rslifka/elasticity
 licenses: []
 metadata: {}
@@ -216,11 +217,14 @@ signing_key:
 specification_version: 4
 summary: Streamlined, programmatic access to Amazon's Elastic Map Reduce service.
 test_files:
-- spec/lib/elasticity/aws_request_v2_spec.rb
+- spec/factories/cluster_status_factory.rb
+- spec/factories/cluster_step_status_factory.rb
 - spec/lib/elasticity/aws_request_v4_spec.rb
 - spec/lib/elasticity/aws_session_spec.rb
 - spec/lib/elasticity/aws_utils_spec.rb
 - spec/lib/elasticity/bootstrap_action_spec.rb
+- spec/lib/elasticity/cluster_status_spec.rb
+- spec/lib/elasticity/cluster_step_status_spec.rb
 - spec/lib/elasticity/custom_jar_step_spec.rb
 - spec/lib/elasticity/emr_spec.rb
 - spec/lib/elasticity/ganglia_bootstrap_action_spec.rb
@@ -230,8 +234,6 @@ test_files:
 - spec/lib/elasticity/instance_group_spec.rb
 - spec/lib/elasticity/job_flow_integration_spec.rb
 - spec/lib/elasticity/job_flow_spec.rb
-- spec/lib/elasticity/job_flow_status_spec.rb
-- spec/lib/elasticity/job_flow_status_step_spec.rb
 - spec/lib/elasticity/job_flow_step_spec.rb
 - spec/lib/elasticity/looper_spec.rb
 - spec/lib/elasticity/pig_step_spec.rb
@@ -242,3 +244,4 @@ test_files:
 - spec/lib/elasticity/sync_to_s3_spec.rb
 - spec/spec_helper.rb
 - spec/support/be_a_hash_including_matcher.rb
+- spec/support/factory_girl.rb

data/lib/elasticity/aws_request_v2.rb

@@ -1,42 +0,0 @@
-module Elasticity
-
-  class AwsRequestV2
-
-    def initialize(aws_session, ruby_service_hash)
-      @aws_session = aws_session
-      @ruby_service_hash = ruby_service_hash
-    end
-
-    def url
-      "https://elasticmapreduce.#{@aws_session.region}.amazonaws.com"
-    end
-
-    def headers
-      {
-        :content_type => 'application/x-www-form-urlencoded; charset=utf-8'
-      }
-    end
-
-    # (Used from RightScale's right_aws gem.)
-    # EC2, SQS, SDB and EMR requests must be signed by this guy.
-    # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
-    #      http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
-    def payload
-      service_hash = AwsUtils.convert_ruby_to_aws(@ruby_service_hash)
-      service_hash.merge!({
-          'AWSAccessKeyId' => @aws_session.access_key,
-          'Timestamp' => Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.000Z'),
-          'SignatureVersion' => '2',
-          'SignatureMethod' => 'HmacSHA256'
-        })
-      canonical_string = service_hash.keys.sort.map do |key|
-        "#{AwsUtils.aws_escape(key)}=#{AwsUtils.aws_escape(service_hash[key])}"
-      end.join('&')
-      string_to_sign = "POST\n#{@aws_session.host.downcase}\n/\n#{canonical_string}"
-      signature = AwsUtils.aws_escape(Base64.encode64(OpenSSL::HMAC.digest('sha256', @aws_session.secret_key, string_to_sign)).strip)
-      "#{canonical_string}&Signature=#{signature}"
-    end
-
-  end
-
-end

data/lib/elasticity/job_flow_status.rb

@@ -1,91 +0,0 @@
-module Elasticity
-
-  class JobFlowStatus
-
-    attr_accessor :name
-    attr_accessor :jobflow_id
-    attr_accessor :state
-    attr_accessor :steps
-    attr_accessor :created_at
-    attr_accessor :started_at
-    attr_accessor :ready_at
-    attr_accessor :ended_at
-    attr_accessor :duration
-    attr_accessor :instance_count
-    attr_accessor :master_instance_type
-    attr_accessor :master_instance_id
-    attr_accessor :slave_instance_type
-    attr_accessor :last_state_change_reason
-    attr_accessor :installed_steps
-    attr_accessor :master_public_dns_name
-    attr_accessor :normalized_instance_hours
-
-    def initialize
-      @steps = []
-      @installed_steps = []
-    end
-
-    # http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/ProcessingCycle.html
-    def active?
-      %w{RUNNING STARTING BOOTSTRAPPING WAITING SHUTTING_DOWN}.include? state
-    end
-
-    # Create a jobflow from an AWS <member> (Nokogiri::XML::Element):
-    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member
-    def self.from_member_element(xml_element)
-      jobflow_status = JobFlowStatus.new
-
-      jobflow_status.name = xml_element.xpath('./Name').text.strip
-      jobflow_status.jobflow_id = xml_element.xpath('./JobFlowId').text.strip
-      jobflow_status.state = xml_element.xpath('./ExecutionStatusDetail/State').text.strip
-      jobflow_status.last_state_change_reason = xml_element.xpath('./ExecutionStatusDetail/LastStateChangeReason').text.strip
-
-      jobflow_status.steps = JobFlowStatusStep.from_members_nodeset(xml_element.xpath('./Steps/member'))
-
-      step_names = jobflow_status.steps.map(&:name)
-      Elasticity::JobFlowStep.steps_requiring_installation.each do |step|
-        jobflow_status.installed_steps << step if step_names.include?(step.aws_installation_step_name)
-      end
-
-      jobflow_status.created_at = Time.parse(xml_element.xpath('./ExecutionStatusDetail/CreationDateTime').text.strip)
-
-      ready_at = xml_element.xpath('./ExecutionStatusDetail/ReadyDateTime').text.strip
-      jobflow_status.ready_at = (ready_at == '') ? (nil) : (Time.parse(ready_at))
-
-      started_at = xml_element.xpath('./ExecutionStatusDetail/StartDateTime').text.strip
-      jobflow_status.started_at = (started_at == '') ? (nil) : (Time.parse(started_at))
-
-      ended_at = xml_element.xpath('./ExecutionStatusDetail/EndDateTime').text.strip
-      jobflow_status.ended_at = (ended_at == '') ? (nil) : (Time.parse(ended_at))
-
-      if jobflow_status.ended_at && jobflow_status.started_at
-        jobflow_status.duration = ((jobflow_status.ended_at - jobflow_status.started_at) / 60).to_i
-      end
-
-      jobflow_status.instance_count = xml_element.xpath('./Instances/InstanceCount').text.strip
-      jobflow_status.master_instance_type = xml_element.xpath('./Instances/MasterInstanceType').text.strip
-      master_instance_id = xml_element.xpath('./Instances/MasterInstanceId').text.strip
-      jobflow_status.master_instance_id = (master_instance_id == '') ? (nil) : (master_instance_id)
-      jobflow_status.slave_instance_type = xml_element.xpath('./Instances/SlaveInstanceType').text.strip
-
-      master_public_dns_name = xml_element.xpath('./Instances/MasterPublicDnsName').text.strip
-      jobflow_status.master_public_dns_name = (master_public_dns_name == '') ? (nil) : (master_public_dns_name)
-
-      jobflow_status.normalized_instance_hours = xml_element.xpath('./Instances/NormalizedInstanceHours').text.strip
-
-      jobflow_status
-    end
-
-    # Create JobFlows from a collection of AWS <member> nodes (Nokogiri::XML::NodeSet):
-    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows
-    def self.from_members_nodeset(members_nodeset)
-      jobflow_statuses = []
-      members_nodeset.each do |member|
-        jobflow_statuses << from_member_element(member)
-      end
-      jobflow_statuses
-    end
-
-  end
-
-end