elasticity 2.3 → 2.3.1

data/HISTORY.md

@@ -1,3 +1,10 @@
+## 2.3.1 - August 23, 2012
+
++ Birthday release! ;)
++ Bumped the default version of Hadoop to 1.0.3.
++ Amazon now requires the --hive-versions argument when installing Hive (thanks to Johannes Wuerbach).
++ ```JobFlowStatus#master_public_dns_name``` is now available (thanks to Johannes Wuerbach).
+
 ## 2.3 - July 28, 2012
 
 + ```JobFlow``` now supports specifying availbility zone instance specification via ```JobFlow#placement=```.

data/README.md

@@ -35,7 +35,7 @@ jobflow = Elasticity::JobFlow.new('AWS access key', 'AWS secret key')
 step = Elasticity::CustomJarStep.new('s3n://elasticmapreduce/samples/cloudburst/cloudburst.jar')
 
 # Here are the arguments to pass to the jar
-c.arguments = %w(s3n://elasticmapreduce/samples/cloudburst/input/s_suis.br s3n://elasticmapreduce/samples/cloudburst/input/100k.br s3n://slif-output/cloudburst/output/2012-06-22 36 3 0 1 240 48 24 24 128 16)
+step.arguments = %w(s3n://elasticmapreduce/samples/cloudburst/input/s_suis.br s3n://elasticmapreduce/samples/cloudburst/input/100k.br s3n://slif-output/cloudburst/output/2012-06-22 36 3 0 1 240 48 24 24 128 16)
 
 # Add the step to the jobflow
 jobflow.add_step(step)
@@ -126,7 +126,6 @@ ig.count = 10                       # Provision 10 instances
 ig.type  = 'c1.medium'              # See the EMR docs for a list of supported types
 ig.set_on_demand_instances          # This is the default setting
 
-
 jobflow.set_core_instance_group(ig)
 ```
 
@@ -140,7 +139,6 @@ ig.count = 10                       # Provision 10 instances
 ig.type  = 'c1.medium'              # See the EMR docs for a list of supported types
 ig.set_spot_instances(0.25)         # Makes this a SPOT group with a $0.25 bid price
 
-
 jobflow.set_core_instance_group(ig)
 ```
 

data/lib/elasticity/custom_jar_step.rb

@@ -2,7 +2,7 @@ module Elasticity
 
   class CustomJarStep
 
-    include JobFlowStep
+    include Elasticity::JobFlowStep
 
     attr_accessor :name
     attr_accessor :jar

data/lib/elasticity/hive_step.rb

@@ -2,7 +2,7 @@ module Elasticity
 
   class HiveStep
 
-    include JobFlowStep
+    include Elasticity::JobFlowStep
 
     attr_accessor :name
     attr_accessor :script
@@ -45,7 +45,9 @@ module Elasticity
             's3://elasticmapreduce/libs/hive/hive-script',
               '--base-path',
               's3://elasticmapreduce/libs/hive/',
-              '--install-hive'
+              '--install-hive',
+              '--hive-versions',
+              'latest'
           ],
         },
         :name => 'Elasticity - Install Hive'

data/lib/elasticity/job_flow.rb

@@ -21,8 +21,7 @@ module Elasticity
 
     def initialize(access, secret)
       @action_on_failure = 'TERMINATE_JOB_FLOW'
-      @ec2_key_name = 'default'
-      @hadoop_version = '0.20.205'
+      @hadoop_version = '1.0.3'
       @name = 'Elasticity Job Flow'
       @ami_version = 'latest'
       @keep_job_flow_alive_when_no_steps = false
@@ -144,12 +143,12 @@ module Elasticity
         :ami_version => @ami_version,
         :instances => {
           :keep_job_flow_alive_when_no_steps => @keep_job_flow_alive_when_no_steps,
-          :ec2_key_name => @ec2_key_name,
           :hadoop_version => @hadoop_version,
           :instance_groups => jobflow_instance_groups
         }
       }
       preamble.merge!(:ec2_subnet_id => @ec2_subnet_id) if @ec2_subnet_id
+      preamble[:instances].merge!(:ec2_key_name => @ec2_key_name) if @ec2_key_name
       preamble
     end
 

data/lib/elasticity/job_flow_status.rb

@@ -14,6 +14,7 @@ module Elasticity
     attr_accessor :slave_instance_type
     attr_accessor :last_state_change_reason
     attr_accessor :installed_steps
+    attr_accessor :master_public_dns_name
 
     def initialize
       @steps = []
@@ -49,6 +50,9 @@ module Elasticity
       jobflow.master_instance_type = xml_element.xpath('./Instances/MasterInstanceType').text.strip
       jobflow.slave_instance_type = xml_element.xpath('./Instances/SlaveInstanceType').text.strip
 
+      master_public_dns_name = xml_element.xpath('./Instances/MasterPublicDnsName').text.strip
+      jobflow.master_public_dns_name = (master_public_dns_name == '') ? (nil) : (master_public_dns_name)
+
       jobflow
     end
 

data/lib/elasticity/pig_step.rb

@@ -2,7 +2,7 @@ module Elasticity
 
   class PigStep
 
-    include JobFlowStep
+    include Elasticity::JobFlowStep
 
     attr_accessor :name
     attr_accessor :script

data/lib/elasticity/streaming_step.rb

@@ -2,7 +2,7 @@ module Elasticity
 
   class StreamingStep
 
-    include JobFlowStep
+    include Elasticity::JobFlowStep
 
     attr_accessor :name
     attr_accessor :action_on_failure

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '2.3'
+  VERSION = '2.3.1'
 end

data/spec/lib/elasticity/hive_step_spec.rb

@@ -8,7 +8,7 @@ describe Elasticity::HiveStep do
 
   its(:name) { should == 'Elasticity Hive Step (script.hql)' }
   its(:script) { should == 'script.hql' }
-  its(:variables) { should == { } }
+  its(:variables) { should == {} }
   its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
 
   describe '#to_aws_step' do
@@ -62,7 +62,9 @@ describe Elasticity::HiveStep do
             's3://elasticmapreduce/libs/hive/hive-script',
               '--base-path',
               's3://elasticmapreduce/libs/hive/',
-              '--install-hive'
+              '--install-hive',
+              '--hive-versions',
+              'latest'
           ],
         },
         :name => 'Elasticity - Install Hive'

data/spec/lib/elasticity/job_flow_integration_spec.rb

@@ -30,8 +30,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
         :ami_version => 'latest',
         :instances => {
           :keep_job_flow_alive_when_no_steps => false,
-          :ec2_key_name => 'default',
-          :hadoop_version => '0.20.205',
+          :hadoop_version => '1.0.3',
           :instance_groups => [
             {
               :instance_count => 1,
@@ -56,7 +55,9 @@ describe 'Elasticity::JobFlow Integration Examples' do
                 's3://elasticmapreduce/libs/hive/hive-script',
                   '--base-path',
                   's3://elasticmapreduce/libs/hive/',
-                  '--install-hive'
+                  '--install-hive',
+                  '--hive-versions',
+                  'latest'
               ],
             },
             :name => 'Elasticity - Install Hive'
@@ -109,8 +110,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
         :ami_version => 'latest',
         :instances => {
           :keep_job_flow_alive_when_no_steps => false,
-          :ec2_key_name => 'default',
-          :hadoop_version => '0.20.205',
+          :hadoop_version => '1.0.3',
           :instance_groups => [
             {
               :instance_count => 1,
@@ -192,8 +192,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
         :ami_version => 'latest',
         :instances => {
           :keep_job_flow_alive_when_no_steps => false,
-          :ec2_key_name => 'default',
-          :hadoop_version => '0.20.205',
+          :hadoop_version => '1.0.3',
           :instance_groups => [
             {
               :instance_count => 1,

data/spec/lib/elasticity/job_flow_spec.rb

@@ -5,8 +5,9 @@ describe Elasticity::JobFlow do
   end
 
   its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
-  its(:ec2_key_name) { should == 'default' }
-  its(:hadoop_version) { should == '0.20.205' }
+  its(:ec2_key_name) { should == nil }
+  its(:ec2_subnet_id) { should == nil }
+  its(:hadoop_version) { should == '1.0.3' }
   its(:instance_count) { should == 2 }
   its(:log_uri) { should == nil }
   its(:master_instance_type) { should == 'm1.small' }
@@ -340,8 +341,7 @@ describe Elasticity::JobFlow do
         :ami_version => 'latest',
         :instances => {
           :keep_job_flow_alive_when_no_steps => false,
-          :ec2_key_name => 'default',
-          :hadoop_version => '0.20.205',
+          :hadoop_version => '1.0.3',
           :instance_groups => ['INSTANCE_GROUP_CONFIGURATION']
         }
       }
@@ -355,6 +355,13 @@ describe Elasticity::JobFlow do
       subject.send(:jobflow_preamble).should == basic_preamble
     end
 
+    context 'when a key name is provided' do
+      it 'should include it in the preamble' do
+        subject.ec2_key_name = 'default'
+        subject.send(:jobflow_preamble)[:instances].should be_a_hash_including({:ec2_key_name => 'default'})
+      end
+    end
+
     context 'when a VPC subnet ID is specified' do
       it 'should include it in the preamble' do
         subject.ec2_subnet_id = 'subnet-118b9d79'

data/spec/lib/elasticity/job_flow_status_spec.rb

@@ -65,6 +65,9 @@ describe Elasticity::JobFlowStatus do
                 </member>
               </Steps>
               <Instances>
+                 <MasterPublicDnsName>
+                   ec2-107-22-77-99.compute-1.amazonaws.com
+                 </MasterPublicDnsName>
                  <Placement>
                     <AvailabilityZone>
                       eu-west-1a
@@ -154,6 +157,7 @@ describe Elasticity::JobFlowStatus do
       single_jobflow.slave_instance_type.should == 'm1.small'
       single_jobflow.instance_count.should == '4'
       single_jobflow.last_state_change_reason.should == 'Steps completed with errors'
+      single_jobflow.master_public_dns_name.should == 'ec2-107-22-77-99.compute-1.amazonaws.com'
     end
   end
 
@@ -169,6 +173,7 @@ describe Elasticity::JobFlowStatus do
       multiple_jobflows.map(&:slave_instance_type).should == %w(m1.small c1.medium)
       multiple_jobflows.map(&:instance_count).should == %w(4 2)
       multiple_jobflows.map(&:last_state_change_reason).should == ['Steps completed with errors', 'Steps completed']
+      multiple_jobflows.map(&:master_public_dns_name).should == ['ec2-107-22-77-99.compute-1.amazonaws.com', nil]
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: '2.3'
+  version: 2.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-29 00:00:00.000000000 Z
+date: 2012-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -135,12 +135,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 812718050856923137
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 812718050856923137
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24