elasticity 2.3 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/HISTORY.md CHANGED
@@ -1,3 +1,10 @@
1
+ ## 2.3.1 - August 23, 2012
2
+
3
+ + Birthday release! ;)
4
+ + Bumped the default version of Hadoop to 1.0.3.
5
+ + Amazon now requires the --hive-versions argument when installing Hive (thanks to Johannes Wuerbach).
6
+ + ```JobFlowStatus#master_public_dns_name``` is now available (thanks to Johannes Wuerbach).
7
+
1
8
  ## 2.3 - July 28, 2012
2
9
 
3
10
  + ```JobFlow``` now supports specifying availbility zone instance specification via ```JobFlow#placement=```.
data/README.md CHANGED
@@ -35,7 +35,7 @@ jobflow = Elasticity::JobFlow.new('AWS access key', 'AWS secret key')
35
35
  step = Elasticity::CustomJarStep.new('s3n://elasticmapreduce/samples/cloudburst/cloudburst.jar')
36
36
 
37
37
  # Here are the arguments to pass to the jar
38
- c.arguments = %w(s3n://elasticmapreduce/samples/cloudburst/input/s_suis.br s3n://elasticmapreduce/samples/cloudburst/input/100k.br s3n://slif-output/cloudburst/output/2012-06-22 36 3 0 1 240 48 24 24 128 16)
38
+ step.arguments = %w(s3n://elasticmapreduce/samples/cloudburst/input/s_suis.br s3n://elasticmapreduce/samples/cloudburst/input/100k.br s3n://slif-output/cloudburst/output/2012-06-22 36 3 0 1 240 48 24 24 128 16)
39
39
 
40
40
  # Add the step to the jobflow
41
41
  jobflow.add_step(step)
@@ -126,7 +126,6 @@ ig.count = 10 # Provision 10 instances
126
126
  ig.type = 'c1.medium' # See the EMR docs for a list of supported types
127
127
  ig.set_on_demand_instances # This is the default setting
128
128
 
129
-
130
129
  jobflow.set_core_instance_group(ig)
131
130
  ```
132
131
 
@@ -140,7 +139,6 @@ ig.count = 10 # Provision 10 instances
140
139
  ig.type = 'c1.medium' # See the EMR docs for a list of supported types
141
140
  ig.set_spot_instances(0.25) # Makes this a SPOT group with a $0.25 bid price
142
141
 
143
-
144
142
  jobflow.set_core_instance_group(ig)
145
143
  ```
146
144
 
@@ -2,7 +2,7 @@ module Elasticity
2
2
 
3
3
  class CustomJarStep
4
4
 
5
- include JobFlowStep
5
+ include Elasticity::JobFlowStep
6
6
 
7
7
  attr_accessor :name
8
8
  attr_accessor :jar
@@ -2,7 +2,7 @@ module Elasticity
2
2
 
3
3
  class HiveStep
4
4
 
5
- include JobFlowStep
5
+ include Elasticity::JobFlowStep
6
6
 
7
7
  attr_accessor :name
8
8
  attr_accessor :script
@@ -45,7 +45,9 @@ module Elasticity
45
45
  's3://elasticmapreduce/libs/hive/hive-script',
46
46
  '--base-path',
47
47
  's3://elasticmapreduce/libs/hive/',
48
- '--install-hive'
48
+ '--install-hive',
49
+ '--hive-versions',
50
+ 'latest'
49
51
  ],
50
52
  },
51
53
  :name => 'Elasticity - Install Hive'
@@ -21,8 +21,7 @@ module Elasticity
21
21
 
22
22
  def initialize(access, secret)
23
23
  @action_on_failure = 'TERMINATE_JOB_FLOW'
24
- @ec2_key_name = 'default'
25
- @hadoop_version = '0.20.205'
24
+ @hadoop_version = '1.0.3'
26
25
  @name = 'Elasticity Job Flow'
27
26
  @ami_version = 'latest'
28
27
  @keep_job_flow_alive_when_no_steps = false
@@ -144,12 +143,12 @@ module Elasticity
144
143
  :ami_version => @ami_version,
145
144
  :instances => {
146
145
  :keep_job_flow_alive_when_no_steps => @keep_job_flow_alive_when_no_steps,
147
- :ec2_key_name => @ec2_key_name,
148
146
  :hadoop_version => @hadoop_version,
149
147
  :instance_groups => jobflow_instance_groups
150
148
  }
151
149
  }
152
150
  preamble.merge!(:ec2_subnet_id => @ec2_subnet_id) if @ec2_subnet_id
151
+ preamble[:instances].merge!(:ec2_key_name => @ec2_key_name) if @ec2_key_name
153
152
  preamble
154
153
  end
155
154
 
@@ -14,6 +14,7 @@ module Elasticity
14
14
  attr_accessor :slave_instance_type
15
15
  attr_accessor :last_state_change_reason
16
16
  attr_accessor :installed_steps
17
+ attr_accessor :master_public_dns_name
17
18
 
18
19
  def initialize
19
20
  @steps = []
@@ -49,6 +50,9 @@ module Elasticity
49
50
  jobflow.master_instance_type = xml_element.xpath('./Instances/MasterInstanceType').text.strip
50
51
  jobflow.slave_instance_type = xml_element.xpath('./Instances/SlaveInstanceType').text.strip
51
52
 
53
+ master_public_dns_name = xml_element.xpath('./Instances/MasterPublicDnsName').text.strip
54
+ jobflow.master_public_dns_name = (master_public_dns_name == '') ? (nil) : (master_public_dns_name)
55
+
52
56
  jobflow
53
57
  end
54
58
 
@@ -2,7 +2,7 @@ module Elasticity
2
2
 
3
3
  class PigStep
4
4
 
5
- include JobFlowStep
5
+ include Elasticity::JobFlowStep
6
6
 
7
7
  attr_accessor :name
8
8
  attr_accessor :script
@@ -2,7 +2,7 @@ module Elasticity
2
2
 
3
3
  class StreamingStep
4
4
 
5
- include JobFlowStep
5
+ include Elasticity::JobFlowStep
6
6
 
7
7
  attr_accessor :name
8
8
  attr_accessor :action_on_failure
@@ -1,3 +1,3 @@
1
1
  module Elasticity
2
- VERSION = '2.3'
2
+ VERSION = '2.3.1'
3
3
  end
@@ -8,7 +8,7 @@ describe Elasticity::HiveStep do
8
8
 
9
9
  its(:name) { should == 'Elasticity Hive Step (script.hql)' }
10
10
  its(:script) { should == 'script.hql' }
11
- its(:variables) { should == { } }
11
+ its(:variables) { should == {} }
12
12
  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
13
13
 
14
14
  describe '#to_aws_step' do
@@ -62,7 +62,9 @@ describe Elasticity::HiveStep do
62
62
  's3://elasticmapreduce/libs/hive/hive-script',
63
63
  '--base-path',
64
64
  's3://elasticmapreduce/libs/hive/',
65
- '--install-hive'
65
+ '--install-hive',
66
+ '--hive-versions',
67
+ 'latest'
66
68
  ],
67
69
  },
68
70
  :name => 'Elasticity - Install Hive'
@@ -30,8 +30,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
30
30
  :ami_version => 'latest',
31
31
  :instances => {
32
32
  :keep_job_flow_alive_when_no_steps => false,
33
- :ec2_key_name => 'default',
34
- :hadoop_version => '0.20.205',
33
+ :hadoop_version => '1.0.3',
35
34
  :instance_groups => [
36
35
  {
37
36
  :instance_count => 1,
@@ -56,7 +55,9 @@ describe 'Elasticity::JobFlow Integration Examples' do
56
55
  's3://elasticmapreduce/libs/hive/hive-script',
57
56
  '--base-path',
58
57
  's3://elasticmapreduce/libs/hive/',
59
- '--install-hive'
58
+ '--install-hive',
59
+ '--hive-versions',
60
+ 'latest'
60
61
  ],
61
62
  },
62
63
  :name => 'Elasticity - Install Hive'
@@ -109,8 +110,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
109
110
  :ami_version => 'latest',
110
111
  :instances => {
111
112
  :keep_job_flow_alive_when_no_steps => false,
112
- :ec2_key_name => 'default',
113
- :hadoop_version => '0.20.205',
113
+ :hadoop_version => '1.0.3',
114
114
  :instance_groups => [
115
115
  {
116
116
  :instance_count => 1,
@@ -192,8 +192,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
192
192
  :ami_version => 'latest',
193
193
  :instances => {
194
194
  :keep_job_flow_alive_when_no_steps => false,
195
- :ec2_key_name => 'default',
196
- :hadoop_version => '0.20.205',
195
+ :hadoop_version => '1.0.3',
197
196
  :instance_groups => [
198
197
  {
199
198
  :instance_count => 1,
@@ -5,8 +5,9 @@ describe Elasticity::JobFlow do
5
5
  end
6
6
 
7
7
  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
8
- its(:ec2_key_name) { should == 'default' }
9
- its(:hadoop_version) { should == '0.20.205' }
8
+ its(:ec2_key_name) { should == nil }
9
+ its(:ec2_subnet_id) { should == nil }
10
+ its(:hadoop_version) { should == '1.0.3' }
10
11
  its(:instance_count) { should == 2 }
11
12
  its(:log_uri) { should == nil }
12
13
  its(:master_instance_type) { should == 'm1.small' }
@@ -340,8 +341,7 @@ describe Elasticity::JobFlow do
340
341
  :ami_version => 'latest',
341
342
  :instances => {
342
343
  :keep_job_flow_alive_when_no_steps => false,
343
- :ec2_key_name => 'default',
344
- :hadoop_version => '0.20.205',
344
+ :hadoop_version => '1.0.3',
345
345
  :instance_groups => ['INSTANCE_GROUP_CONFIGURATION']
346
346
  }
347
347
  }
@@ -355,6 +355,13 @@ describe Elasticity::JobFlow do
355
355
  subject.send(:jobflow_preamble).should == basic_preamble
356
356
  end
357
357
 
358
+ context 'when a key name is provided' do
359
+ it 'should include it in the preamble' do
360
+ subject.ec2_key_name = 'default'
361
+ subject.send(:jobflow_preamble)[:instances].should be_a_hash_including({:ec2_key_name => 'default'})
362
+ end
363
+ end
364
+
358
365
  context 'when a VPC subnet ID is specified' do
359
366
  it 'should include it in the preamble' do
360
367
  subject.ec2_subnet_id = 'subnet-118b9d79'
@@ -65,6 +65,9 @@ describe Elasticity::JobFlowStatus do
65
65
  </member>
66
66
  </Steps>
67
67
  <Instances>
68
+ <MasterPublicDnsName>
69
+ ec2-107-22-77-99.compute-1.amazonaws.com
70
+ </MasterPublicDnsName>
68
71
  <Placement>
69
72
  <AvailabilityZone>
70
73
  eu-west-1a
@@ -154,6 +157,7 @@ describe Elasticity::JobFlowStatus do
154
157
  single_jobflow.slave_instance_type.should == 'm1.small'
155
158
  single_jobflow.instance_count.should == '4'
156
159
  single_jobflow.last_state_change_reason.should == 'Steps completed with errors'
160
+ single_jobflow.master_public_dns_name.should == 'ec2-107-22-77-99.compute-1.amazonaws.com'
157
161
  end
158
162
  end
159
163
 
@@ -169,6 +173,7 @@ describe Elasticity::JobFlowStatus do
169
173
  multiple_jobflows.map(&:slave_instance_type).should == %w(m1.small c1.medium)
170
174
  multiple_jobflows.map(&:instance_count).should == %w(4 2)
171
175
  multiple_jobflows.map(&:last_state_change_reason).should == ['Steps completed with errors', 'Steps completed']
176
+ multiple_jobflows.map(&:master_public_dns_name).should == ['ec2-107-22-77-99.compute-1.amazonaws.com', nil]
172
177
  end
173
178
  end
174
179
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elasticity
3
3
  version: !ruby/object:Gem::Version
4
- version: '2.3'
4
+ version: 2.3.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-07-29 00:00:00.000000000 Z
12
+ date: 2012-08-23 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rest-client
@@ -135,12 +135,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
135
135
  - - ! '>='
136
136
  - !ruby/object:Gem::Version
137
137
  version: '0'
138
+ segments:
139
+ - 0
140
+ hash: 812718050856923137
138
141
  required_rubygems_version: !ruby/object:Gem::Requirement
139
142
  none: false
140
143
  requirements:
141
144
  - - ! '>='
142
145
  - !ruby/object:Gem::Version
143
146
  version: '0'
147
+ segments:
148
+ - 0
149
+ hash: 812718050856923137
144
150
  requirements: []
145
151
  rubyforge_project:
146
152
  rubygems_version: 1.8.24