elasticity 3.0.4 → 4.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MmY0NTJjZmJkMmVhNTYwNTJlZmY3MjAwOTE0NDhmYjczY2MxNzAwOA==
+    MmZlODg3MzM2NmYzOGQyZjdiYjcyZjZkNTA2MThmMWZlZTQzYmM5Mw==
   data.tar.gz: !binary |-
-    MmIwNDRmMjM3MTA3NzliZDM4ZWY4MGZmODRlNTYwODlhYTMwYmYwZQ==
+    ODA1ZWJiMDM5YzQ5YTBjY2Y3MzY0NGFmY2Y5MTA3MWFjNjAxNTUzNA==
 SHA512:
   metadata.gz: !binary |-
-    YzI5NzgyMjc5YTlmZjljMjY5MzQ2MzU4ZmZiOWFiYmViMDgwMmYyNDJiZjE2
-    NzY4MTllZGFlN2Q1ZjcyNTVjZGIxYWQ2NTNmMmE4NDE2MTNmOTc1ODcyYjZl
-    MTljMDM3OWZhNWM2OThkYjBkMjI1MGE4YzgyMWZkZjQ3M2ZlMDU=
+    OGRjMjY3Y2M3MWQxNTRjNmJiNDM1NGI3YTAyNDc5NTUxYmY2MjNkYzk0YmJj
+    ZjcxMDExYWNlMzY1M2MxODVkMWUyMmM5NmFhYzJjZTdlNmU5YmJmYzU4MWFj
+    YWIzM2ZkN2U3ZTZhMmZmYjc1NWYyODYzZjY1OTgyYjAxZTM4M2E=
   data.tar.gz: !binary |-
-    NmU2N2MxM2EyODdhZGMyYzQ2MTA1YmQ4ZGFiMTI2MTM3ZGQ0Mzg0MGM3ODg3
-    MjQ2NTljMTg5NzYxODZhM2UyNzQ2NTZjMTU1MmIxOTRmNWJiNmVmYmNlMTBi
-    MDg3ZjlmZTE1ZTUyODBmMDQ0MjI0NzBhN2ZmOGFhMGRmZTI2MmU=
+    OWNjMjg3ZTI5NzJhNWVhZmQ2NDg0ZDViZmRhMDEwYWQ1ZTI4YWJjNTEwODE1
+    YmFjNzRlYWFjZTM4YjNhOTE2ZjhjNjQxOTE3MDI3ZDUyMTFjNGJkOTUwOGY5
+    NzlkZTQ4NjE1Nzk5MzE5M2VmOWMyMGEyM2RhYjM0ZGY4M2FmYWU=

data/.travis.yml

@@ -2,5 +2,5 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.0
-  - 2.1.1
+  - 2.1.1
+  - jruby-19mode

data/HISTORY.md

@@ -1,3 +1,9 @@
+## 4.0 - May 21, 2014
+
+- Fix for issue [#69](https://github.com/rslifka/elasticity/issues/69).  The AWS region was previously being derived from the placement.  With the advent of VPC/subnet IDs being set, placement is not always relevant (as the VPC subnet ID implicitly defines a placement).  Since region cannot be derived it is now available directly on `JobFlow`.
+- Fix for issue [#79](https://github.com/rslifka/elasticity/issues/70).  Removing the ability to set the now deprecated Hadoop version.  It is now set via specifying the AMI version.  See the [EMR docs](http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/emr-plan-hadoop-version.html) for more details.
+- Building against JRuby 1.8.  While I don't necessarily have the resources to optimize for JRuby compatibility (given that I know next to nothing about it :) I'll use this as a canary to tackle any low-hanging issues that may arise.
+
 ## 3.0.4 - April 30, 2014
 
 - Fix for issues [#65](https://github.com/rslifka/elasticity/issues/65) and [#66](https://github.com/rslifka/elasticity/issues/66).

data/README.md

@@ -1,10 +1,8 @@
 [![Gem Version](https://badge.fury.io/rb/elasticity.png)](http://badge.fury.io/rb/elasticity)
 
-**(April 10, 2014)** Taking requests! I have a few ideas for what might be cool features though I'd rather work on what the community wants.  Go ahead and file an issue!
-
 Elasticity provides programmatic access to Amazon's Elastic Map Reduce service.  The aim is to conveniently abstract away the complex EMR REST API and make working with job flows more productive and more enjoyable.
 
-[![Build Status](https://secure.travis-ci.org/rslifka/elasticity.png)](http://travis-ci.org/rslifka/elasticity) 1.9.3, 2.0.0, 2.1.0, 2.1.1
+[![Build Status](https://secure.travis-ci.org/rslifka/elasticity.png)](http://travis-ci.org/rslifka/elasticity) 1.9.3, 2.0.0, 2.1.0, 2.1.1, JRuby 1.9
 
 Elasticity provides two ways to access EMR:
 
@@ -20,7 +18,7 @@ gem install elasticity
 or in your Gemfile
 
 ```
-gem 'elasticity', '~> 3.0'
+gem 'elasticity', '~> 4.0'
 ```
 
 This will ensure that you protect yourself from API changes, which will only be made in major revisions.
@@ -112,13 +110,13 @@ jobflow.ec2_subnet_id                     = nil
 jobflow.action_on_failure                 = 'TERMINATE_JOB_FLOW'
 jobflow.keep_job_flow_alive_when_no_steps = false
 jobflow.ami_version                       = 'latest'
-jobflow.hadoop_version                    = '1.0.3'
 jobflow.log_uri                           = nil
 jobflow.enable_debugging                  = false # Requires a log_uri to enable
 
 jobflow.ec2_key_name                      = nil
 jobflow.visible_to_all_users              = false
 jobflow.placement                         = 'us-east-1a'
+jobflow.region                            = 'us-east-1'
 jobflow.instance_count                    = 2
 jobflow.master_instance_type              = 'm1.small'
 jobflow.slave_instance_type               = 'm1.small'
@@ -412,7 +410,7 @@ Unfortunately, the documentation is sometimes incorrect and sometimes missing.
 # License
 
 ```
-  Copyright 2011-2013 Robert Slifka
+  Copyright 2011-2014 Robert Slifka
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.

data/lib/elasticity/aws_request.rb

@@ -2,6 +2,8 @@ module Elasticity
 
   class MissingKeyError < StandardError;
   end
+  class MissingRegionError < StandardError;
+  end
 
   class AwsRequest
 
@@ -14,6 +16,11 @@ module Elasticity
     #  :region - AWS region (e.g. us-west-1)
     #  :secure - true or false, default true.
     def initialize(access=nil, secret=nil, options={})
+      # There is a cryptic error if this isn't set
+      if options.has_key?(:region) && options[:region] == nil
+        raise MissingRegionError, 'A valid :region is required to connect to EMR'
+      end
+
       @access_key = get_access_key(access)
       @secret_key = get_secret_key(secret)
       @host = "elasticmapreduce.#{{:region => 'us-east-1'}.merge(options)[:region]}.amazonaws.com"

data/lib/elasticity/job_flow.rb

@@ -4,13 +4,13 @@ module Elasticity
   class JobFlowNotStartedError < StandardError; end
   class JobFlowMissingStepsError < StandardError; end
   class LogUriMissingError < StandardError; end
+  class UnknownPlacementError < StandardError; end
 
   class JobFlow
 
     attr_accessor :action_on_failure
     attr_accessor :ec2_key_name
     attr_accessor :name
-    attr_accessor :hadoop_version
     attr_accessor :instance_count
     attr_accessor :log_uri
     attr_accessor :master_instance_type
@@ -19,6 +19,7 @@ module Elasticity
     attr_accessor :keep_job_flow_alive_when_no_steps
     attr_accessor :ec2_subnet_id
     attr_accessor :placement
+    attr_accessor :region
     attr_accessor :visible_to_all_users
     attr_accessor :enable_debugging
 
@@ -27,11 +28,10 @@ module Elasticity
 
     def initialize(access=nil, secret=nil)
       @action_on_failure = 'TERMINATE_JOB_FLOW'
-      @hadoop_version = '1.0.3'
       @name = 'Elasticity Job Flow'
       @ami_version = 'latest'
       @keep_job_flow_alive_when_no_steps = false
-      @placement = 'us-east-1a'
+      self.placement = 'us-east-1a'
       @enable_debugging = false
 
       @access_key = access
@@ -61,6 +61,20 @@ module Elasticity
       end
     end
 
+    def placement=(new_placement)
+      @placement = new_placement
+      return unless @placement
+
+      # The region has to be set so we know where to launch the job flow, and it has
+      # to be in sync with the placement.  Not setting this will lead to strange, difficult to
+      # track errors, for example the job flow won't exist where you think it should, etc.
+      if @placement =~ /\w+-\w+-\d+/
+        @region = @placement.match(/(\w+-\w+-\d+)/)[0]
+      else
+        raise UnknownPlacementError, "'#{@placement}' is not a valid EMR placement"
+      end
+    end
+
     def enable_debugging=(enabled)
       if enabled
         raise LogUriMissingError, 'To enable debugging, please set a #log_uri' unless @log_uri
@@ -158,7 +172,6 @@ module Elasticity
     end
 
     def emr
-      @region ||= @placement.match(/(\w+-\w+-\d+)/)[0]
       @emr ||= Elasticity::EMR.new(@access_key, @secret_key, :region => @region)
     end
 
@@ -183,7 +196,6 @@ module Elasticity
         :visible_to_all_users => @visible_to_all_users,
         :instances => {
           :keep_job_flow_alive_when_no_steps => @keep_job_flow_alive_when_no_steps,
-          :hadoop_version => @hadoop_version,
           :instance_groups => jobflow_instance_groups,
           :placement => {
             :availability_zone => @placement

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '3.0.4'
+  VERSION = '4.0'
 end

data/spec/lib/elasticity/aws_request_spec.rb

@@ -21,6 +21,14 @@ describe Elasticity::AwsRequest do
       end
     end
 
+    context 'when :region is nil' do
+      it 'should be an error' do
+        expect {
+          Elasticity::AwsRequest.new('_', '_', :region => nil)
+        }.to raise_error Elasticity::MissingRegionError, 'A valid :region is required to connect to EMR'
+      end
+    end
+
     context 'when either access or secret key is not provided or nil' do
 
       context 'when the proper environment variables are set' do

data/spec/lib/elasticity/job_flow_integration_spec.rb

@@ -31,7 +31,6 @@ describe 'Elasticity::JobFlow Integration Examples' do
         :visible_to_all_users => false,
         :instances => {
           :keep_job_flow_alive_when_no_steps => false,
-          :hadoop_version => '1.0.3',
           :instance_groups => [
             {
               :instance_count => 1,
@@ -119,7 +118,6 @@ describe 'Elasticity::JobFlow Integration Examples' do
         :visible_to_all_users => false,
         :instances => {
           :keep_job_flow_alive_when_no_steps => false,
-          :hadoop_version => '1.0.3',
           :instance_groups => [
             {
               :instance_count => 1,
@@ -206,7 +204,6 @@ describe 'Elasticity::JobFlow Integration Examples' do
         :visible_to_all_users => false,
         :instances => {
           :keep_job_flow_alive_when_no_steps => false,
-          :hadoop_version => '1.0.3',
           :instance_groups => [
             {
               :instance_count => 1,

data/spec/lib/elasticity/job_flow_spec.rb

@@ -9,7 +9,6 @@ describe Elasticity::JobFlow do
   its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
   its(:ec2_key_name) { should == nil }
   its(:ec2_subnet_id) { should == nil }
-  its(:hadoop_version) { should == '1.0.3' }
   its(:instance_count) { should == 2 }
   its(:log_uri) { should == nil }
   its(:master_instance_type) { should == 'm1.small' }
@@ -20,6 +19,7 @@ describe Elasticity::JobFlow do
   its(:placement) { should == 'us-east-1a' }
   its(:visible_to_all_users) { should == false }
   its(:enable_debugging) { should == false }
+  its(:region) { should == 'us-east-1' }
 
   describe '.initialize' do
     it 'should set the access and secret keys to nil by default' do
@@ -33,6 +33,41 @@ describe Elasticity::JobFlow do
     end
   end
 
+  describe '#placement=' do
+
+    context 'when the placement is set' do
+
+      context 'when the placement is valid' do
+        before do
+          subject.placement = 'us-west-1a'
+        end
+
+        it 'should set the region' do
+          subject.region.should == 'us-west-1'
+        end
+      end
+
+      context 'when the placement is not valid' do
+        it 'should set the region' do
+          expect {
+            subject.placement = 'BAD_PLACEMENT'
+          }.to raise_error(Elasticity::UnknownPlacementError, "'BAD_PLACEMENT' is not a valid EMR placement")
+        end
+      end
+
+    end
+
+    context 'when the placement is not set' do
+      before do
+        subject.placement = nil
+      end
+      it 'should not modify the region' do
+        subject.region.should == 'us-east-1'
+      end
+    end
+
+  end
+
   describe '#enable_debugging=' do
 
     context 'when a log_uri is present' do
@@ -136,7 +171,7 @@ describe Elasticity::JobFlow do
     end
 
     it 'should unset placement (which has a default value) because having both set is EMR-invalid' do
-      subject.placement = 'TEST_PLACEMENT'
+      subject.placement = 'us-east-1d'
 
       subject.ec2_subnet_id = '_'
       subject.placement.should == nil
@@ -406,7 +441,6 @@ describe Elasticity::JobFlow do
         :visible_to_all_users => false,
         :instances => {
           :keep_job_flow_alive_when_no_steps => false,
-          :hadoop_version => '1.0.3',
           :instance_groups => ['INSTANCE_GROUP_CONFIGURATION'],
           :placement => {
             :availability_zone => 'us-east-1a'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: 3.0.4
+  version: '4.0'
 platform: ruby
 authors:
 - Robert Slifka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-30 00:00:00.000000000 Z
+date: 2014-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client