elasticity 1.5 → 2.0

data/spec/spec_helper.rb

@@ -1,56 +1,7 @@
 require 'rubygems'
 require 'bundler/setup'
-
-require 'vcr'
-
 require 'elasticity'
 
 ENV["RAILS_ENV"] ||= 'test'
 
-$:.unshift File.dirname(__FILE__)
-
-uri_regexp_matcher = lambda do |real_request, recorded_request|
-  real_request.uri =~ recorded_request.uri
-end
-
-VCR.configure do |c|
-  c.cassette_library_dir = 'spec/fixtures/vcr_cassettes'
-  c.hook_into :webmock
-  c.default_cassette_options = {
-    :match_requests_on => [:method, uri_regexp_matcher]
-  }
-  c.before_http_request do |_|
-    require_aws_credentials unless VCR.current_cassette.record_mode == :none
-  end
-end
-
-RSpec.configure do |c|
-  c.extend VCR::RSpec::Macros
-end
-
-AWS_ACCESS_KEY_ID = ENV['AWS_ACCESS_KEY_ID'] ||= 'default'
-AWS_SECRET_KEY = ENV['AWS_SECRET_KEY'] ||= 'default'
-
-def require_aws_credentials
-  if AWS_ACCESS_KEY_ID == 'default' && AWS_SECRET_KEY == 'default'
-    puts "\n\e[33m**********************************************************************************************"
-    puts "\e[32mIf you want to record new cassettes, you'll need to provide a set of AWS credentials so"
-    puts "Elasticity can interact with EMR.  These keys can be found on your AWS Account > Security"
-    puts "Credentials page, at the following URL:"
-    puts ""
-    puts "  \e[0;1mhttps://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key"
-    puts ""
-    bourne = !ENV['SHELL'].nil? && ENV['SHELL'] =~ /(bash|ksh|zsh)/
-    if bourne
-      puts "\e[32mbash, zsh, ksh:"
-      puts "  export AWS_ACCESS_KEY_ID=01234"
-      puts "  export AWS_SECRET_KEY=56789"
-    else
-      puts "\e[32mcsh, tcsh:"
-      puts "  setenv AWS_ACCESS_KEY_ID 01234"
-      puts "  setenv AWS_SECRET_KEY 56789"
-    end
-    puts "\e[33m**********************************************************************************************\n"
-    exit
-  end
-end
+Dir[File.join(File.dirname(__FILE__), "support", "**", "*.rb")].each { |f| require f }

data/spec/support/be_a_hash_including_matcher.rb

@@ -0,0 +1,35 @@
+RSpec::Matchers.define :be_a_hash_including do |expected|
+  match do |actual|
+    next false unless actual.is_a? Hash
+    next false unless expected.is_a? Hash
+    expected.keys.each do |key|
+      break false if actual[key] != expected[key]
+      true
+    end
+  end
+end
+
+describe :be_a_hash_including do
+
+  context 'when actual is not a Hash' do
+    subject { 'I AM NOT A HASH' }
+    it { should_not be_a_hash_including({}) }
+  end
+
+  context 'when expected is not a Hash' do
+    subject { {} }
+    it { should_not be_a_hash_including('') }
+  end
+
+  context 'when expected is included in actual' do
+    subject {{:actual_key1 => 'value1'}}
+    it { should be_a_hash_including({:actual_key1 => 'value1'})}
+  end
+
+  context 'when expected is not included in actual' do
+    subject {{:actual_key1 => 'value1'}}
+    it { should_not be_a_hash_including({:actual_key3 => 'value3'})}
+  end
+
+end
+

metadata

@@ -1,94 +1,87 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: elasticity
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: '2.0'
   prerelease: 
-  version: "1.5"
 platform: ruby
-authors: 
+authors:
 - Robert Slifka
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-03-05 00:00:00 -08:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-07-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rest-client
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: rake
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
-  type: :development
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: rspec
-  prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 2.8.0
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: vcr
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: "2.0"
+      - !ruby/object:Gem::Version
+        version: 2.10.0
   type: :development
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: webmock
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        version: 1.8.0
-  type: :development
-  version_requirements: *id006
-description: Programmatic access to Amazon's Elastic Map Reduce service, driven by the Sharethrough team's requirements for belting out EMR jobs.
+      - !ruby/object:Gem::Version
+        version: 2.10.0
+description: Streamlined, Programmatic access to Amazon's Elastic Map Reduce service,
+  driven by the Sharethrough team's requirements for belting out EMR jobs.
 email: 
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- .autotest
+files:
 - .gitignore
 - .rspec
 - .rvmrc
@@ -101,84 +94,73 @@ files:
 - elasticity.gemspec
 - lib/elasticity.rb
 - lib/elasticity/aws_request.rb
-- lib/elasticity/custom_jar_job.rb
+- lib/elasticity/custom_jar_step.rb
 - lib/elasticity/emr.rb
-- lib/elasticity/hive_job.rb
+- lib/elasticity/hadoop_bootstrap_action.rb
+- lib/elasticity/hive_step.rb
 - lib/elasticity/job_flow.rb
+- lib/elasticity/job_flow_status.rb
+- lib/elasticity/job_flow_status_step.rb
 - lib/elasticity/job_flow_step.rb
-- lib/elasticity/pig_job.rb
-- lib/elasticity/simple_job.rb
+- lib/elasticity/pig_step.rb
+- lib/elasticity/support/conditional_raise.rb
 - lib/elasticity/version.rb
-- spec/fixtures/vcr_cassettes/add_instance_groups/one_group_successful.yml
-- spec/fixtures/vcr_cassettes/add_instance_groups/one_group_unsuccessful.yml
-- spec/fixtures/vcr_cassettes/add_jobflow_steps/add_multiple_steps.yml
-- spec/fixtures/vcr_cassettes/custom_jar_job/cloudburst.yml
-- spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
-- spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
-- spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
-- spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
-- spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml
-- spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml
-- spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
-- spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml
-- spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml
-- spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
 - spec/lib/elasticity/aws_request_spec.rb
-- spec/lib/elasticity/custom_jar_job_spec.rb
+- spec/lib/elasticity/custom_jar_step_spec.rb
 - spec/lib/elasticity/emr_spec.rb
-- spec/lib/elasticity/hive_job_spec.rb
+- spec/lib/elasticity/hadoop_bootstrap_action_spec.rb
+- spec/lib/elasticity/hive_step_spec.rb
+- spec/lib/elasticity/job_flow_integration_spec.rb
 - spec/lib/elasticity/job_flow_spec.rb
+- spec/lib/elasticity/job_flow_status_spec.rb
+- spec/lib/elasticity/job_flow_status_step_spec.rb
 - spec/lib/elasticity/job_flow_step_spec.rb
-- spec/lib/elasticity/pig_job_spec.rb
+- spec/lib/elasticity/pig_step_spec.rb
+- spec/lib/elasticity/support/conditional_raise_spec.rb
 - spec/spec_helper.rb
-has_rdoc: true
+- spec/support/be_a_hash_including_matcher.rb
 homepage: http://www.github.com/rslifka/elasticity
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 4070655408797594419
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 4070655408797594419
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.5.2
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: Programmatic access to Amazon's Elastic Map Reduce service.
-test_files: 
-- spec/fixtures/vcr_cassettes/add_instance_groups/one_group_successful.yml
-- spec/fixtures/vcr_cassettes/add_instance_groups/one_group_unsuccessful.yml
-- spec/fixtures/vcr_cassettes/add_jobflow_steps/add_multiple_steps.yml
-- spec/fixtures/vcr_cassettes/custom_jar_job/cloudburst.yml
-- spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
-- spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
-- spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
-- spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
-- spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml
-- spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml
-- spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
-- spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml
-- spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml
-- spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
+summary: Streamlined, programmatic access to Amazon's Elastic Map Reduce service.
+test_files:
 - spec/lib/elasticity/aws_request_spec.rb
-- spec/lib/elasticity/custom_jar_job_spec.rb
+- spec/lib/elasticity/custom_jar_step_spec.rb
 - spec/lib/elasticity/emr_spec.rb
-- spec/lib/elasticity/hive_job_spec.rb
+- spec/lib/elasticity/hadoop_bootstrap_action_spec.rb
+- spec/lib/elasticity/hive_step_spec.rb
+- spec/lib/elasticity/job_flow_integration_spec.rb
 - spec/lib/elasticity/job_flow_spec.rb
+- spec/lib/elasticity/job_flow_status_spec.rb
+- spec/lib/elasticity/job_flow_status_step_spec.rb
 - spec/lib/elasticity/job_flow_step_spec.rb
-- spec/lib/elasticity/pig_job_spec.rb
+- spec/lib/elasticity/pig_step_spec.rb
+- spec/lib/elasticity/support/conditional_raise_spec.rb
 - spec/spec_helper.rb
+- spec/support/be_a_hash_including_matcher.rb

data/.autotest

@@ -1,2 +0,0 @@
-require 'autotest/fsevent'
-require 'autotest/growl'

data/lib/elasticity/custom_jar_job.rb

@@ -1,38 +0,0 @@
-module Elasticity
-
-  class CustomJarJob < Elasticity::SimpleJob
-
-    def initialize(aws_access_key_id, aws_secret_access_key)
-      super
-      @name = "Elasticity Custom Jar Job"
-    end
-
-    def run(jar, arguments=nil)
-      jobflow_config = {
-        :name => @name,
-        :instances => {
-          :ec2_key_name => @ec2_key_name,
-          :hadoop_version => @hadoop_version,
-          :instance_count => @instance_count,
-          :master_instance_type => @master_instance_type,
-          :slave_instance_type => @slave_instance_type,
-        },
-        :steps => [
-          {
-            :action_on_failure => @action_on_failure,
-            :hadoop_jar_step => {
-              :jar => jar
-            },
-            :name => "Execute Custom Jar"
-          }
-        ]
-      }
-
-      jobflow_config[:steps].first[:hadoop_jar_step][:args] = arguments if arguments
-
-      run_job(jobflow_config)
-    end
-
-  end
-  
-end

data/lib/elasticity/hive_job.rb

@@ -1,69 +0,0 @@
-module Elasticity
-
-  # HiveJob allows you quickly easily kick off a Hive jobflow without
-  # having to understand the entirety of the EMR API.
-  class HiveJob < Elasticity::SimpleJob
-
-    def initialize(aws_access_key_id, aws_secret_access_key)
-      super
-      @name = "Elasticity Hive Job"
-    end
-
-    # Run the specified Hive script with the specified variables.
-    #
-    #   hive = Elasticity::HiveJob.new("access", "secret")
-    #   jobflow_id = hive.run('s3n://slif-hive/test.q', {
-    #     'SCRIPTS' => 's3n://slif-test/scripts',
-    #     'OUTPUT'  => 's3n://slif-test/output',
-    #     'XREFS'   => 's3n://slif-test/xrefs'
-    #   })
-    #
-    # The variables are accessible within your Hive scripts by using the
-    # standard ${NAME} syntax.  E.g.
-    #
-    #   ADD JAR ${SCRIPTS}/jsonserde.jar;
-    def run(hive_script, hive_variables={})
-      script_arguments = ["s3://elasticmapreduce/libs/hive/hive-script", "--run-hive-script", "--args"]
-      script_arguments.concat(["-f", hive_script])
-      hive_variables.each do |variable_name, value|
-        script_arguments.concat(["-d", "#{variable_name}=#{value}"])
-      end
-      jobflow_config = {
-        :name => @name,
-        :instances => {
-          :ec2_key_name => @ec2_key_name,
-          :hadoop_version => @hadoop_version,
-          :instance_count => @instance_count,
-          :master_instance_type => @master_instance_type,
-          :slave_instance_type => @slave_instance_type,
-        },
-        :steps => [
-          {
-            :action_on_failure => "TERMINATE_JOB_FLOW",
-            :hadoop_jar_step => {
-              :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
-              :args => [
-                "s3://elasticmapreduce/libs/hive/hive-script",
-                  "--base-path", "s3://elasticmapreduce/libs/hive/",
-                  "--install-hive"
-              ],
-            },
-            :name => "Setup Hive"
-          },
-            {
-              :action_on_failure => @action_on_failure,
-              :hadoop_jar_step => {
-                :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
-                :args => script_arguments,
-              },
-              :name => "Run Hive Script"
-            }
-        ]
-      }
-
-      run_job(jobflow_config)
-    end
-
-  end
-
-end