buzzoink 0.0.3

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2012 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,50 @@
+= Buzzoink
+
+A very simple interface for Amazon's EMR service.  It will start interactive Hive, Pig and Streaming sessions.  
+It only exposes a few pieces of information like state, name and DNS name for the master machine.  It also allows only a few 
+machine configurations
+
+{<img src="https://secure.travis-ci.org/Raybeam/buzzoink.png?branch=master" />}[http://travis-ci.org/Raybeam/buzzoink]
+
+== Cluster sizes
+
+[test] Very small, 1 master and 1 slave.  It basically for running tests on the gem
+
+[development] Around 10 slaves.  It's for testing small EMR jobs
+
+[production] At least 20 machines.  Up to 60 if the spot reservations come online
+
+
+== Examples
+
+=== Configuration
+
+For a full list of configuration options, check the documentation.
+
+  Buzzoink.configure do | c |
+    c.aws_access_key_id = 'accesskey'
+    c.aws_secret_access_key = 'secretkey'
+    c.instance_settings = :production
+    c.max_machines = 20 # This will be 60 if spot reservations fire
+    c.key_name = 'mysecretkey'
+    c.epoch = DateTime.now.ago(1.hour)
+  end
+
+
+=== Start a hive instance
+
+You can also start a Pig or Streaming instance
+
+  hive = Buzzoink::Job.find_or_start_hive
+  begin
+    sleep(20)
+  end until hive.ready?
+  puts "Connect here: #{hive.public_dns}"
+
+=== Kill all jobs managed by Buzzoink
+
+This kills jobs of all types (e.g. Hive, Pig, etc)
+
+  Buzzoink::Job.kill_all
+
+This project uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,27 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+begin
+  require 'rdoc/task'
+rescue LoadError
+  require 'rdoc/rdoc'
+  require 'rake/rdoctask'
+  RDoc::Task = Rake::RDocTask
+end
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Buzzoink'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+Bundler::GemHelper.install_tasks
+

data/lib/buzzoink/configuration.rb

@@ -0,0 +1,192 @@
+require 'fog'
+
+module Buzzoink
+  class Configuration
+
+    # Sets the prefix of the process name.  It's 
+    # a way to keep track of all buzzoink processes.
+    # Don't change this unless you know what you're
+    # doing.  Use "name=" instead if you want to just
+    # set your process apart.
+    #
+    # @example
+    #   Buzzoink.configure do | c | 
+    #     c.name_prefix = "My favorite hive process"
+    #   end
+    #
+    # @default Managed Buzzoink process
+    #
+    attr_writer :name_prefix
+    def name_prefix
+      @name_prefix ||= 'Managed Buzzoink process'
+    end
+
+    # Sets the process name.
+    #
+    # @example
+    #   Buzzoinks.configure do | c | 
+    #     c.name = "One off test"
+    #   end
+    #
+    # @default Main EMR process
+    #
+    attr_writer :name
+    def name
+      @name ||= 'Main EMR process'
+    end
+
+    # Full name of the process
+    def full_name *args
+      suffix_options = args.extract_options!
+
+      "#{name_prefix} : #{name}".tap do | str |
+        unless suffix_options.blank?
+          suffix = " " + suffix_options.map { | k, v | "#{k} => #{v}" }.join(", ")
+          str << suffix
+        end
+      end
+    end
+
+    # Sets the a backstop datetime for all EMR
+    # queries.  Set it to a time before which
+    # you care nothing about.
+    #
+    # @example
+    #   Buzzoink.configure do | c | 
+    #     c.epoch = DateTime.today.ago(3.weeks)
+    #   end
+    #
+    # @default One day ago
+    #
+    attr_writer :epoch
+    def epoch
+      @epoch ||= DateTime.now.ago(1.day)
+      @epoch.iso8601
+    end
+
+    # Sets the max number of on demand slaves
+    # for this job flow.
+    # If there are spot instances included
+    # in the deployment, their number will
+    # double this setting
+    #
+    # For example, if you leave the default of 20
+    # set, there will be 20 on demand instances, 
+    # 40 spot instances and a master resulting
+    # in 61 total machines.
+    #
+    attr_writer :max_instances
+    def max_instances
+      @max_instances ||= 20
+    end
+
+    attr_accessor :aws_access_key_id
+    attr_accessor :aws_secret_access_key
+    attr_accessor :key_name
+    attr_accessor :key_path
+    attr_accessor :s3_log_location
+
+    def emr
+      @emr ||= Fog::AWS::EMR.new(
+        :aws_access_key_id => @aws_access_key_id,
+        :aws_secret_access_key => @aws_secret_access_key  
+      )
+    end
+
+    def full_emr_configuration
+      instances = {
+        :ec2_key_name => key_name,
+        :instance_groups => instance_settings_config,
+        :keep_job_flow_alive_when_no_steps => true,
+        :termination_protected => false
+      }
+
+      config = {
+        :ami_version => '2.0', # Latest
+        :instances => instances,
+      }
+      config[:log_uri] = s3_log_location unless s3_log_location.nil?
+
+      Buzzoink.convert_hash_keys(config)
+    end
+
+    # Gets one of the EMR preset configurations
+    # for instance groups.  The presets are currently
+    # test and production.  Test is a single machine, Production
+    # is 60 machines.
+    #
+    # @example
+    #   Buzzoink.configure do | c | 
+    #     c.instance_settings = :production
+    #   end
+    #
+    # @default :test
+    attr_writer :instance_settings
+    def instance_settings *args
+      @instance_settings ||= :test
+    end
+
+    def instance_settings_config
+      meth = "instance_settings_#{instance_settings.to_s}"
+      self.send(meth)
+    end
+
+    def instance_settings_production
+      instance_groups = []
+
+      # Master
+      instance_groups << {
+        :instance_count => 1,
+        :instance_role => 'MASTER',
+        :instance_type => 'm1.small',
+        :market => 'ON_DEMAND',
+        :name => 'Master group'
+      }
+
+      # Core
+      instance_groups << {
+        :instance_count => max_instances,
+        :instance_role => 'CORE',
+        :instance_type => 'c1.medium',
+        :market => 'ON_DEMAND',
+        :name => 'Compute group'
+      }
+
+      # Task :: Spot instances
+      instance_groups << {
+        :instance_count => (max_instances * 2),
+        :bid_price => '0.08',
+        :instance_role => 'TASK',
+        :instance_type => 'c1.medium',
+        :market => 'SPOT',
+        :name => 'Flex group'
+      }
+
+      instance_groups
+    end
+
+    def instance_settings_test
+      instance_groups = []
+
+      # Master
+      instance_groups << {
+        :instance_count => 1,
+        :instance_role => 'MASTER',
+        :instance_type => 'm1.small',
+        :market => 'ON_DEMAND',
+        :name => 'Master group'
+      }
+
+      # Core
+      instance_groups << {
+        :instance_count => 1,
+        :instance_role => 'CORE',
+        :instance_type => 'c1.medium',
+        :market => 'ON_DEMAND',
+        :name => 'Compute group'
+      }
+
+      instance_groups
+    end
+  end
+end

data/lib/buzzoink/job.rb

@@ -0,0 +1,273 @@
+module Buzzoink
+  # A representation of the job in Amazon's EMR system.  There are a few dynamic methods
+  # created in this object. The are basically candy backed by another method in this class
+  #
+  # [start_hive] start(:type => :hive)
+  # [start_pig] start(:type => :pig)
+  # [find_or_start_hive] find_or_start(:type => :hive)
+  # [find_or_start_pig] find_or_start(:type => :pig)
+  #
+  class Job
+    attr_reader :job_flow_id
+    attr_reader :body
+
+    READY_STATES = ['RUNNING', 'WAITING']
+    PENDING_STATES = ['STARTING', 'BOOTSTRAPPING']
+    ACTIVE_STATES = READY_STATES + PENDING_STATES
+
+    class << self
+      # Gets an EMR job based on job flow ID
+      # This is not limited to Buzzoink jobs
+      #
+      # @example
+      #   Buzzoink::Job.get 'j-234324325'
+      #
+      #
+      def get job_id
+        begin
+          job_flows = Buzzoink.emr.describe_job_flows('JobFlowIds' => [job_id]).body
+
+          self.new(job_flows['JobFlows'].first)
+        rescue Excon::Errors::Error => e
+          raise Buzzoink::NoJobError if e.message =~ /Specified job flow ID not valid/
+          raise e
+        end
+      end
+
+      # Get all EMR jobs created after the epoch.  The EMR
+      # jobs are instantiated as Buzzoink::Job objects
+      # Not limited to Buzzoink jobs
+      #
+      # @example
+      #   jobs = Buzzoink::Job.get_jobs
+      #
+      #
+      def get_jobs
+        jobs = []
+        raw = Buzzoink.emr.describe_job_flows('CreatedAfter' => Buzzoink.configure.epoch).body
+        raw['JobFlows'].each do | jf |
+          jobs << self.new(jf)
+        end
+        jobs
+      end
+
+      # Gets all jobs managed by Buzzoink.  Pass
+      # a :type to get only certain job types
+      #
+      # @example
+      #   all_buzz_jobs = Buzzoink::Job.get_managed_jobs
+      #   just_hive_jobs = Buzzoink::Job.get_managed_jobs :type => :hive
+      #
+      # @default all Buzzoink jobs
+      def get_managed_jobs *args
+        options = args.extract_options!
+        jobs = get_jobs
+
+        return jobs.select { | j | j.body['Name'] =~ /^#{Buzzoink.configure.name_prefix}/} if options.blank?
+
+        full_name = Buzzoink.configure.full_name(options)
+        jobs.select { | j | j.body['Name'] == full_name}
+      end
+
+      # Gets all active jobs managed by Buzzoink.  Pass
+      # a :type to get only certain job types
+      #
+      # @example
+      #   all_buzz_jobs = Buzzoink::Job.get_active_managed_jobs
+      #   just_hive_jobs = Buzzoink::Job.get_active_managed_jobs :type => :hive
+      #
+      # @default all active Buzzoink jobs
+      def get_active_managed_jobs *args
+        options = args.extract_options!
+        jobs = get_managed_jobs options
+
+        jobs.select { | j | ACTIVE_STATES.include?(j.body['ExecutionStatusDetail']['State'])}
+      end
+
+      # If a job type is already running, return it.  Otherwise start
+      # a new job of that type
+      #
+      # @example
+      #   job = Buzzoink::Job.find_or_start :type => :hive
+      #
+      # @default :type => :pig
+      def find_or_start *args
+        options = args.extract_options!
+        options.reverse_merge! :type => :pig
+
+        jobs = get_active_managed_jobs :type => options[:type]
+        return jobs.first unless jobs.blank?
+
+        self.start options
+      end
+
+      # Kill all Buzzoink managed jobs
+      #
+      # @example
+      #   Buzzoink::Job.kill_all
+      #
+      def kill_all
+        jobs = get_managed_jobs
+        return false if jobs.blank?
+
+        ids = jobs.map(&:id)
+        kill_jobs ids
+      end
+
+      # Kill jobs based on job flow ID.
+      # This is not limited to Buzzoink jobs
+      #
+      # @example
+      #   true_or_false = Buzzoink::Job.kill_jobs 'j-23432432', 'j-54654'
+      #
+      def kill_jobs job_flow_ids
+        # In case a single ID is sent
+        flows = []
+        flows << job_flow_ids
+
+        result = Buzzoink.emr.terminate_job_flows('JobFlowIds' => flows.flatten)
+        !!result.body['RequestId']
+      end
+
+      # Starts a job flow in EMR.  This job flow
+      # can either be streaming, hive or pig.
+      #
+      # @example
+      #   Buzzoink::Job.start(:type => :hive)
+      #
+      # @default :type => :pig
+      #
+      def start *args
+        options = args.extract_options!
+        options.reverse_merge! :type => :pig
+
+        active_jobs = get_active_managed_jobs :type => options[:type]
+        raise Buzzoink::DuplicateJobError unless active_jobs.blank?
+
+        conf = Buzzoink.configure
+        job = Buzzoink.emr.run_job_flow(conf.full_name(:type => options[:type]), conf.full_emr_configuration)
+
+        # Add appropriate step for interactive job
+        if options[:type] == :hive
+          Buzzoink.emr.add_job_flow_steps(job.body['JobFlowId'], {'Steps' => [hive_step]})
+        elsif options[:type] == :pig
+          Buzzoink.emr.add_job_flow_steps(job.body['JobFlowId'], {'Steps' => [pig_step]})
+        else
+          raise Buzzoink::Error, "Current type is invalid :: #{options[:type]}"
+        end
+
+        get job.body['JobFlowId']
+      end
+
+      # :nodoc:
+      def hive_step
+        {
+          'Name' => 'Hive',
+          'ActionOnFailure' => 'TERMINATE_JOB_FLOW',
+          'HadoopJarStep' => {
+            'Jar' => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+            'Args' => [
+              's3://elasticmapreduce/libs/hive/hive-script',
+              '--base-path', 's3://elasticmapreduce/libs/hive/',
+              '--install-hive',
+              '--hive-versions', '0.7.1.1'
+            ]
+          }
+        }
+      end
+
+      # :nodoc:
+      def pig_step
+        {
+          'Name' => 'Pig',
+          'ActionOnFailure' => 'TERMINATE_JOB_FLOW',
+          'HadoopJarStep' => {
+            'Jar' => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+            'Args' => [
+              's3://elasticmapreduce/libs/pig/pig-script',
+              '--base-path',  's3://elasticmapreduce/libs/pig/',
+              '--install-pig'
+            ]
+          }
+        }
+      end
+
+      # Some helper methods are included for each type of 
+      # job.
+      #
+      # @example
+      #   j = Buzzoink::Job.start_hive
+      #   j = Buzzoink::Job.find_or_start_pig
+      #
+      [:hive, :pig, :streaming].each do | type |
+        self.class_eval <<-RUBY
+          def start_#{type.to_s}
+            start(:type => :#{type})
+          end
+
+          def find_or_start_#{type.to_s}
+            find_or_start(:type => :#{type})
+          end
+        RUBY
+      end
+    end
+
+    def initialize body
+      @job_flow_id = body['JobFlowId']
+      @body = body
+    end
+    alias_method :id, :job_flow_id
+  
+    # Kills the current job
+    #
+    # @example
+    #   job.kill
+    #
+    #
+    def kill
+      self.class.kill_jobs self.id
+    end
+
+    # Pulls the latest description for this
+    # job from EMR
+    def refresh!
+      new_job = self.class.get id
+      @body = new_job.body
+      self  
+    end
+
+    # The current boot state of the instance
+    def state
+      body['ExecutionStatusDetail']['State']
+    end
+
+    # Boolean for determining whether the instance
+    # is ready for operation
+    def ready?
+      READY_STATES.include?(state)
+    end
+
+    def steps
+      body['Steps']
+    end
+
+    # Address to use for connection
+    def public_dns
+      body['Instances']['MasterPublicDnsName']
+    end
+
+    # Is this instance protected from termination
+    def termination_protected?
+      body['Instances']['TerminationProtected'] == 'true'
+    end
+
+    # The type of job (e.g. Hive, Pig, etc)
+    def type
+      if body['Name'] =~ /type => (\w+)/
+        return $1.to_sym
+      end
+
+      return nil
+    end
+  end
+end

data/lib/buzzoink/version.rb

@@ -0,0 +1,3 @@
+module Buzzoink
+  VERSION = "0.0.3"
+end

data/lib/buzzoink.rb

@@ -0,0 +1,54 @@
+require 'active_support/core_ext'
+
+require 'buzzoink/configuration'
+require 'buzzoink/job'
+
+module Buzzoink
+  extend self
+
+  def configure
+    @configuration ||= Configuration.new
+    if block_given?
+      yield @configuration
+    end
+    @configuration
+  end
+
+  def emr
+    configure.emr
+  end
+
+  def clear_configuration
+    @configuration = nil
+  end
+
+  #
+  # Candy so I can use underscore symbol keys
+  #
+
+  # Camel case an underscore symbol
+  def camelcase_key(k)
+    k.to_s.camelize
+  end
+
+  # Recursively set hash keys
+  def convert_hash_keys(value)
+    case value
+      when Array
+        value.map { |v| convert_hash_keys(v) }
+      when Hash
+        Hash[value.map { |k, v| [camelcase_key(k), convert_hash_keys(v)] }]
+      else
+        value
+     end
+  end
+
+  class BuzzoinkError < StandardError
+  end
+
+  class DuplicateJobError < BuzzoinkError
+  end
+
+  class NoJobError < BuzzoinkError
+  end
+end

data/lib/tasks/buzzoink_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :buzzoink do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification 
+name: buzzoink
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.3
+platform: ruby
+authors: 
+- Chris Hagar
+- Bob Briski
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-02-10 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "3.0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: addressable
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "2.2"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: andand
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "1.3"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: fog
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rspec-rails
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: webmock
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: fabrication
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: vcr
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id008
+description: Simple interface for Pig, Hive and Streaming over EMR.
+email: 
+- chagar@raybeam.com
+- bbriski@raybeam.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- config/buzzoink.yml
+- lib/buzzoink/configuration.rb
+- lib/buzzoink/job.rb
+- lib/buzzoink/version.rb
+- lib/buzzoink.rb
+- lib/tasks/buzzoink_tasks.rake
+- MIT-LICENSE
+- Rakefile
+- README.rdoc
+homepage: https://github.com/Raybeam/buzzoink
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3861862602005810237
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3861862602005810237
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Start hive or pig processes in EMR
+test_files: []
+