refinery 0.9.0

data/lib/refinery/worker.rb

@@ -0,0 +1,65 @@
+module Refinery #:nodoc:
+  # Base class for workers. Place subclasses of this in the workers
+  # directory.
+  #
+  # Workers may include validation logic to verify that the message 
+  # has the correct keys and values before processing.
+  class Worker 
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Utilities
+    include Refinery::Validations
+    include Refinery::Queueable
+    
+    # Initialize the worker with the given daemon.
+    def initialize(daemon)
+      @daemon = daemon
+    end
+    
+    # Run the worker with the given message. The result from the worker's
+    # <code>execute</code> method is returned along with the run time.
+    #
+    # Validation will occur prior to calling execute.
+    def run(message)
+      result = false
+      
+      validate(message)
+      
+      logger.debug "Executing worker #{self.class.name}"
+      time = Benchmark.realtime do
+        begin
+          result = execute(message)
+        rescue Exception => e
+          logger.error "Error executing worker #{self.class.name}: #{e.message}"
+          raise e
+        end
+      end
+      logger.debug "Completed worker #{self.class.name} in #{time} seconds"
+      return result, time
+    end
+    
+    # Get the data store for the worker.
+    #
+    # The data store is provided through the Moneta interface.
+    #
+    # If the configuration providers a data_store:class option then that class
+    # will be used (the class must be in the Moneta module), otherwise 
+    # Moneta::S3 will be used.
+    def data_store(options)
+      class_name = processor_config['workers']['data_store']['class'] rescue 'S3'
+      ds_class = Moneta.const_get(camelize(class_name))
+      (@data_store ||= {})[options] ||= ds_class.new(
+        :access_key_id => config['aws']['credentials']['access_key_id'],  
+        :secret_access_key => config['aws']['credentials']['secret_access_key'],
+        :bucket => options[:bucket],
+        :multi_thread => true
+      )
+    end
+    
+    private
+    # Get's the config element starting at the processer
+    def processor_config
+      config['processors'][daemon.name]
+    end
+  end
+end

data/lib/refinery.rb

@@ -0,0 +1,74 @@
+$:.unshift(File.dirname(__FILE__))
+
+require 'logger'
+require 'socket'
+require 'benchmark'
+
+# The Refinery module contains all of the classes for the refinery system.
+module Refinery
+  
+  # Require the specified library.
+  #
+  # The short name is the require path and the display_name will be shown
+  # if the library cannot be loaded.
+  def self.require_library(short_name, display_name)
+    begin
+      require short_name
+    rescue LoadError
+      puts "#{display_name} is required, please install it"
+      exit
+    end
+  end
+  
+  # Require all of the dependencies.
+  def self.require_libraries
+    require_library('rubygems', 'Rubygems')
+    require_library('right_aws', 'RightScale AWS gem')
+    require_library('json', 'JSON gem')
+    require_library('moneta', 'Moneta gem')
+    require_library('moneta/s3', 'Moneta S3 implementation')
+  end
+  
+  def self.require_optional_library(short_name, display_name)
+    begin
+      require short_name
+    rescue LoadError
+    end
+  end
+  
+  def self.require_optional_libraries
+    require_optional_library('sequel', 'Sequel gem')
+    require_optional_library('ramaze', 'Ramaze')
+  end
+  
+  # Require internal code files
+  def self.require_internals
+    require 'refinery/loggable'
+    require 'refinery/configurable'
+    require 'refinery/queueable'
+
+    require 'refinery/utilities'
+    
+    require 'refinery/validations'
+
+    require 'refinery/config'
+    require 'refinery/heartbeat'
+    require 'refinery/server'
+    require 'refinery/daemon'
+    require 'refinery/worker'
+    require 'refinery/event_publisher'
+    require 'refinery/publisher'
+    require 'refinery/monitor'
+    require 'refinery/statistics'
+    require 'refinery/stats_server'
+    
+  end
+  
+  # Raised if a source file cannot be loaded
+  class SourceFileNotFound < RuntimeError
+  end
+end
+
+Refinery::require_libraries
+Refinery::require_optional_libraries
+Refinery::require_internals

data/logs/README

@@ -0,0 +1 @@
+Log files from Refinery will be written here. Do not remove this directory.

data/publishers/error.rb

@@ -0,0 +1,8 @@
+# An example publisher that posts a message to the queue that should raise an error.
+class Error < Refinery::Publisher
+  def execute
+    if waiting_queue.size == 0
+      publish({'text' => 'fire an error, please'})
+    end
+  end
+end

data/publishers/sample.rb

@@ -0,0 +1,8 @@
+# A sample publisher that posts a message to the queue.
+class Sample < Refinery::Publisher
+  def execute
+    if waiting_queue.size == 0
+      publish({'text' => 'hey there!'})
+    end
+  end
+end

data/publishers/sleep.rb

@@ -0,0 +1,7 @@
+class Sleep < Refinery::Publisher
+  def execute
+    if waiting_queue.size == 0
+      publish({'seconds' => rand(5) + 0.5})
+    end
+  end
+end

data/refinery.gemspec

@@ -0,0 +1,105 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{refinery}
+  s.version = "0.9.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Anthony Eden"]
+  s.date = %q{2009-06-01}
+  s.description = %q{Process data in a distributed fashion.}
+  s.email = %q{anthonyeden@gmail.com}
+  s.executables = ["epub", "monitor", "pubnow", "refinery"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc",
+     "README.textile"
+  ]
+  s.files = [
+    ".autotest",
+     ".gitignore",
+     "CHANGELOG",
+     "LICENSE",
+     "README.rdoc",
+     "README.textile",
+     "Rakefile",
+     "VERSION",
+     "bin/epub",
+     "bin/monitor",
+     "bin/pubnow",
+     "bin/refinery",
+     "config/config.example.yml",
+     "lib/refinery.rb",
+     "lib/refinery/config.rb",
+     "lib/refinery/configurable.rb",
+     "lib/refinery/daemon.rb",
+     "lib/refinery/event_publisher.rb",
+     "lib/refinery/heartbeat.rb",
+     "lib/refinery/loggable.rb",
+     "lib/refinery/monitor.rb",
+     "lib/refinery/publisher.rb",
+     "lib/refinery/queueable.rb",
+     "lib/refinery/server.rb",
+     "lib/refinery/statistics.rb",
+     "lib/refinery/stats_server.rb",
+     "lib/refinery/utilities.rb",
+     "lib/refinery/validations.rb",
+     "lib/refinery/worker.rb",
+     "logs/README",
+     "publishers/error.rb",
+     "publishers/sample.rb",
+     "publishers/sleep.rb",
+     "refinery.gemspec",
+     "test/config.yml",
+     "test/test_helper.rb",
+     "test/unit/config_test.rb",
+     "test/unit/configurable_test.rb",
+     "test/unit/daemon_test.rb",
+     "test/unit/event_publisher_test.rb",
+     "test/unit/heartbeat_test.rb",
+     "test/unit/loggable_test.rb",
+     "test/unit/publisher_test.rb",
+     "test/unit/queueable_test.rb",
+     "test/unit/server_test.rb",
+     "test/unit/statistics_test.rb",
+     "test/unit/utilities_test.rb",
+     "test/unit/validations_test.rb",
+     "test/unit/worker_test.rb",
+     "workers/error.rb",
+     "workers/sample.rb",
+     "workers/sleep.rb"
+  ]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/aeden/refinery}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{refinery}
+  s.rubygems_version = %q{1.3.1}
+  s.summary = %q{Refinery processes data in a distributed environment.}
+  s.test_files = [
+    "test/test_helper.rb",
+     "test/unit/config_test.rb",
+     "test/unit/configurable_test.rb",
+     "test/unit/daemon_test.rb",
+     "test/unit/event_publisher_test.rb",
+     "test/unit/heartbeat_test.rb",
+     "test/unit/loggable_test.rb",
+     "test/unit/publisher_test.rb",
+     "test/unit/queueable_test.rb",
+     "test/unit/server_test.rb",
+     "test/unit/statistics_test.rb",
+     "test/unit/utilities_test.rb",
+     "test/unit/validations_test.rb",
+     "test/unit/worker_test.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/test/config.yml

@@ -0,0 +1,10 @@
+aws:
+  credentials:
+    access_key_id: "aaa"
+    secret_access_key: "bbb"
+processors:
+  sample:
+    publishers:
+      delay: 10
+    workers:
+      initial: 3

data/test/test_helper.rb

@@ -0,0 +1,21 @@
+require 'test/unit'
+require 'rubygems'
+require 'shoulda'
+require 'mocha'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+class Test::Unit::TestCase
+  def setup_default_config
+    Refinery::Config.stubs(:default).returns(Refinery::Config.new(
+      {
+        'aws' => {
+          'credentials' => {
+            'access_key_id' => 'aki',
+            'secret_access_key' => 'sak'
+          }
+        },
+        'processors' => []
+      }
+    ))
+  end
+end

data/test/unit/config_test.rb

@@ -0,0 +1,42 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class ConfigTest < Test::Unit::TestCase
+  context "the config class" do
+    should "provide a default configuration" do
+      assert_not_nil Refinery::Config.default
+    end
+    
+    context "default configuration" do
+      setup do
+        @config = Refinery::Config.default
+      end
+      should "provide an empty aws credentials hash" do
+        assert_equal Hash.new, @config['aws']['credentials']
+      end
+      should "provide an empty processors hash" do
+        assert_equal Hash.new, @config['processors']
+      end
+    end
+    
+    context "after loading configuration from a YAML file" do
+      setup do
+        @config_file = File.dirname(__FILE__) + '/../config.yml'
+        @config = Refinery::Config.new
+        @config.load_file(@config_file)
+      end
+      should "have aws credentials" do
+        assert_equal 'aaa', @config['aws']['credentials']['access_key_id']
+        assert_equal 'bbb', @config['aws']['credentials']['secret_access_key']
+      end
+      should "reload the file when changed" do
+        `touch #{@config_file}`
+        YAML.expects(:load_file).once
+        @config.refresh
+      end
+      should "not reload the file when not changed" do
+        YAML.expects(:load_file).never
+        @config.refresh
+      end
+    end
+  end
+end

data/test/unit/configurable_test.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class ConfigureMe
+  include Refinery::Configurable
+end
+class ConfigurableTest < Test::Unit::TestCase
+  context "a class with the configurable module" do
+    should "provide a config" do
+      assert_not_nil ConfigureMe.new.config
+    end
+  end
+end

data/test/unit/daemon_test.rb

@@ -0,0 +1,37 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class DaemonTest < Test::Unit::TestCase
+  context "a daemon" do
+    setup do
+      @server = stub('Server')
+      @waiting_queue = stub('Queue(waiting)')
+      @waiting_queue.stubs(:receive)
+      @error_queue = stub('Queue(error)')
+      @done_queue = stub('Queue(done)')
+    end
+    should "be startable" do
+      assert_nothing_raised do
+        daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue)
+      end
+    end
+    should "have logging" do
+      daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue)
+      assert_not_nil daemon.logger
+    end
+    context "that is started" do
+      setup do
+        @daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue)
+      end
+      should "have a state of running" do
+        assert @daemon.running?
+      end
+      context "after calling stop" do
+        setup do
+          @daemon.stop
+        end
+        should "not be running" do
+          assert !@daemon.running?
+        end
+      end
+    end
+  end
+end

data/test/unit/event_publisher_test.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class EventPublisherTest < Test::Unit::TestCase
+  context "an event publisher" do
+    should "raise an error if credentials are not set" do
+      publishing_settings = {'sample' => {'delay' => 10}}
+      Refinery::Config.any_instance.stubs(:publishing).returns(publishing_settings)
+      event_publisher = Refinery::EventPublisher.new
+      event_publisher.run
+    end
+  end
+end

data/test/unit/heartbeat_test.rb

@@ -0,0 +1,22 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class HeartbeatTest < Test::Unit::TestCase
+  context "a heartbeat" do
+    setup do
+      setup_default_config
+      
+      @server = stub('server')
+      @server.stubs(:daemons).returns([])
+      
+      heartbeat_queue = stub('heartbeat queue')
+      heartbeat_queue.stubs(:send_message)
+      queue_provider = stub('queue provider')
+      queue_provider.expects(:queue).with('heartbeat').returns(heartbeat_queue)
+      RightAws::SqsGen2.stubs(:new).with(
+        'aki', 'sak', {:multi_thread => true}
+      ).returns(queue_provider)
+    end
+    should "be initializable" do
+      Refinery::Heartbeat.new(@server)
+    end
+  end
+end

data/test/unit/loggable_test.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class LogMe
+  include Refinery::Loggable
+end
+class LoggableTest < Test::Unit::TestCase
+  context "a class with Loggable mixed in" do
+    should "have a logger" do
+      assert_not_nil LogMe.new.logger
+    end
+  end
+end

data/test/unit/publisher_test.rb

@@ -0,0 +1,13 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class SamplePublisher < Refinery::Publisher
+end
+
+class PublisherTest < Test::Unit::TestCase
+  context "a publisher" do
+    should "be instantiable" do
+      waiting_queue = stub('waiting queue')
+      SamplePublisher.new(waiting_queue)
+    end
+  end
+end

data/test/unit/queueable_test.rb

@@ -0,0 +1,24 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class QueueMe
+  include Refinery::Configurable
+  include Refinery::Queueable
+end
+
+class QueueableTest < Test::Unit::TestCase
+  context "a class with the Queuable module" do
+    should "provide a queue" do
+      setup_default_config
+      
+      queue = stub('queue')
+      queue_provider = stub('queue provider')
+      queue_provider.expects(:queue).with('a_queue').returns(queue)
+      RightAws::SqsGen2.expects(:new).with(
+        'aki', 'sak', {:multi_thread => true}
+      ).returns(queue_provider)
+      
+      queueable = QueueMe.new
+      assert_not_nil queueable.queue('a_queue')
+    end
+  end
+end

data/test/unit/server_test.rb

@@ -0,0 +1,39 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class ServerTest < Test::Unit::TestCase
+  context "the server class" do
+    should "provide a logger" do
+      assert_not_nil Refinery::Server.logger
+    end
+    context "logger" do
+      should "default to WARN level" do
+        assert_equal Logger::WARN, Refinery::Server.logger.level
+      end
+    end
+  end
+  context "a server instance" do
+    setup do
+      @server = Refinery::Server.new
+    end
+    should "have a config" do
+      assert_not_nil @server.config
+    end
+    should "be runnable" do
+      setup_default_config
+      
+      heartbeat_queue = stub('heartbeat queue')
+      heartbeat_queue.stubs(:send_message)
+      queue_provider = stub('queue provider')
+      queue_provider.expects(:queue).with('heartbeat').returns(heartbeat_queue)
+      RightAws::SqsGen2.expects(:new).with(
+        'aki', 'sak', {:multi_thread => true}
+      ).returns(queue_provider)
+      
+      assert_nothing_raised do
+        thread = Thread.new do
+          @server.run
+        end
+        @server.stop
+      end
+    end
+  end
+end

data/test/unit/statistics_test.rb

@@ -0,0 +1,41 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class StatisticsTest < Test::Unit::TestCase
+  context "a statistics instance" do
+    should "record a done message" do
+      dataset = stub('dataset')
+      dataset.expects(:<<) # TODO: improve this expectation
+      
+      db = stub('db')
+      db.stubs(:table_exists?).returns(true)
+      db.stubs(:[]).with(:completed_jobs).returns(dataset)
+      
+      Sequel.expects(:connect).with('sqlite://stats.db').returns(db)
+      
+      message = {
+        'host_info' => {'hostname' => 'test', 'pid' => 1},
+        'run_time' => 1,
+        'original' => ''
+      }
+      Refinery::Statistics.new.record_done(message)
+    end
+    
+    should "record an error message" do
+      dataset = stub('dataset')
+      dataset.expects(:<<) # TODO: improve this expectation
+      
+      db = stub('db')
+      db.stubs(:table_exists?).returns(true)
+      db.stubs(:[]).with(:errors).returns(dataset)
+      
+      Sequel.expects(:connect).with('sqlite://stats.db').returns(db)
+      
+      message = {
+        'host_info' => {'hostname' => 'test', 'pid' => 1},
+        'error' => {'class' => 'Error', 'message' => 'An error occurred.'},
+        'original' => ''
+      }
+      Refinery::Statistics.new.record_error(message)
+    end
+  end
+end

data/test/unit/utilities_test.rb

@@ -0,0 +1,25 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class UtilitiesIncluded
+  include Refinery::Utilities
+end
+
+class UtilitiesTest < Test::Unit::TestCase
+  context "a class with utilities included" do
+    setup do
+      @o = UtilitiesIncluded.new
+    end
+    should "camelize a word" do
+      
+      assert_equal 'ClassName', @o.camelize('class_name')
+    end
+    should "encode and decode message" do
+      message = {'some' => 'message'}
+      assert_equal message, @o.decode_message(@o.encode_message(message))
+    end
+    should "provide host info" do
+      assert_not_nil @o.host_info['pid']
+      assert_not_nil @o.host_info['hostname']
+    end
+  end
+end

data/test/unit/validations_test.rb

@@ -0,0 +1,37 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class ValidatedWorker < Refinery::Worker
+  include Refinery::Validations
+  validate_with do |message|
+    raise Refinery::InvalidMessageError, "A message is required" unless message
+  end
+  validates_key_exists :test
+  
+  def execute(message)
+  end
+end
+class ValidationsTest < Test::Unit::TestCase
+  context "a validated worker" do
+    setup do
+      daemon = stub('daemon')
+      @worker = ValidatedWorker.new(daemon)
+    end
+    should "raise a validation error if the key does not exist" do
+      message = {}
+      assert_raise Refinery::InvalidMessageError do
+        @worker.run(message)
+      end
+    end
+    should "not raise a validation error if the key does exist" do
+      message = {:test => 'ding!'}
+      assert_nothing_raised do
+        @worker.run(message)
+      end
+    end
+    should "raise a validation error if the message is nil" do
+      assert_raise Refinery::InvalidMessageError do
+        @worker.run(nil)
+      end
+    end
+  end
+end

data/test/unit/worker_test.rb

@@ -0,0 +1,44 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class SampleWorker < Refinery::Worker
+  attr_reader :message
+  def execute(message)
+    @message = message
+  end
+end
+
+class WorkerTest < Test::Unit::TestCase
+  context "a worker" do
+    setup do
+      daemon = stub('daemon')
+      @worker = SampleWorker.new(daemon)
+      @message = {'test' => 'value'}
+    end
+    should "run" do
+      @worker.run(@message)
+      assert_equal @message, @worker.message
+    end
+    should "provide a data store" do
+      options = {:bucket => 'bucket'}
+      
+      data_store = stub('data store')
+      
+      Moneta::S3.expects(:new).with(
+        :access_key_id => 'aki',  
+        :secret_access_key => 'sak',
+        :bucket => options[:bucket],
+        :multi_thread => true
+      ).returns(data_store)
+      
+      setup_default_config
+      assert_not_nil @worker.data_store(options)
+    end
+    should "provide a queue" do
+      queue = stub('queue')
+      queue_provider = stub('queue provider')
+      queue_provider.expects(:queue).with('a_queue').returns(queue)
+      @worker.expects(:queue_provider).returns(queue_provider)      
+      assert_not_nil @worker.queue('a_queue')
+    end
+  end
+end