aeden-refinery 0.9.0 → 0.9.1

data/lib/refinery/utilities.rb

@@ -0,0 +1,33 @@
+module Refinery #:nodoc:
+  # Utilities that can be mixed into a class
+  module Utilities
+    # Camelize the given word.
+    def camelize(word, first_letter_in_uppercase = true)
+      if first_letter_in_uppercase
+        word.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+      else
+        word.first.downcase + camelize(word)[1..-1]
+      end
+    end
+    
+    # Decode the message_body from Base 64 and then parse from JSON.
+    def decode_message(message_body)
+      JSON.parse(Base64.decode64(message_body))
+    end
+    
+    # Convert the given message_data object to JSON and then Base 64
+    # encode it
+    def encode_message(message_data)
+      Base64.encode64(message_data.to_json)
+    end
+    
+    # Get a Hash of useful host information that can be sent with
+    # messages to the monitoring system.
+    def host_info
+      {
+        'hostname' => Socket.gethostname,
+        'pid' => $$
+      }
+    end
+  end
+end

data/lib/refinery/validations.rb

@@ -0,0 +1,48 @@
+module Refinery #:nodoc:
+  # Error that is raised when a message is invalid.
+  class InvalidMessageError < RuntimeError
+  end
+  
+  # Module containing all validations.
+  module Validations
+    def self.included(base) # :nodoc:
+      base.extend(ClassMethods)
+    end
+    
+    # Class methods that are added to the worker.
+    module ClassMethods
+      # A list of all of the validators. Validators are lambdas
+      # that will be called with the message as its only arg.
+      # Note that the order of validators is retained.
+      def validators
+        @validators ||= []
+      end
+      
+      # Validate with the given block. The block must receive a single
+      # argument that is the message
+      def validate_with(&block)
+        validators << block
+      end
+      alias :validate :validate_with
+      
+      # Validate that each of the keys exists in the message.
+      def validate_key_exists(*args)
+        args.each do |key|
+          validators << lambda do |message|
+            raise Refinery::InvalidMessageError, "Key does not exist in message: #{key}" unless message[key]
+          end
+        end
+      end
+      alias :validates_key_exists :validate_key_exists
+      alias :validates_presence_of :validate_key_exists
+    end
+    
+    # Validate the given message
+    protected
+    def validate(message)
+      self.class.validators.each do |validator|
+        validator.call(message)
+      end
+    end
+  end
+end

data/lib/refinery/worker.rb

@@ -0,0 +1,65 @@
+module Refinery #:nodoc:
+  # Base class for workers. Place subclasses of this in the workers
+  # directory.
+  #
+  # Workers may include validation logic to verify that the message 
+  # has the correct keys and values before processing.
+  class Worker 
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Utilities
+    include Refinery::Validations
+    include Refinery::Queueable
+    
+    # Initialize the worker with the given daemon.
+    def initialize(daemon)
+      @daemon = daemon
+    end
+    
+    # Run the worker with the given message. The result from the worker's
+    # <code>execute</code> method is returned along with the run time.
+    #
+    # Validation will occur prior to calling execute.
+    def run(message)
+      result = false
+      
+      validate(message)
+      
+      logger.debug "Executing worker #{self.class.name}"
+      time = Benchmark.realtime do
+        begin
+          result = execute(message)
+        rescue Exception => e
+          logger.error "Error executing worker #{self.class.name}: #{e.message}"
+          raise e
+        end
+      end
+      logger.debug "Completed worker #{self.class.name} in #{time} seconds"
+      return result, time
+    end
+    
+    # Get the data store for the worker.
+    #
+    # The data store is provided through the Moneta interface.
+    #
+    # If the configuration providers a data_store:class option then that class
+    # will be used (the class must be in the Moneta module), otherwise 
+    # Moneta::S3 will be used.
+    def data_store(options)
+      class_name = processor_config['workers']['data_store']['class'] rescue 'S3'
+      ds_class = Moneta.const_get(camelize(class_name))
+      (@data_store ||= {})[options] ||= ds_class.new(
+        :access_key_id => config['aws']['credentials']['access_key_id'],  
+        :secret_access_key => config['aws']['credentials']['secret_access_key'],
+        :bucket => options[:bucket],
+        :multi_thread => true
+      )
+    end
+    
+    private
+    # Get's the config element starting at the processer
+    def processor_config
+      config['processors'][daemon.name]
+    end
+  end
+end

data/logs/README

@@ -0,0 +1 @@
+Log files from Refinery will be written here. Do not remove this directory.

data/publishers/error.rb

@@ -0,0 +1,8 @@
+# An example publisher that posts a message to the queue that should raise an error.
+class Error < Refinery::Publisher
+  def execute
+    if waiting_queue.size == 0
+      publish({'text' => 'fire an error, please'})
+    end
+  end
+end

data/publishers/sample.rb

@@ -0,0 +1,8 @@
+# A sample publisher that posts a message to the queue.
+class Sample < Refinery::Publisher
+  def execute
+    if waiting_queue.size == 0
+      publish({'text' => 'hey there!'})
+    end
+  end
+end

data/publishers/sleep.rb

@@ -0,0 +1,7 @@
+class Sleep < Refinery::Publisher
+  def execute
+    if waiting_queue.size == 0
+      publish({'seconds' => rand(5) + 0.5})
+    end
+  end
+end

data/refinery.gemspec

@@ -0,0 +1,105 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{refinery}
+  s.version = "0.9.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Anthony Eden"]
+  s.date = %q{2009-06-01}
+  s.description = %q{Process data in a distributed fashion.}
+  s.email = %q{anthonyeden@gmail.com}
+  s.executables = ["epub", "monitor", "pubnow", "refinery"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc",
+     "README.textile"
+  ]
+  s.files = [
+    ".autotest",
+     ".gitignore",
+     "CHANGELOG",
+     "LICENSE",
+     "README.rdoc",
+     "README.textile",
+     "Rakefile",
+     "VERSION",
+     "bin/epub",
+     "bin/monitor",
+     "bin/pubnow",
+     "bin/refinery",
+     "config/config.example.yml",
+     "lib/refinery.rb",
+     "lib/refinery/config.rb",
+     "lib/refinery/configurable.rb",
+     "lib/refinery/daemon.rb",
+     "lib/refinery/event_publisher.rb",
+     "lib/refinery/heartbeat.rb",
+     "lib/refinery/loggable.rb",
+     "lib/refinery/monitor.rb",
+     "lib/refinery/publisher.rb",
+     "lib/refinery/queueable.rb",
+     "lib/refinery/server.rb",
+     "lib/refinery/statistics.rb",
+     "lib/refinery/stats_server.rb",
+     "lib/refinery/utilities.rb",
+     "lib/refinery/validations.rb",
+     "lib/refinery/worker.rb",
+     "logs/README",
+     "publishers/error.rb",
+     "publishers/sample.rb",
+     "publishers/sleep.rb",
+     "refinery.gemspec",
+     "test/config.yml",
+     "test/test_helper.rb",
+     "test/unit/config_test.rb",
+     "test/unit/configurable_test.rb",
+     "test/unit/daemon_test.rb",
+     "test/unit/event_publisher_test.rb",
+     "test/unit/heartbeat_test.rb",
+     "test/unit/loggable_test.rb",
+     "test/unit/publisher_test.rb",
+     "test/unit/queueable_test.rb",
+     "test/unit/server_test.rb",
+     "test/unit/statistics_test.rb",
+     "test/unit/utilities_test.rb",
+     "test/unit/validations_test.rb",
+     "test/unit/worker_test.rb",
+     "workers/error.rb",
+     "workers/sample.rb",
+     "workers/sleep.rb"
+  ]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/aeden/refinery}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{refinery}
+  s.rubygems_version = %q{1.3.1}
+  s.summary = %q{Refinery processes data in a distributed environment.}
+  s.test_files = [
+    "test/test_helper.rb",
+     "test/unit/config_test.rb",
+     "test/unit/configurable_test.rb",
+     "test/unit/daemon_test.rb",
+     "test/unit/event_publisher_test.rb",
+     "test/unit/heartbeat_test.rb",
+     "test/unit/loggable_test.rb",
+     "test/unit/publisher_test.rb",
+     "test/unit/queueable_test.rb",
+     "test/unit/server_test.rb",
+     "test/unit/statistics_test.rb",
+     "test/unit/utilities_test.rb",
+     "test/unit/validations_test.rb",
+     "test/unit/worker_test.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/test/config.yml

@@ -0,0 +1,10 @@
+aws:
+  credentials:
+    access_key_id: "aaa"
+    secret_access_key: "bbb"
+processors:
+  sample:
+    publishers:
+      delay: 10
+    workers:
+      initial: 3

data/test/test_helper.rb

@@ -0,0 +1,21 @@
+require 'test/unit'
+require 'rubygems'
+require 'shoulda'
+require 'mocha'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+class Test::Unit::TestCase
+  def setup_default_config
+    Refinery::Config.stubs(:default).returns(Refinery::Config.new(
+      {
+        'aws' => {
+          'credentials' => {
+            'access_key_id' => 'aki',
+            'secret_access_key' => 'sak'
+          }
+        },
+        'processors' => []
+      }
+    ))
+  end
+end

data/test/unit/config_test.rb

@@ -0,0 +1,42 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class ConfigTest < Test::Unit::TestCase
+  context "the config class" do
+    should "provide a default configuration" do
+      assert_not_nil Refinery::Config.default
+    end
+    
+    context "default configuration" do
+      setup do
+        @config = Refinery::Config.default
+      end
+      should "provide an empty aws credentials hash" do
+        assert_equal Hash.new, @config['aws']['credentials']
+      end
+      should "provide an empty processors hash" do
+        assert_equal Hash.new, @config['processors']
+      end
+    end
+    
+    context "after loading configuration from a YAML file" do
+      setup do
+        @config_file = File.dirname(__FILE__) + '/../config.yml'
+        @config = Refinery::Config.new
+        @config.load_file(@config_file)
+      end
+      should "have aws credentials" do
+        assert_equal 'aaa', @config['aws']['credentials']['access_key_id']
+        assert_equal 'bbb', @config['aws']['credentials']['secret_access_key']
+      end
+      should "reload the file when changed" do
+        `touch #{@config_file}`
+        YAML.expects(:load_file).once
+        @config.refresh
+      end
+      should "not reload the file when not changed" do
+        YAML.expects(:load_file).never
+        @config.refresh
+      end
+    end
+  end
+end

data/test/unit/configurable_test.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class ConfigureMe
+  include Refinery::Configurable
+end
+class ConfigurableTest < Test::Unit::TestCase
+  context "a class with the configurable module" do
+    should "provide a config" do
+      assert_not_nil ConfigureMe.new.config
+    end
+  end
+end

data/test/unit/daemon_test.rb

@@ -0,0 +1,37 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class DaemonTest < Test::Unit::TestCase
+  context "a daemon" do
+    setup do
+      @server = stub('Server')
+      @waiting_queue = stub('Queue(waiting)')
+      @waiting_queue.stubs(:receive)
+      @error_queue = stub('Queue(error)')
+      @done_queue = stub('Queue(done)')
+    end
+    should "be startable" do
+      assert_nothing_raised do
+        daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue)
+      end
+    end
+    should "have logging" do
+      daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue)
+      assert_not_nil daemon.logger
+    end
+    context "that is started" do
+      setup do
+        @daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue)
+      end
+      should "have a state of running" do
+        assert @daemon.running?
+      end
+      context "after calling stop" do
+        setup do
+          @daemon.stop
+        end
+        should "not be running" do
+          assert !@daemon.running?
+        end
+      end
+    end
+  end
+end

data/test/unit/event_publisher_test.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class EventPublisherTest < Test::Unit::TestCase
+  context "an event publisher" do
+    should "raise an error if credentials are not set" do
+      publishing_settings = {'sample' => {'delay' => 10}}
+      Refinery::Config.any_instance.stubs(:publishing).returns(publishing_settings)
+      event_publisher = Refinery::EventPublisher.new
+      event_publisher.run
+    end
+  end
+end

data/test/unit/heartbeat_test.rb

@@ -0,0 +1,22 @@
+require File.dirname(__FILE__) + '/../test_helper'
+class HeartbeatTest < Test::Unit::TestCase
+  context "a heartbeat" do
+    setup do
+      setup_default_config
+      
+      @server = stub('server')
+      @server.stubs(:daemons).returns([])
+      
+      heartbeat_queue = stub('heartbeat queue')
+      heartbeat_queue.stubs(:send_message)
+      queue_provider = stub('queue provider')
+      queue_provider.expects(:queue).with('heartbeat').returns(heartbeat_queue)
+      RightAws::SqsGen2.stubs(:new).with(
+        'aki', 'sak', {:multi_thread => true}
+      ).returns(queue_provider)
+    end
+    should "be initializable" do
+      Refinery::Heartbeat.new(@server)
+    end
+  end
+end