refinery 0.12.2 → 1.0

data/lib/refinery/validations.rb

@@ -1,48 +0,0 @@
-module Refinery #:nodoc:
-  # Error that is raised when a message is invalid.
-  class InvalidMessageError < RuntimeError
-  end
-  
-  # Module containing all validations.
-  module Validations
-    def self.included(base) # :nodoc:
-      base.extend(ClassMethods)
-    end
-    
-    # Class methods that are added to the worker.
-    module ClassMethods
-      # A list of all of the validators. Validators are lambdas
-      # that will be called with the message as its only arg.
-      # Note that the order of validators is retained.
-      def validators
-        @validators ||= []
-      end
-      
-      # Validate with the given block. The block must receive a single
-      # argument that is the message
-      def validate_with(&block)
-        validators << block
-      end
-      alias :validate :validate_with
-      
-      # Validate that each of the keys exists in the message.
-      def validate_key_exists(*args)
-        args.each do |key|
-          validators << lambda do |message|
-            raise Refinery::InvalidMessageError, "Key does not exist in message: #{key}" unless message[key]
-          end
-        end
-      end
-      alias :validates_key_exists :validate_key_exists
-      alias :validates_presence_of :validate_key_exists
-    end
-    
-    # Validate the given message
-    protected
-    def validate(message)
-      self.class.validators.each do |validator|
-        validator.call(message)
-      end
-    end
-  end
-end

data/lib/refinery/worker.rb

@@ -1,65 +0,0 @@
-module Refinery #:nodoc:
-  # Base class for workers. Place subclasses of this in the workers
-  # directory.
-  #
-  # Workers may include validation logic to verify that the message 
-  # has the correct keys and values before processing.
-  class Worker 
-    include Refinery::Loggable
-    include Refinery::Configurable
-    include Refinery::Utilities
-    include Refinery::Validations
-    include Refinery::Queueable
-    
-    # Initialize the worker with the given daemon.
-    def initialize(daemon)
-      @daemon = daemon
-    end
-    
-    # Run the worker with the given message. The result from the worker's
-    # <code>execute</code> method is returned along with the run time.
-    #
-    # Validation will occur prior to calling execute.
-    def run(message)
-      result = false
-      
-      validate(message)
-      
-      logger.debug "Executing worker #{self.class.name}"
-      time = Benchmark.realtime do
-        begin
-          result = execute(message)
-        rescue Exception => e
-          logger.error "Error executing worker #{self.class.name}: #{e.message}"
-          raise e
-        end
-      end
-      logger.debug "Completed worker #{self.class.name} in #{time} seconds"
-      return result, time
-    end
-    
-    # Get the data store for the worker.
-    #
-    # The data store is provided through the Moneta interface.
-    #
-    # If the configuration providers a data_store:class option then that class
-    # will be used (the class must be in the Moneta module), otherwise 
-    # Moneta::S3 will be used.
-    def data_store(options)
-      class_name = processor_config['workers']['data_store']['class'] rescue 'S3'
-      ds_class = Moneta.const_get(camelize(class_name))
-      (@data_store ||= {})[options] ||= ds_class.new(
-        :access_key_id => config['aws']['credentials']['access_key_id'],  
-        :secret_access_key => config['aws']['credentials']['secret_access_key'],
-        :bucket => options[:bucket],
-        :multi_thread => true
-      )
-    end
-    
-    protected
-    # Get's the config element starting at the processer
-    def processor_config
-      config['processors'][daemon.name]
-    end
-  end
-end

data/logs/README

@@ -1 +0,0 @@
-Log files from Refinery will be written here. Do not remove this directory.

data/publishers/error.rb

@@ -1,6 +0,0 @@
-# An example publisher that posts a message to the queue that should raise an error.
-class ErrorPublisher < Refinery::Publisher
-  def execute
-    publish_if_empty({'text' => 'fire an error, please'})
-  end
-end

data/publishers/sample.rb

@@ -1,6 +0,0 @@
-# A sample publisher that posts a message to the queue.
-class SamplePublisher < Refinery::Publisher
-  def execute
-    publish_if_empty({'text' => 'hey there!'})
-  end
-end

data/publishers/sleep.rb

@@ -1,5 +0,0 @@
-class SleepPublisher < Refinery::Publisher
-  def execute
-    publish({'seconds' => rand(5) + 0.5})
-  end
-end

data/test/config.yml

@@ -1,10 +0,0 @@
-aws:
-  credentials:
-    access_key_id: "aaa"
-    secret_access_key: "bbb"
-processors:
-  sample:
-    publishers:
-      delay: 10
-    workers:
-      initial: 3

data/test/test_helper.rb

@@ -1,21 +0,0 @@
-require 'test/unit'
-require 'rubygems'
-require 'shoulda'
-require 'mocha'
-require 'refinery'
-
-class Test::Unit::TestCase
-  def setup_default_config
-    Refinery::Config.stubs(:default).returns(Refinery::Config.new(
-      {
-        'aws' => {
-          'credentials' => {
-            'access_key_id' => 'aki',
-            'secret_access_key' => 'sak'
-          }
-        },
-        'processors' => []
-      }
-    ))
-  end
-end

data/test/unit/config_test.rb

@@ -1,42 +0,0 @@
-require 'test_helper'
-
-class ConfigTest < Test::Unit::TestCase
-  context "the config class" do
-    should "provide a default configuration" do
-      assert_not_nil Refinery::Config.default
-    end
-    
-    context "default configuration" do
-      setup do
-        @config = Refinery::Config.default
-      end
-      should "provide an empty aws credentials hash" do
-        assert_equal Hash.new, @config['aws']['credentials']
-      end
-      should "provide an empty processors hash" do
-        assert_equal Hash.new, @config['processors']
-      end
-    end
-    
-    context "after loading configuration from a YAML file" do
-      setup do
-        @config_file = File.dirname(__FILE__) + '/../config.yml'
-        @config = Refinery::Config.new
-        @config.load_file(@config_file)
-      end
-      should "have aws credentials" do
-        assert_equal 'aaa', @config['aws']['credentials']['access_key_id']
-        assert_equal 'bbb', @config['aws']['credentials']['secret_access_key']
-      end
-      should "reload the file when changed" do
-        `touch #{@config_file}`
-        YAML.expects(:load_file).once
-        @config.refresh
-      end
-      should "not reload the file when not changed" do
-        YAML.expects(:load_file).never
-        @config.refresh
-      end
-    end
-  end
-end

data/test/unit/configurable_test.rb

@@ -1,13 +0,0 @@
-require 'test_helper'
-
-class ConfigureMe
-  include Refinery::Configurable
-end
-
-class ConfigurableTest < Test::Unit::TestCase
-  context "a class with the configurable module" do
-    should "provide a config" do
-      assert_not_nil ConfigureMe.new.config
-    end
-  end
-end

data/test/unit/daemon_test.rb

@@ -1,63 +0,0 @@
-require 'test_helper'
-
-class DaemonTest < Test::Unit::TestCase
-  context "a daemon" do
-    setup do
-      @server = stub('Server')
-      @processor = stub('Processor', :server => @server)
-      
-      @waiting_queue = stub('Queue(waiting)')
-      @error_queue = stub('Queue(error)')
-      @done_queue = stub('Queue(done)')
-      
-      @provider = stub('QueueProvider')
-      if defined?(Typica)
-        Typica::Sqs::QueueService.stubs(:new).returns(@provider)
-      else
-        RightAws::SqsGen2.stubs(:new).returns(@provider)
-      end
-      @provider.stubs(:queue).with('sample_waiting').returns(@waiting_queue)
-      @provider.stubs(:queue).with('sample_error').returns(@error_queue)
-      @provider.stubs(:queue).with('sample_done').returns(@done_queue)
-    end
-    should "be startable" do
-      @waiting_queue.stubs(:receive)
-      assert_nothing_raised do
-        daemon = Refinery::Daemon.new(@processor, 'sample')
-      end
-    end
-    should "have logging" do
-      @waiting_queue.stubs(:receive)
-      daemon = Refinery::Daemon.new(@processor, 'sample')
-      assert_not_nil daemon.logger
-    end
-    should "allow visibility setting" do
-      @waiting_queue.stubs(:receive).with(600)
-      daemon = Refinery::Daemon.new(@processor, 'sample', '', {'visibility' => 600})
-    end
-    should "have a queue name" do
-      @waiting_queue.stubs(:receive)
-      @provider.stubs(:queue).with(
-      'prefix_sample_waiting').returns(@waiting_queue)
-      daemon = Refinery::Daemon.new(@processor, 'sample', 'prefix_')
-      assert_equal 'prefix_sample', daemon.queue_name
-    end
-    context "that is started" do
-      setup do
-        @waiting_queue.stubs(:receive)
-        @daemon = Refinery::Daemon.new(@processor, 'sample')
-      end
-      should "have a state of running" do
-        assert @daemon.running?
-      end
-#       context "after calling stop" do
-#         setup do
-#           @daemon.stop
-#         end
-#         should "not be running" do
-#           assert !@daemon.running?
-#         end
-#       end
-    end
-  end
-end

data/test/unit/event_publisher_test.rb

@@ -1,12 +0,0 @@
-require 'test_helper'
-
-class EventPublisherTest < Test::Unit::TestCase
-  context "an event publisher" do
-    should "raise an error if credentials are not set" do
-      publishing_settings = {'sample' => {'delay' => 10}}
-      Refinery::Config.any_instance.stubs(:publishing).returns(publishing_settings)
-      event_publisher = Refinery::EventPublisher.new
-      event_publisher.run
-    end
-  end
-end

data/test/unit/heartbeat_test.rb

@@ -1,25 +0,0 @@
-require 'test_helper'
-
-class HeartbeatTest < Test::Unit::TestCase
-  context "a heartbeat" do
-    setup do
-      setup_default_config
-      
-      @server = stub('server')
-      @server.stubs(:daemons).returns([])
-      
-      heartbeat_queue = stub('heartbeat queue')
-      heartbeat_queue.stubs(:send_message)
-      provider = stub('queue provider')
-      provider.stubs(:queue).with('heartbeat').returns(heartbeat_queue)
-      if defined?(Typica)
-        Typica::Sqs::QueueService.stubs(:new).returns(provider)
-      else
-        RightAws::SqsGen2.stubs(:new).returns(provider)
-      end
-    end
-    should "be initializable" do
-      Refinery::Heartbeat.new(@server)
-    end
-  end
-end

data/test/unit/loggable_test.rb

@@ -1,12 +0,0 @@
-require 'test_helper'
-
-class LogMe
-  include Refinery::Loggable
-end
-class LoggableTest < Test::Unit::TestCase
-  context "a class with Loggable mixed in" do
-    should "have a logger" do
-      assert_not_nil LogMe.new.logger
-    end
-  end
-end

data/test/unit/processor_test.rb

@@ -1,34 +0,0 @@
-require 'test_helper'
-
-class ProcessorTest < Test::Unit::TestCase
-  context "a processor" do
-    setup do
-      @server = stub('Server')
-      @settings = {
-        'workers' => {
-          'initial' => 1
-        }
-      }
-      
-      @waiting_queue = stub('Queue(waiting)')
-      @error_queue = stub('Queue(error)')
-      @done_queue = stub('Queue(done)')
-      
-      provider = stub('QueueProvider')
-      if defined?(Typica)
-        Typica::Sqs::QueueService.stubs(:new).returns(provider)
-      else
-        RightAws::SqsGen2.stubs(:new).returns(provider)
-      end
-      provider.stubs(:queue).with('sample_waiting').returns(@waiting_queue)
-      provider.stubs(:queue).with('sample_error').returns(@error_queue)
-      provider.stubs(:queue).with('sample_done').returns(@done_queue)
-    end
-    should "initialize" do
-      assert_nothing_raised do
-        @waiting_queue.stubs(:receive)
-        Refinery::Processor.new(@server, 'sample', @settings)
-      end
-    end
-  end
-end

data/test/unit/publisher_test.rb

@@ -1,13 +0,0 @@
-require 'test_helper'
-
-class SamplePublisher < Refinery::Publisher
-end
-
-class PublisherTest < Test::Unit::TestCase
-  context "a publisher" do
-    should "be instantiable" do
-      waiting_queue = stub('waiting queue')
-      SamplePublisher.new(waiting_queue)
-    end
-  end
-end

data/test/unit/queueable_test.rb

@@ -1,26 +0,0 @@
-require 'test_helper'
-
-class QueueMe
-  include Refinery::Configurable
-  include Refinery::Queueable
-end
-
-class QueueableTest < Test::Unit::TestCase
-  context "a class with the Queuable module" do
-    should "provide a queue" do
-      setup_default_config
-      
-      queue = stub('queue')
-      provider = stub('queue provider')
-      provider.expects(:queue).with('a_queue').returns(queue)
-      if defined?(Typica)
-        Typica::Sqs::QueueService.expects(:new).returns(provider)
-      else
-        RightAws::SqsGen2.expects(:new).with('aki', 'sak', {:multi_thread => true}).returns(provider)
-      end
-      
-      queueable = QueueMe.new
-      assert_not_nil queueable.queue('a_queue')
-    end
-  end
-end

data/test/unit/server_test.rb

@@ -1,34 +0,0 @@
-require 'test_helper'
-
-class ServerTest < Test::Unit::TestCase
-  context "the server class" do
-    should "provide a logger" do
-      assert_not_nil Refinery::Server.logger
-    end
-    context "logger" do
-      should "default to WARN level" do
-        assert_equal Logger::WARN, Refinery::Server.logger.level
-      end
-    end
-  end
-  context "a server instance" do
-    setup do
-      @server = Refinery::Server.new
-    end
-    should "have a config" do
-      assert_not_nil @server.config
-    end
-    should "be runnable" do
-      setup_default_config
-      
-      #Refinery::Heartbeat.expects(:new)
-      
-      assert_nothing_raised do
-        thread = Thread.new do
-          @server.run
-        end
-        @server.stop
-      end
-    end
-  end
-end