aeden-refinery 0.9.0 → 0.9.1

data/.autotest

@@ -0,0 +1,10 @@
+module Autotest::CustomTestMatch
+  Autotest.add_hook :initialize do |at|
+    at.add_mapping(/test/) do |f, _|
+      at.files_matching(/_test\.rb$/)
+    end
+    at.add_mapping(/lib\/.*/) do |f, _|
+      at.files_matching(/_test\.rb$/)
+    end
+  end
+end

data/.gitignore

@@ -0,0 +1,5 @@
+logs/*.log
+config/config.yml
+rdoc/*
+refinery-*.gem
+stats.db

data/Rakefile

@@ -20,4 +20,20 @@ Rake::RDocTask.new(:rdoc) do |rdoc|
   rdoc.rdoc_files.include('README.rdoc')
   rdoc.rdoc_files.include('lib/*.rb')
   rdoc.rdoc_files.include('lib/**/*.rb')
-end
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "refinery"
+    gemspec.summary = "Refinery processes data in a distributed environment."
+    gemspec.email = "anthonyeden@gmail.com"
+    gemspec.homepage = "http://github.com/aeden/refinery"
+    gemspec.description = "Process data in a distributed fashion."
+    gemspec.authors = ["Anthony Eden"]
+    gemspec.files.exclude 'docs/**/*'
+    gemspec.rubyforge_project = 'refinery'
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION

@@ -0,0 +1 @@
+0.9.1

data/config/config.example.yml

@@ -0,0 +1,18 @@
+aws:
+  credentials:
+    access_key_id: "access_key_id"
+    secret_access_key: "secret_access_key"
+processors:
+  sample:
+    queue: 'sample' # can be omitted
+    publishers:
+      delay: 30
+    workers:
+      initial: 3
+      data_store:
+        class: s3
+  error:
+    publishers:
+      delay: 30
+    workers:
+      initial: 1

data/lib/refinery.rb

@@ -0,0 +1,74 @@
+$:.unshift(File.dirname(__FILE__))
+
+require 'logger'
+require 'socket'
+require 'benchmark'
+
+# The Refinery module contains all of the classes for the refinery system.
+module Refinery
+  
+  # Require the specified library.
+  #
+  # The short name is the require path and the display_name will be shown
+  # if the library cannot be loaded.
+  def self.require_library(short_name, display_name)
+    begin
+      require short_name
+    rescue LoadError
+      puts "#{display_name} is required, please install it"
+      exit
+    end
+  end
+  
+  # Require all of the dependencies.
+  def self.require_libraries
+    require_library('rubygems', 'Rubygems')
+    require_library('right_aws', 'RightScale AWS gem')
+    require_library('json', 'JSON gem')
+    require_library('moneta', 'Moneta gem')
+    require_library('moneta/s3', 'Moneta S3 implementation')
+  end
+  
+  def self.require_optional_library(short_name, display_name)
+    begin
+      require short_name
+    rescue LoadError
+    end
+  end
+  
+  def self.require_optional_libraries
+    require_optional_library('sequel', 'Sequel gem')
+    require_optional_library('ramaze', 'Ramaze')
+  end
+  
+  # Require internal code files
+  def self.require_internals
+    require 'refinery/loggable'
+    require 'refinery/configurable'
+    require 'refinery/queueable'
+
+    require 'refinery/utilities'
+    
+    require 'refinery/validations'
+
+    require 'refinery/config'
+    require 'refinery/heartbeat'
+    require 'refinery/server'
+    require 'refinery/daemon'
+    require 'refinery/worker'
+    require 'refinery/event_publisher'
+    require 'refinery/publisher'
+    require 'refinery/monitor'
+    require 'refinery/statistics'
+    require 'refinery/stats_server'
+    
+  end
+  
+  # Raised if a source file cannot be loaded
+  class SourceFileNotFound < RuntimeError
+  end
+end
+
+Refinery::require_libraries
+Refinery::require_optional_libraries
+Refinery::require_internals

data/lib/refinery/config.rb

@@ -0,0 +1,48 @@
+module Refinery #:nodoc:
+  # Configuration class.
+  class Config
+    # Get a shared configuration
+    def self.default
+      @default ||= new({
+        'aws' => {
+          'credentials' => {}
+        },
+        'processors' => {}
+      })
+    end
+    
+    # Initialize the config with the given data
+    def initialize(data={})
+      @data = data
+    end
+    
+    # Get the configuration value
+    def [](key)
+      data[key.to_s]
+    end
+    
+    # Set the configuration value
+    def []=(key, value)
+      data[key.to_s] = value
+    end
+    
+    # Load configuration from a YAML file
+    def load_file(file)
+      @file = file
+      @data = YAML::load_file(@file)
+      @last_load = File.mtime(@file)
+    end
+    
+    # Refresh the configuration from the YAML file if necessary.
+    def refresh
+      if File.mtime(@file) != @last_load
+        @data = YAML::load_file(@file)
+      end
+    end
+    
+    private
+    def data
+      @data ||= {}
+    end
+  end
+end

data/lib/refinery/configurable.rb

@@ -0,0 +1,15 @@
+module Refinery #:nodoc:
+  # Include this module to get access to a shared configuration
+  module Configurable
+    # Get the configuration. If the config is nil then this will use
+    # the default shared configuration.
+    def config
+      @config ||= Refinery::Config.default
+    end
+    
+    # Set the configuration.
+    def config=(config)
+      @config = config
+    end
+  end
+end

data/lib/refinery/daemon.rb

@@ -0,0 +1,124 @@
+module Refinery #:nodoc:
+  # A daemon provides a thread to run workers in.
+  class Daemon
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Utilities
+    
+    RUNNING = 'running'
+    STOPPED = 'stopped'
+    
+    # The daemon's thread
+    attr_reader :thread
+    # The name of the daemon
+    attr_reader :name
+    # The queue for incoming messages to process
+    attr_reader :waiting_queue
+    # The queue for outgoing messages once they've been processed
+    attr_reader :done_queue
+    # The queue for error messages
+    attr_reader :error_queue
+    
+    # Stop the daemon
+    def stop
+      self.state = STOPPED
+    end
+    
+    # Return the daemon state
+    def state
+      @state ||= RUNNING
+    end
+    
+    # Set the daemon state.
+    def state=(state)
+      @state = state
+    end
+    protected :state
+    
+    # Return true if the daemon state is running.
+    def running?
+      state == RUNNING
+    end
+    
+    # Initialize the daemon.
+    #
+    # * <tt>server</tt>: The server instance
+    # * <tt>name</tt>: The processor name
+    # * <tt>waiting_queue</tt>: The waiting queue that provides messages to be processed
+    # * <tt>error_queue</tt>: The queue where errors are posted.
+    # * <tt>done_queue</tt>: The queue for messages that have been processed.
+    def initialize(server, name, waiting_queue, error_queue, done_queue)
+      Refinery::Server.logger.debug "Starting daemon"
+      
+      @server = server
+      @name = name
+      @waiting_queue = waiting_queue
+      @error_queue = error_queue
+      @done_queue = done_queue
+      
+      @thread = Thread.new(self) do |daemon|
+        logger.debug "Running daemon thread"
+        while(running?)
+          begin
+            while (message = waiting_queue.receive)
+              worker = load_worker_class(name).new(self)
+              begin
+                result, run_time = worker.run(decode_message(message.body))
+                if result
+                  done_message = {
+                    'host_info' => host_info,
+                    'original' => message.body,
+                    'run_time' => run_time
+                  }
+                  logger.debug "Sending 'done' message to #{done_queue.name}"
+                  done_queue.send_message(encode_message(done_message))
+                
+                  logger.debug "Deleting message from queue"
+                  message.delete()
+                end
+              rescue Exception => e
+                error_message = {
+                  'error' => {
+                    'message' => e.message, 
+                    'class' => e.class.name
+                  }, 
+                  'host_info' => host_info,
+                  'original' => message.body
+                }
+                error_queue.send_message(encode_message(error_message))
+                message.delete()
+              end
+            end
+            sleep(1)
+          rescue Exception => e
+            logger.error "An error occurred while receiving from the waiting queue: #{e.message}"
+          end
+        end
+        logger.debug "Exiting daemon thread"
+      end
+    end
+    
+    # A hash of worker classes
+    def workers
+      @workers ||= {}
+    end
+    
+    private
+    # Load the appropriate worker class
+    def load_worker_class(name)
+      source_file = "#{@server.workers_directory}/#{name}.rb"
+      if File.exist?(source_file)
+        modified_at = File.mtime(source_file)
+        if workers[name] != modified_at
+          logger.debug "Loading #{source_file}"
+          load(source_file)
+          workers[name] = modified_at
+        end
+      else
+        raise SourceFileNotFound, "Source file not found: #{source_file}"
+      end
+      
+      Object.const_get(camelize(name))
+    end
+  end
+end

data/lib/refinery/event_publisher.rb

@@ -0,0 +1,120 @@
+module Refinery #:nodoc:
+  # Publish events.
+  class EventPublisher
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Queueable
+    include Refinery::Utilities
+    
+    STARTING = 'starting' #:nodoc:
+    RUNNING = 'running' #:nodoc:
+    STOPPED = 'stopped' #:nodoc:
+    
+    attr_accessor :publishers_directory
+    
+    # Initialize the event publisher
+    #
+    # Options:
+    # * <tt>:debug</tt>: Set to true to enable debug logging
+    # * <tt>:config</tt>: Provide a file path to load that config
+    def initialize(options={})
+      logger.level = Logger::INFO if options[:verbose]
+      logger.level = Logger::DEBUG if options[:debug]
+      config.load_file(options[:config]) if options[:config]
+      self.publishers_directory = options[:publishers] if options[:publishers]
+    end
+    
+    # Get the event publisher state
+    def state
+      @state ||= STARTING
+    end
+    
+    # Return true if the event publisher is running
+    def running?
+      state == RUNNING
+    end
+    
+    # The directory where publishers are found. Defaults to ./publishers
+    def publishers_directory
+      @publishers_directory ||= './publishers'
+    end
+    
+    # A hash of all publisher classes mapped to last modified timestamps.
+    def publishers
+      @publishers ||= {}
+    end
+    
+    # Run the specified publisher once and return
+    def run_once(key)
+      settings = config['processors'][key]
+      raise RuntimeError, "No processor configuration found for #{key}" unless settings
+      queue_name = settings['queue'] || key
+      logger.debug "Using queue #{queue_name}_waiting"
+      waiting_queue = queue("#{queue_name}_waiting")
+      load_publisher_class(key).new(waiting_queue).execute
+    end
+    
+    # Run the event publisher
+    def run
+      @state = RUNNING
+      logger.info "Starting event publisher"
+      config['processors'].each do |key, settings|
+        run_publisher(key, settings)
+      end
+      
+      begin
+        threads.each { |thread| thread.join }
+      rescue Interrupt => e
+      end
+      
+      logger.info "Exiting event publisher"
+    end
+    
+    private
+    # An array of threads, one for each publisher instance
+    def threads
+      @threads ||= []
+    end
+    
+    # Run the publisher for the given key
+    def run_publisher(key, settings)
+      logger.info "Creating publisher for #{key}"
+      queue_name = settings['queue'] || key
+      logger.debug "Using queue #{queue_name}_waiting"
+      waiting_queue = queue("#{queue_name}_waiting")
+      
+      threads << Thread.new(waiting_queue, settings) do |waiting_queue, settings|
+        while(running?)
+          begin
+            load_publisher_class(key).new(waiting_queue).execute
+          rescue Exception => e
+            logger.error e
+            raise e
+          end
+          
+          delay = settings['publishers']['delay'] || 60
+          logger.debug "Sleeping #{delay} seconds"
+          sleep delay
+          
+        end
+      end
+    end
+    
+    def load_publisher_class(key)
+      source_file = "#{publishers_directory}/#{key}.rb"
+      if File.exist?(source_file)
+        modified_at = File.mtime(source_file)
+        if publishers[key] != modified_at
+          logger.debug "Loading #{source_file}"
+          load(source_file)
+          publishers[key] = modified_at
+        end
+      else
+        raise SourceFileNotFound, "Source file not found: #{source_file}"
+      end
+      
+      Object.const_get(camelize(key))
+    end
+    
+  end
+end