refinery 0.9.0

data/lib/refinery/event_publisher.rb

@@ -0,0 +1,120 @@
+module Refinery #:nodoc:
+  # Publish events.
+  class EventPublisher
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Queueable
+    include Refinery::Utilities
+    
+    STARTING = 'starting' #:nodoc:
+    RUNNING = 'running' #:nodoc:
+    STOPPED = 'stopped' #:nodoc:
+    
+    attr_accessor :publishers_directory
+    
+    # Initialize the event publisher
+    #
+    # Options:
+    # * <tt>:debug</tt>: Set to true to enable debug logging
+    # * <tt>:config</tt>: Provide a file path to load that config
+    def initialize(options={})
+      logger.level = Logger::INFO if options[:verbose]
+      logger.level = Logger::DEBUG if options[:debug]
+      config.load_file(options[:config]) if options[:config]
+      self.publishers_directory = options[:publishers] if options[:publishers]
+    end
+    
+    # Get the event publisher state
+    def state
+      @state ||= STARTING
+    end
+    
+    # Return true if the event publisher is running
+    def running?
+      state == RUNNING
+    end
+    
+    # The directory where publishers are found. Defaults to ./publishers
+    def publishers_directory
+      @publishers_directory ||= './publishers'
+    end
+    
+    # A hash of all publisher classes mapped to last modified timestamps.
+    def publishers
+      @publishers ||= {}
+    end
+    
+    # Run the specified publisher once and return
+    def run_once(key)
+      settings = config['processors'][key]
+      raise RuntimeError, "No processor configuration found for #{key}" unless settings
+      queue_name = settings['queue'] || key
+      logger.debug "Using queue #{queue_name}_waiting"
+      waiting_queue = queue("#{queue_name}_waiting")
+      load_publisher_class(key).new(waiting_queue).execute
+    end
+    
+    # Run the event publisher
+    def run
+      @state = RUNNING
+      logger.info "Starting event publisher"
+      config['processors'].each do |key, settings|
+        run_publisher(key, settings)
+      end
+      
+      begin
+        threads.each { |thread| thread.join }
+      rescue Interrupt => e
+      end
+      
+      logger.info "Exiting event publisher"
+    end
+    
+    private
+    # An array of threads, one for each publisher instance
+    def threads
+      @threads ||= []
+    end
+    
+    # Run the publisher for the given key
+    def run_publisher(key, settings)
+      logger.info "Creating publisher for #{key}"
+      queue_name = settings['queue'] || key
+      logger.debug "Using queue #{queue_name}_waiting"
+      waiting_queue = queue("#{queue_name}_waiting")
+      
+      threads << Thread.new(waiting_queue, settings) do |waiting_queue, settings|
+        while(running?)
+          begin
+            load_publisher_class(key).new(waiting_queue).execute
+          rescue Exception => e
+            logger.error e
+            raise e
+          end
+          
+          delay = settings['publishers']['delay'] || 60
+          logger.debug "Sleeping #{delay} seconds"
+          sleep delay
+          
+        end
+      end
+    end
+    
+    def load_publisher_class(key)
+      source_file = "#{publishers_directory}/#{key}.rb"
+      if File.exist?(source_file)
+        modified_at = File.mtime(source_file)
+        if publishers[key] != modified_at
+          logger.debug "Loading #{source_file}"
+          load(source_file)
+          publishers[key] = modified_at
+        end
+      else
+        raise SourceFileNotFound, "Source file not found: #{source_file}"
+      end
+      
+      Object.const_get(camelize(key))
+    end
+    
+  end
+end

data/lib/refinery/heartbeat.rb

@@ -0,0 +1,30 @@
+module Refinery #:nodoc:
+  # A heartbeat publisher that indicates a server is alive.
+  class Heartbeat
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Queueable
+    include Refinery::Utilities
+    
+    # Initialize the heartbeat for the given server.
+    def initialize(server)
+      @server = server
+      @thread = Thread.new(queue('heartbeat')) do |heartbeat_queue|
+        loop do
+          begin
+            logger.debug "Send heartbeat"
+            message = {
+              'host_info' => host_info,
+              'timestamp' => Time.now.utc,
+              'running_daemons' => @server.daemons.length
+            }
+            heartbeat_queue.send_message(Base64.encode64(message.to_json))
+            sleep(60)
+          rescue Exception => e
+            logger.error "Error sending heartbeat: #{e.message}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/refinery/loggable.rb

@@ -0,0 +1,9 @@
+module Refinery
+  # Include this module to get access to the server logger
+  module Loggable
+    # Get the logger.
+    def logger
+      @logger ||= Refinery::Server.logger
+    end
+  end
+end

data/lib/refinery/monitor.rb

@@ -0,0 +1,116 @@
+module Refinery #:nodoc:
+  # The monitor is responsible for monitoring the health of the various
+  # components of refinery.
+  class Monitor
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Queueable
+    include Refinery::Utilities
+    
+    # Initialize the monitor.
+    #
+    # Options:
+    # * <tt>:verbose</tt>: Enable INFO level logging
+    # * <tt>:debug</tt>: Enable DEBUG level logging
+    # * <tt>:config</tt>: The config file
+    def initialize(options)
+      logger.level = Logger::INFO if options[:verbose]
+      logger.level = Logger::DEBUG if options[:debug]
+      config.load_file(options[:config]) if options[:config]
+    end
+    
+    # Execute the monitor. The monitor will start one heartbeat 
+    # monitor thread and one thread for each done queue and error
+    # queue as specified in the configuration.
+    def run
+      logger.info "Starting up monitor"
+      heartbeat_monitor_thread = run_heartbeat_monitor
+      done_monitor_threads = run_done_monitors
+      error_monitor_threads = run_error_monitors
+      
+      logger.info "Monitor running"
+      
+      Refinery::StatsServer.new.run
+      
+      begin
+        heartbeat_monitor_thread.join
+        done_monitor_threads.each { |t| t.join }
+        error_monitor_threads.each { |t| t.join }
+      rescue Interrupt => e
+      end
+      
+      logger.info "Monitor is exiting"
+    end
+    
+    private
+    def statistics
+      @statistics ||= Refinery::Statistics.new
+    end
+    
+    def run_heartbeat_monitor
+      logger.info "Starting heartbeat monitor"
+      Thread.new(queue('heartbeat')) do |heartbeat_queue|
+        loop do
+          begin
+            while (message = heartbeat_queue.receive)
+              logger.debug decode_message(message.body).inspect
+              message.delete()
+            end
+          rescue Exception => e
+            logger.error e
+          end
+          sleep(2)
+        end
+      end
+    end
+    
+    def run_done_monitors
+      config['processors'].collect do |key, settings|
+        queue_name = settings['queue'] || key
+        done_queue_name = "#{queue_name}_done"
+        logger.debug "Starting monitor for queue #{done_queue_name}"
+        Thread.new(queue(done_queue_name)) do |done_queue|
+          loop do
+            begin
+              while (message = done_queue.receive)
+                done_message = decode_message(message.body)
+                processed = decode_message(done_message['original'])
+                logger.info "Done: #{processed.inspect}"
+                message.delete()
+                statistics.record_done(done_message)
+              end
+            rescue Exception => e
+              logger.error e
+            end
+            sleep(2)
+          end
+        end
+      end
+    end
+    
+    def run_error_monitors
+      config['processors'].collect do |key, settings|
+        queue_name = settings['queue'] || key
+        error_queue_name = "#{queue_name}_error"
+        logger.info "Starting error monitor for queue #{error_queue_name}"
+        Thread.new(queue(error_queue_name)) do |error_queue|
+          loop do
+            begin
+              while (message = error_queue.receive)
+                error_message = decode_message(message.body)
+                processed = decode_message(error_message['original'])
+                logger.info "Error: #{processed.inspect}"
+                message.delete()
+                statistics.record_error(error_message)
+              end
+            rescue Exception => e
+              logger.error e
+            end
+            sleep(2)
+          end
+        end
+      end
+    end
+    
+  end
+end

data/lib/refinery/publisher.rb

@@ -0,0 +1,24 @@
+module Refinery #:nodoc:
+  # Base class for publishers to be implemented by subclasses.
+  class Publisher
+    include Refinery::Loggable
+    
+    # Initialize the publisher with the queue to publish messages to.
+    def initialize(waiting_queue)
+      @waiting_queue = waiting_queue
+    end
+    
+    protected
+    # Get the publish queue
+    def waiting_queue
+      @waiting_queue
+    end
+    
+    # Publish the message. The message will be converted to JSON and pushed
+    # into the queue associated with the publisher.
+    def publish(message)
+      logger.debug "Message: #{message.to_json}"
+      waiting_queue.send_message(Base64.encode64(message.to_json))
+    end
+  end
+end

data/lib/refinery/queueable.rb

@@ -0,0 +1,20 @@
+module Refinery #:nodoc:
+  # Mix this module in to classes that want to access a queue.
+  module Queueable
+    # Get a named queue
+    def queue(name)
+      queue_provider.queue(name)
+    end
+    
+    protected
+    # Get the queue provider. Defaults to RightAws::SqsGen2 running
+    # in multi-thread mode.
+    def queue_provider
+      @queue_provider ||= RightAws::SqsGen2.new(
+        config['aws']['credentials']["access_key_id"], 
+        config['aws']['credentials']["secret_access_key"],
+        {:multi_thread => true}
+      )
+    end
+  end
+end

data/lib/refinery/server.rb

@@ -0,0 +1,86 @@
+module Refinery #:nodoc:
+  # The server instance provides a runtime environment for daemons.
+  # To start the server create an Refinery::Server instance and invoke run.
+  class Server
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Queueable
+    include Refinery::Utilities
+    
+    # The directory where worker source files are stored. Defaults to
+    # ./workers
+    attr_accessor :workers_directory
+    
+    # Get a server-wide logger
+    def self.logger
+      @logger ||= begin
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::WARN
+        logger
+      end
+    end
+    
+    # Initialize the server.
+    #
+    # Options:
+    # * <tt>:config</tt>: Provide a file path to load that config
+    # * <tt>:debug</tt>: Set to true to enable debug logging
+    # * <tt>:verbose</tt>: Set to true to enable info logging
+    # * <tt>:workers</tt>: The workers directory
+    def initialize(options={})
+      logger.level = Logger::INFO if options[:verbose]
+      logger.level = Logger::DEBUG if options[:debug]
+      config.load_file(options[:config]) if options[:config]
+      self.workers_directory = options[:workers] if options[:workers]
+    end
+    
+    # The directory where workers are found. Defaults to ./workers
+    def workers_directory
+      @workers_directory ||= "./workers"
+    end
+    
+    # Stop the server
+    def stop
+      logger.info "Stopping Refinery Server"
+      daemons.each { |daemon| daemon.stop }
+    end
+    
+    # An array of all daemons
+    def daemons
+      @daemons ||= []
+    end
+    
+    # Run the server
+    def run
+      logger.info "Starting Refinery server"
+      execute_daemons
+      logger.info "Server is exiting"
+    end
+    
+    private
+    def execute_daemons
+      config['processors'].each do |key, settings|
+        logger.debug "Creating daemons for #{key}"
+        
+        queue_name = settings['queue'] || key
+        logger.debug "Using queue #{queue_name}"
+        waiting_queue = queue("#{queue_name}_waiting")
+        error_queue = queue("#{queue_name}_error")
+        done_queue = queue("#{queue_name}_done")
+        
+        1.upto(settings['workers']['initial']) do
+          daemons << Refinery::Daemon.new(self, key, waiting_queue, error_queue, done_queue)
+        end
+        
+        logger.debug "Running #{daemons.length} daemons"
+      end
+      
+      Heartbeat.new(self)
+      
+      begin
+        daemons.each { |daemon| daemon.thread.join }
+      rescue Interrupt => e
+      end
+    end
+  end
+end

data/lib/refinery/statistics.rb

@@ -0,0 +1,61 @@
+module Refinery #:nodoc:
+  # The statistics class provides a means to record runtime stats
+  # about completed jobs and errors. The stats are stored in a SQL
+  # database (using SQLite3 by default).
+  class Statistics
+    include Refinery::Loggable
+    
+    # Record the done record into the 
+    def record_done(message)
+      db[:completed_jobs] << {
+        :host => message['host_info']['hostname'],
+        :pid => message['host_info']['pid'],
+        :run_time => message['run_time'],
+        :original_message => message['original'],
+        :when => Time.now
+      }
+    end
+    
+    # Record the error message into the statistics database.
+    def record_error(message)
+      db[:errors] << {
+        :host => message['host_info']['hostname'],
+        :pid => message['host_info']['pid'],
+        :error_class => message['error']['class'],
+        :error_message => message['error']['message'],
+        :original_message => message['original'],
+        :when => Time.now
+      }
+    end
+    
+    private
+    # Get a Sequel connection to the stats database
+    def db
+      @db ||= begin
+        db = Sequel.connect('sqlite://stats.db')
+        unless db.table_exists?(:completed_jobs)
+          db.create_table :completed_jobs do
+            primary_key :id
+            column :host, :text
+            column :pid, :integer
+            column :run_time, :float
+            column :original_message, :text
+            column :when, :time
+          end
+        end
+        unless db.table_exists?(:errors)
+          db.create_table :errors do
+            primary_key :id
+            column :host, :text
+            column :pid, :integer
+            column :error_class, :text
+            column :error_message, :text
+            column :original_message, :text
+            column :when, :time
+          end
+        end
+        db
+      end
+    end
+  end
+end

data/lib/refinery/stats_server.rb

@@ -0,0 +1,134 @@
+module Refinery #:nodoc:
+  # The StatsServer class provides a build in web server that provides
+  # a view into the refinery statistics.
+  class StatsServer
+    include Refinery::Loggable
+    
+    # Run the stats server.
+    def run
+      begin
+        Ramaze::Log.loggers.clear # supress all Ramaze logging
+        Ramaze.start              # start the Ramaze server on port 7000
+      rescue NameError
+        self.logger.warn "Install Remaze to enable the stats server"
+      end
+    end
+    
+    if const_defined?(:Ramaze)
+      class MainController < ::Ramaze::Controller #:nodoc:
+        map '/'
+    
+        def index
+          %(
+            <html>
+            <head>
+            <title>Refinery Stats</title>
+            <style>
+            .widget { border: 1px solid #777; margin-bottom: 10px; padding: 4px; }
+            .widget h2 { font-size: 14pt; margin-top: 2px; margin-bottom: 2px; }
+            #left-column { float: left; width: 600px; }
+            #right-column { margin-left: 610px; width: 300px; }
+            table { background-color: #ddd; width: 100%; }
+            table td { background-color: #eee; }
+            table th { background-color: #ccc; }
+            </style>
+            </head>
+            <body>
+            <h1>Refinery Stats</h1>
+            <div id="left-column">
+              <div class="run_time widget">
+                <h2>Runtime Averages</h2>
+                #{avg_run_time}
+              </div>
+              <div class="errors widget">
+                <h2>Last 5 Errors</h2>
+                #{errors_table}
+              </div>
+              <div class="completed widget">
+                <h2>Last 5 Completed Jobs</h2>
+                #{completed_jobs_table}
+              </div>
+            </div>
+            <div id="right-column">
+              <div class="overview widget">
+                <h2>Overview</h2>
+                <div>#{db[:completed_jobs].count} jobs completed</div>
+                <div>#{db[:errors].count} errors</div>
+              </div>
+          
+            </div>
+            </body>
+            </html>
+          )
+        end
+    
+        private
+        def db
+          Sequel.connect("sqlite://stats.db")
+        end
+    
+        def avg_run_time
+          rows = db[:completed_jobs].group(:host, :pid).select(:host, :pid, :AVG.sql_function(:run_time)).map do |record| 
+            %(<tr>
+            <td>#{record[:host]}</td>
+            <td>#{record[:pid]}</td>
+            <td>#{sprintf("%.6f", record[:"AVG(`run_time`)"])}</td>
+            </tr>)
+          end.join
+          %(
+            <table>
+            <tr>
+            <th>Host</th>
+            <th>PID</th>
+            <th>Avg Run Time</th>
+            </tr>
+            #{rows}
+            </table>
+          )
+        end
+    
+        def completed_jobs_table
+          jobs_list = db[:completed_jobs].limit(5).map do |record|
+            %Q( <tr>
+                <td>#{record[:host]}</td>
+                <td>#{record[:pid]}</td>
+                <td>#{record[:run_time]}</td>
+                </tr>
+            )
+          end
+          %Q( <table>
+                <tr>
+                  <th>Host</th>
+                  <th>PID</th>
+                  <th>Run Time</th>
+                </tr>
+                #{jobs_list.join}
+              </table>
+          )
+        end
+    
+        def errors_table
+          errors = db[:errors].limit(5).map do |record|
+            %(<tr>
+                <td>#{record[:host]}</td>
+                <td>#{record[:pid]}</td>
+                <td>#{record[:error_class]}</td>
+                <td>#{record[:error_message]}</td>
+              </tr>
+            )
+          end
+          %(<table>
+            <tr>
+              <th>Host</th>
+              <th>PID</th>
+              <th>Error Class</th>
+              <th>Error Message</th>
+            </tr>
+            #{errors.join}
+            </table>
+          )
+        end
+      end
+    end
+  end
+end

data/lib/refinery/utilities.rb

@@ -0,0 +1,33 @@
+module Refinery #:nodoc:
+  # Utilities that can be mixed into a class
+  module Utilities
+    # Camelize the given word.
+    def camelize(word, first_letter_in_uppercase = true)
+      if first_letter_in_uppercase
+        word.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+      else
+        word.first.downcase + camelize(word)[1..-1]
+      end
+    end
+    
+    # Decode the message_body from Base 64 and then parse from JSON.
+    def decode_message(message_body)
+      JSON.parse(Base64.decode64(message_body))
+    end
+    
+    # Convert the given message_data object to JSON and then Base 64
+    # encode it
+    def encode_message(message_data)
+      Base64.encode64(message_data.to_json)
+    end
+    
+    # Get a Hash of useful host information that can be sent with
+    # messages to the monitoring system.
+    def host_info
+      {
+        'hostname' => Socket.gethostname,
+        'pid' => $$
+      }
+    end
+  end
+end

data/lib/refinery/validations.rb

@@ -0,0 +1,48 @@
+module Refinery #:nodoc:
+  # Error that is raised when a message is invalid.
+  class InvalidMessageError < RuntimeError
+  end
+  
+  # Module containing all validations.
+  module Validations
+    def self.included(base) # :nodoc:
+      base.extend(ClassMethods)
+    end
+    
+    # Class methods that are added to the worker.
+    module ClassMethods
+      # A list of all of the validators. Validators are lambdas
+      # that will be called with the message as its only arg.
+      # Note that the order of validators is retained.
+      def validators
+        @validators ||= []
+      end
+      
+      # Validate with the given block. The block must receive a single
+      # argument that is the message
+      def validate_with(&block)
+        validators << block
+      end
+      alias :validate :validate_with
+      
+      # Validate that each of the keys exists in the message.
+      def validate_key_exists(*args)
+        args.each do |key|
+          validators << lambda do |message|
+            raise Refinery::InvalidMessageError, "Key does not exist in message: #{key}" unless message[key]
+          end
+        end
+      end
+      alias :validates_key_exists :validate_key_exists
+      alias :validates_presence_of :validate_key_exists
+    end
+    
+    # Validate the given message
+    protected
+    def validate(message)
+      self.class.validators.each do |validator|
+        validator.call(message)
+      end
+    end
+  end
+end