aeden-refinery 0.9.0 → 0.9.1

data/lib/refinery/heartbeat.rb

@@ -0,0 +1,30 @@
+module Refinery #:nodoc:
+  # A heartbeat publisher that indicates a server is alive.
+  class Heartbeat
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Queueable
+    include Refinery::Utilities
+    
+    # Initialize the heartbeat for the given server.
+    def initialize(server)
+      @server = server
+      @thread = Thread.new(queue('heartbeat')) do |heartbeat_queue|
+        loop do
+          begin
+            logger.debug "Send heartbeat"
+            message = {
+              'host_info' => host_info,
+              'timestamp' => Time.now.utc,
+              'running_daemons' => @server.daemons.length
+            }
+            heartbeat_queue.send_message(Base64.encode64(message.to_json))
+            sleep(60)
+          rescue Exception => e
+            logger.error "Error sending heartbeat: #{e.message}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/refinery/loggable.rb

@@ -0,0 +1,9 @@
+module Refinery
+  # Include this module to get access to the server logger
+  module Loggable
+    # Get the logger.
+    def logger
+      @logger ||= Refinery::Server.logger
+    end
+  end
+end

data/lib/refinery/monitor.rb

@@ -0,0 +1,116 @@
+module Refinery #:nodoc:
+  # The monitor is responsible for monitoring the health of the various
+  # components of refinery.
+  class Monitor
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Queueable
+    include Refinery::Utilities
+    
+    # Initialize the monitor.
+    #
+    # Options:
+    # * <tt>:verbose</tt>: Enable INFO level logging
+    # * <tt>:debug</tt>: Enable DEBUG level logging
+    # * <tt>:config</tt>: The config file
+    def initialize(options)
+      logger.level = Logger::INFO if options[:verbose]
+      logger.level = Logger::DEBUG if options[:debug]
+      config.load_file(options[:config]) if options[:config]
+    end
+    
+    # Execute the monitor. The monitor will start one heartbeat 
+    # monitor thread and one thread for each done queue and error
+    # queue as specified in the configuration.
+    def run
+      logger.info "Starting up monitor"
+      heartbeat_monitor_thread = run_heartbeat_monitor
+      done_monitor_threads = run_done_monitors
+      error_monitor_threads = run_error_monitors
+      
+      logger.info "Monitor running"
+      
+      Refinery::StatsServer.new.run
+      
+      begin
+        heartbeat_monitor_thread.join
+        done_monitor_threads.each { |t| t.join }
+        error_monitor_threads.each { |t| t.join }
+      rescue Interrupt => e
+      end
+      
+      logger.info "Monitor is exiting"
+    end
+    
+    private
+    def statistics
+      @statistics ||= Refinery::Statistics.new
+    end
+    
+    def run_heartbeat_monitor
+      logger.info "Starting heartbeat monitor"
+      Thread.new(queue('heartbeat')) do |heartbeat_queue|
+        loop do
+          begin
+            while (message = heartbeat_queue.receive)
+              logger.debug decode_message(message.body).inspect
+              message.delete()
+            end
+          rescue Exception => e
+            logger.error e
+          end
+          sleep(2)
+        end
+      end
+    end
+    
+    def run_done_monitors
+      config['processors'].collect do |key, settings|
+        queue_name = settings['queue'] || key
+        done_queue_name = "#{queue_name}_done"
+        logger.debug "Starting monitor for queue #{done_queue_name}"
+        Thread.new(queue(done_queue_name)) do |done_queue|
+          loop do
+            begin
+              while (message = done_queue.receive)
+                done_message = decode_message(message.body)
+                processed = decode_message(done_message['original'])
+                logger.info "Done: #{processed.inspect}"
+                message.delete()
+                statistics.record_done(done_message)
+              end
+            rescue Exception => e
+              logger.error e
+            end
+            sleep(2)
+          end
+        end
+      end
+    end
+    
+    def run_error_monitors
+      config['processors'].collect do |key, settings|
+        queue_name = settings['queue'] || key
+        error_queue_name = "#{queue_name}_error"
+        logger.info "Starting error monitor for queue #{error_queue_name}"
+        Thread.new(queue(error_queue_name)) do |error_queue|
+          loop do
+            begin
+              while (message = error_queue.receive)
+                error_message = decode_message(message.body)
+                processed = decode_message(error_message['original'])
+                logger.info "Error: #{processed.inspect}"
+                message.delete()
+                statistics.record_error(error_message)
+              end
+            rescue Exception => e
+              logger.error e
+            end
+            sleep(2)
+          end
+        end
+      end
+    end
+    
+  end
+end

data/lib/refinery/publisher.rb

@@ -0,0 +1,24 @@
+module Refinery #:nodoc:
+  # Base class for publishers to be implemented by subclasses.
+  class Publisher
+    include Refinery::Loggable
+    
+    # Initialize the publisher with the queue to publish messages to.
+    def initialize(waiting_queue)
+      @waiting_queue = waiting_queue
+    end
+    
+    protected
+    # Get the publish queue
+    def waiting_queue
+      @waiting_queue
+    end
+    
+    # Publish the message. The message will be converted to JSON and pushed
+    # into the queue associated with the publisher.
+    def publish(message)
+      logger.debug "Message: #{message.to_json}"
+      waiting_queue.send_message(Base64.encode64(message.to_json))
+    end
+  end
+end

data/lib/refinery/queueable.rb

@@ -0,0 +1,20 @@
+module Refinery #:nodoc:
+  # Mix this module in to classes that want to access a queue.
+  module Queueable
+    # Get a named queue
+    def queue(name)
+      queue_provider.queue(name)
+    end
+    
+    protected
+    # Get the queue provider. Defaults to RightAws::SqsGen2 running
+    # in multi-thread mode.
+    def queue_provider
+      @queue_provider ||= RightAws::SqsGen2.new(
+        config['aws']['credentials']["access_key_id"], 
+        config['aws']['credentials']["secret_access_key"],
+        {:multi_thread => true}
+      )
+    end
+  end
+end

data/lib/refinery/server.rb

@@ -0,0 +1,86 @@
+module Refinery #:nodoc:
+  # The server instance provides a runtime environment for daemons.
+  # To start the server create an Refinery::Server instance and invoke run.
+  class Server
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Queueable
+    include Refinery::Utilities
+    
+    # The directory where worker source files are stored. Defaults to
+    # ./workers
+    attr_accessor :workers_directory
+    
+    # Get a server-wide logger
+    def self.logger
+      @logger ||= begin
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::WARN
+        logger
+      end
+    end
+    
+    # Initialize the server.
+    #
+    # Options:
+    # * <tt>:config</tt>: Provide a file path to load that config
+    # * <tt>:debug</tt>: Set to true to enable debug logging
+    # * <tt>:verbose</tt>: Set to true to enable info logging
+    # * <tt>:workers</tt>: The workers directory
+    def initialize(options={})
+      logger.level = Logger::INFO if options[:verbose]
+      logger.level = Logger::DEBUG if options[:debug]
+      config.load_file(options[:config]) if options[:config]
+      self.workers_directory = options[:workers] if options[:workers]
+    end
+    
+    # The directory where workers are found. Defaults to ./workers
+    def workers_directory
+      @workers_directory ||= "./workers"
+    end
+    
+    # Stop the server
+    def stop
+      logger.info "Stopping Refinery Server"
+      daemons.each { |daemon| daemon.stop }
+    end
+    
+    # An array of all daemons
+    def daemons
+      @daemons ||= []
+    end
+    
+    # Run the server
+    def run
+      logger.info "Starting Refinery server"
+      execute_daemons
+      logger.info "Server is exiting"
+    end
+    
+    private
+    def execute_daemons
+      config['processors'].each do |key, settings|
+        logger.debug "Creating daemons for #{key}"
+        
+        queue_name = settings['queue'] || key
+        logger.debug "Using queue #{queue_name}"
+        waiting_queue = queue("#{queue_name}_waiting")
+        error_queue = queue("#{queue_name}_error")
+        done_queue = queue("#{queue_name}_done")
+        
+        1.upto(settings['workers']['initial']) do
+          daemons << Refinery::Daemon.new(self, key, waiting_queue, error_queue, done_queue)
+        end
+        
+        logger.debug "Running #{daemons.length} daemons"
+      end
+      
+      Heartbeat.new(self)
+      
+      begin
+        daemons.each { |daemon| daemon.thread.join }
+      rescue Interrupt => e
+      end
+    end
+  end
+end

data/lib/refinery/statistics.rb

@@ -0,0 +1,61 @@
+module Refinery #:nodoc:
+  # The statistics class provides a means to record runtime stats
+  # about completed jobs and errors. The stats are stored in a SQL
+  # database (using SQLite3 by default).
+  class Statistics
+    include Refinery::Loggable
+    
+    # Record the done record into the 
+    def record_done(message)
+      db[:completed_jobs] << {
+        :host => message['host_info']['hostname'],
+        :pid => message['host_info']['pid'],
+        :run_time => message['run_time'],
+        :original_message => message['original'],
+        :when => Time.now
+      }
+    end
+    
+    # Record the error message into the statistics database.
+    def record_error(message)
+      db[:errors] << {
+        :host => message['host_info']['hostname'],
+        :pid => message['host_info']['pid'],
+        :error_class => message['error']['class'],
+        :error_message => message['error']['message'],
+        :original_message => message['original'],
+        :when => Time.now
+      }
+    end
+    
+    private
+    # Get a Sequel connection to the stats database
+    def db
+      @db ||= begin
+        db = Sequel.connect('sqlite://stats.db')
+        unless db.table_exists?(:completed_jobs)
+          db.create_table :completed_jobs do
+            primary_key :id
+            column :host, :text
+            column :pid, :integer
+            column :run_time, :float
+            column :original_message, :text
+            column :when, :time
+          end
+        end
+        unless db.table_exists?(:errors)
+          db.create_table :errors do
+            primary_key :id
+            column :host, :text
+            column :pid, :integer
+            column :error_class, :text
+            column :error_message, :text
+            column :original_message, :text
+            column :when, :time
+          end
+        end
+        db
+      end
+    end
+  end
+end

data/lib/refinery/stats_server.rb

@@ -0,0 +1,134 @@
+module Refinery #:nodoc:
+  # The StatsServer class provides a build in web server that provides
+  # a view into the refinery statistics.
+  class StatsServer
+    include Refinery::Loggable
+    
+    # Run the stats server.
+    def run
+      begin
+        Ramaze::Log.loggers.clear # supress all Ramaze logging
+        Ramaze.start              # start the Ramaze server on port 7000
+      rescue NameError
+        self.logger.warn "Install Remaze to enable the stats server"
+      end
+    end
+    
+    if const_defined?(:Ramaze)
+      class MainController < ::Ramaze::Controller #:nodoc:
+        map '/'
+    
+        def index
+          %(
+            <html>
+            <head>
+            <title>Refinery Stats</title>
+            <style>
+            .widget { border: 1px solid #777; margin-bottom: 10px; padding: 4px; }
+            .widget h2 { font-size: 14pt; margin-top: 2px; margin-bottom: 2px; }
+            #left-column { float: left; width: 600px; }
+            #right-column { margin-left: 610px; width: 300px; }
+            table { background-color: #ddd; width: 100%; }
+            table td { background-color: #eee; }
+            table th { background-color: #ccc; }
+            </style>
+            </head>
+            <body>
+            <h1>Refinery Stats</h1>
+            <div id="left-column">
+              <div class="run_time widget">
+                <h2>Runtime Averages</h2>
+                #{avg_run_time}
+              </div>
+              <div class="errors widget">
+                <h2>Last 5 Errors</h2>
+                #{errors_table}
+              </div>
+              <div class="completed widget">
+                <h2>Last 5 Completed Jobs</h2>
+                #{completed_jobs_table}
+              </div>
+            </div>
+            <div id="right-column">
+              <div class="overview widget">
+                <h2>Overview</h2>
+                <div>#{db[:completed_jobs].count} jobs completed</div>
+                <div>#{db[:errors].count} errors</div>
+              </div>
+          
+            </div>
+            </body>
+            </html>
+          )
+        end
+    
+        private
+        def db
+          Sequel.connect("sqlite://stats.db")
+        end
+    
+        def avg_run_time
+          rows = db[:completed_jobs].group(:host, :pid).select(:host, :pid, :AVG.sql_function(:run_time)).map do |record| 
+            %(<tr>
+            <td>#{record[:host]}</td>
+            <td>#{record[:pid]}</td>
+            <td>#{sprintf("%.6f", record[:"AVG(`run_time`)"])}</td>
+            </tr>)
+          end.join
+          %(
+            <table>
+            <tr>
+            <th>Host</th>
+            <th>PID</th>
+            <th>Avg Run Time</th>
+            </tr>
+            #{rows}
+            </table>
+          )
+        end
+    
+        def completed_jobs_table
+          jobs_list = db[:completed_jobs].limit(5).map do |record|
+            %Q( <tr>
+                <td>#{record[:host]}</td>
+                <td>#{record[:pid]}</td>
+                <td>#{record[:run_time]}</td>
+                </tr>
+            )
+          end
+          %Q( <table>
+                <tr>
+                  <th>Host</th>
+                  <th>PID</th>
+                  <th>Run Time</th>
+                </tr>
+                #{jobs_list.join}
+              </table>
+          )
+        end
+    
+        def errors_table
+          errors = db[:errors].limit(5).map do |record|
+            %(<tr>
+                <td>#{record[:host]}</td>
+                <td>#{record[:pid]}</td>
+                <td>#{record[:error_class]}</td>
+                <td>#{record[:error_message]}</td>
+              </tr>
+            )
+          end
+          %(<table>
+            <tr>
+              <th>Host</th>
+              <th>PID</th>
+              <th>Error Class</th>
+              <th>Error Message</th>
+            </tr>
+            #{errors.join}
+            </table>
+          )
+        end
+      end
+    end
+  end
+end