refinery 0.12.2 → 1.0

data/lib/refinery/processor.rb

@@ -1,55 +0,0 @@
-module Refinery #:nodoc:
-  # This class is used to monitor all of the threads for a single
-  # processor.
-  class Processor < Thread
-    include Refinery::Configurable
-    include Refinery::Loggable
-    
-    attr_reader :server
-    attr_reader :key
-    attr_reader :settings
-    attr_reader :daemons
-    
-    # Initialize the processor.
-    def initialize(server, key, settings={})
-      @server = server
-      @key = key
-      @settings = settings
-      @daemons = []
-      super do
-        execute
-      end
-    end
-    
-    private
-    # Execute the processor
-    def execute
-      queue_prefix = config['prefix'] || ''
-      
-      if defined?(java.lang.Thread)
-        java.lang.Thread.current_thread.name = "#{key} Processor"
-      end
-      
-      logger.debug "Creating daemons for #{key}"
-      1.upto(settings['workers']['initial']) do
-        daemons << Daemon.new(self, key, queue_prefix, settings)
-      end
-      
-      logger.debug "Running #{daemons.length} daemons"
-      
-      wait = ThreadsWait.new(*daemons)
-      wait.all_waits do |daemon|
-        logger.debug "a #{daemon.name} just died"
-        daemons.delete(daemon)
-        logger.debug "waiting for 60 seconds before starting a new #{key} daemon"
-        sleep(60)
-        logger.debug "starting a new #{key} daemon"
-        daemon = Daemon.new(self, key, queue_prefix, settings)
-        daemons << daemon
-        wait.join(daemon)
-      end
-      
-      logger.debug "Processor #{key} is exiting"
-    end
-  end
-end

data/lib/refinery/publisher.rb

@@ -1,42 +0,0 @@
-module Refinery #:nodoc:
-  # Base class for publishers to be implemented by subclasses.
-  class Publisher
-    include Refinery::Loggable
-    include Refinery::Queueable
-    
-    # Initialize the publisher with the queue to publish messages to.
-    def initialize(waiting_queue_name)
-      @waiting_queue_name = waiting_queue_name
-    end
-    
-    protected
-    # Get the publish queue name
-    def waiting_queue_name
-      @waiting_queue_name
-    end
-    
-    # Publish the message. The message will be converted to JSON and pushed
-    # into the queue associated with the publisher.
-    def publish(message)
-      with_queue(waiting_queue_name) do |waiting_queue|
-        publish_to_queue(waiting_queue, message)
-      end
-    end
-    
-    # Publish the given message if the waiting queue is empty. The message will
-    # be converted to JSON and pushed into the queue associated with the
-    # publisher.
-    def publish_if_empty(message)
-      with_queue(waiting_queue_name) do |waiting_queue|
-        publish_to_queue(waiting_queue, message) if waiting_queue.size == 0
-      end
-    end
-    
-    # Publish the given message to the queue. The message will be converted 
-    # to JSON and pushed into the queue associated with the publisher.
-    def publish_to_queue(queue, message)
-      logger.debug "Publisher #{self.class.name} sending message: #{message.to_json}"
-      queue.send_message(Base64.encode64(message.to_json))
-    end
-  end
-end

data/lib/refinery/queueable.rb

@@ -1,48 +0,0 @@
-module Refinery #:nodoc:
-  # Mix this module in to classes that want to access a queue.
-  module Queueable
-    include Loggable
-    include Configurable
-    # Get a named queue
-    def queue(name)
-      queue_provider.queue(name)
-    end
-    
-    # Given the queue name and a block, yield the named queue into 
-    # the block.
-    def with_queue(name, &block)
-      begin
-        yield queue(name)
-      rescue Exception => e
-        logger.error "An error occurred when communicating with queue #{name}: #{e.message}"
-        sleep(30)
-      end
-    end
-    
-    protected
-    # Get the queue provider. Defaults to RightAws::SqsGen2 running
-    # in multi-thread mode.
-    def queue_provider
-      if queue_engine = config['queue_engine']
-        if queue_engine['provider'] == 'beanstalk' && defined?(Beanstalk)
-          @queue_provider ||= Refinery::BeanstalkQueueProvider.new
-        else
-          raise RuntimeError, "Unknown queue provider: #{queue_engine['provider']}"
-        end
-      else
-        if defined?(Typica)
-          @queue_provider ||= Typica::Sqs::QueueService.new(
-            config['aws']['credentials']["access_key_id"], 
-            config['aws']['credentials']["secret_access_key"]
-          )
-        else
-          @queue_provider ||= RightAws::SqsGen2.new(
-            config['aws']['credentials']["access_key_id"], 
-            config['aws']['credentials']["secret_access_key"],
-            {:multi_thread => true}
-          )
-        end
-      end
-    end
-  end
-end

data/lib/refinery/server.rb

@@ -1,88 +0,0 @@
-module Refinery #:nodoc:
-  # The server instance provides a runtime environment for daemons.
-  # To start the server create an Refinery::Server instance and invoke run.
-  class Server
-    include Refinery::Loggable
-    include Refinery::Configurable
-    include Refinery::Queueable
-    include Refinery::Utilities
-    
-    # The directory where worker source files are stored. Defaults to
-    # ./workers
-    attr_accessor :workers_directory
-    
-    # Get a server-wide logger
-    def self.logger
-      @logger ||= begin
-        logger = Logger.new(STDOUT)
-        logger.level = Logger::WARN
-        logger.formatter = CustomFormatter.new
-        logger
-      end
-    end
-    
-    # Initialize the server.
-    #
-    # Options:
-    # * <tt>:config</tt>: Provide a file path to load that config
-    # * <tt>:debug</tt>: Set to true to enable debug logging
-    # * <tt>:verbose</tt>: Set to true to enable info logging
-    # * <tt>:workers</tt>: The workers directory
-    def initialize(options={})
-      logger.level = Logger::INFO if options[:verbose]
-      logger.level = Logger::DEBUG if options[:debug]
-      config.load_file(options[:config]) if options[:config]
-      self.workers_directory = options[:workers] if options[:workers]
-    end
-    
-    # The directory where workers are found. Defaults to ./workers
-    def workers_directory
-      @workers_directory ||= "./workers"
-    end
-    
-    # Stop the server
-    def stop
-      logger.info "Stopping Refinery Server"
-      daemons.each { |daemon| daemon.stop }
-    end
-    
-    # An array of all daemons
-    def daemons
-      @daemons ||= []
-    end
-    
-    # Run the server
-    def run
-      logger.info "Starting Refinery server"
-      execute_processors
-      logger.info "Server is exiting"
-    end
-    
-    private
-    def execute_processors
-      
-      @processors = config['processors'].map do |key, settings|
-        Processor.new(self, key, settings)
-      end
-      
-      Heartbeat.new(self)
-      
-      begin
-        @processors.each { |p| p.join }
-      rescue Interrupt => e
-      end
-    end
-  end
-  
-  class CustomFormatter
-    def format
-      @format ||= "%s, [%s#%d][%s] %5s -- %s: %s\n"
-    end
-    def call(severity, time, progname, msg)
-      format % [severity[0..0], format_datetime(time.utc), $$, Thread.current.object_id.to_s, severity, progname, msg.to_s]
-    end
-    def format_datetime(time)
-      time.strftime("%Y-%m-%dT%H:%M:%S.") << "%06d " % time.usec
-    end
-  end
-end

data/lib/refinery/statistics.rb

@@ -1,61 +0,0 @@
-module Refinery #:nodoc:
-  # The statistics class provides a means to record runtime stats
-  # about completed jobs and errors. The stats are stored in a SQL
-  # database (using SQLite3 by default).
-  class Statistics
-    include Refinery::Loggable
-    
-    # Record the done record into the 
-    def record_done(message)
-      db[:completed_jobs] << {
-        :host => message['host_info']['hostname'],
-        :pid => message['host_info']['pid'],
-        :run_time => message['run_time'],
-        :original_message => message['original'],
-        :when => Time.now
-      }
-    end
-    
-    # Record the error message into the statistics database.
-    def record_error(message)
-      db[:errors] << {
-        :host => message['host_info']['hostname'],
-        :pid => message['host_info']['pid'],
-        :error_class => message['error']['class'],
-        :error_message => message['error']['message'],
-        :original_message => message['original'],
-        :when => Time.now
-      }
-    end
-    
-    private
-    # Get a Sequel connection to the stats database
-    def db
-      @db ||= begin
-        db = Sequel.connect('sqlite://stats.db')
-        unless db.table_exists?(:completed_jobs)
-          db.create_table :completed_jobs do
-            primary_key :id
-            column :host, :text
-            column :pid, :integer
-            column :run_time, :float
-            column :original_message, :text
-            column :when, :time
-          end
-        end
-        unless db.table_exists?(:errors)
-          db.create_table :errors do
-            primary_key :id
-            column :host, :text
-            column :pid, :integer
-            column :error_class, :text
-            column :error_message, :text
-            column :original_message, :text
-            column :when, :time
-          end
-        end
-        db
-      end
-    end
-  end
-end

data/lib/refinery/stats_server.rb

@@ -1,135 +0,0 @@
-module Refinery #:nodoc:
-  # The StatsServer class provides a build in web server that provides
-  # a view into the refinery statistics. This functionality is very
-  # experimental.
-  class StatsServer
-    include Refinery::Loggable
-    
-    # Run the stats server.
-    def run
-      begin
-        Ramaze::Log.loggers.clear # supress all Ramaze logging
-        Ramaze.start              # start the Ramaze server on port 7000
-      rescue NameError
-        self.logger.warn "Install Ramaze to enable the stats server"
-      end
-    end
-    
-    if const_defined?(:Ramaze)
-      class MainController < ::Ramaze::Controller #:nodoc:
-        map '/'
-    
-        def index
-          %(
-            <html>
-            <head>
-            <title>Refinery Stats</title>
-            <style>
-            .widget { border: 1px solid #777; margin-bottom: 10px; padding: 4px; }
-            .widget h2 { font-size: 14pt; margin-top: 2px; margin-bottom: 2px; }
-            #left-column { float: left; width: 600px; }
-            #right-column { margin-left: 610px; width: 300px; }
-            table { background-color: #ddd; width: 100%; }
-            table td { background-color: #eee; }
-            table th { background-color: #ccc; }
-            </style>
-            </head>
-            <body>
-            <h1>Refinery Stats</h1>
-            <div id="left-column">
-              <div class="run_time widget">
-                <h2>Runtime Averages</h2>
-                #{avg_run_time}
-              </div>
-              <div class="errors widget">
-                <h2>Last 5 Errors</h2>
-                #{errors_table}
-              </div>
-              <div class="completed widget">
-                <h2>Last 5 Completed Jobs</h2>
-                #{completed_jobs_table}
-              </div>
-            </div>
-            <div id="right-column">
-              <div class="overview widget">
-                <h2>Overview</h2>
-                <div>#{db[:completed_jobs].count} jobs completed</div>
-                <div>#{db[:errors].count} errors</div>
-              </div>
-          
-            </div>
-            </body>
-            </html>
-          )
-        end
-    
-        private
-        def db
-          Sequel.connect("sqlite://stats.db")
-        end
-    
-        def avg_run_time
-          rows = db[:completed_jobs].group(:host, :pid).select(:host, :pid, :AVG.sql_function(:run_time)).map do |record| 
-            %(<tr>
-            <td>#{record[:host]}</td>
-            <td>#{record[:pid]}</td>
-            <td>#{sprintf("%.6f", record[:"AVG(`run_time`)"])}</td>
-            </tr>)
-          end.join
-          %(
-            <table>
-            <tr>
-            <th>Host</th>
-            <th>PID</th>
-            <th>Avg Run Time</th>
-            </tr>
-            #{rows}
-            </table>
-          )
-        end
-    
-        def completed_jobs_table
-          jobs_list = db[:completed_jobs].limit(5).map do |record|
-            %Q( <tr>
-                <td>#{record[:host]}</td>
-                <td>#{record[:pid]}</td>
-                <td>#{record[:run_time]}</td>
-                </tr>
-            )
-          end
-          %Q( <table>
-                <tr>
-                  <th>Host</th>
-                  <th>PID</th>
-                  <th>Run Time</th>
-                </tr>
-                #{jobs_list.join}
-              </table>
-          )
-        end
-    
-        def errors_table
-          errors = db[:errors].limit(5).map do |record|
-            %(<tr>
-                <td>#{record[:host]}</td>
-                <td>#{record[:pid]}</td>
-                <td>#{record[:error_class]}</td>
-                <td>#{record[:error_message]}</td>
-              </tr>
-            )
-          end
-          %(<table>
-            <tr>
-              <th>Host</th>
-              <th>PID</th>
-              <th>Error Class</th>
-              <th>Error Message</th>
-            </tr>
-            #{errors.join}
-            </table>
-          )
-        end
-      end
-    end
-  end
-end

data/lib/refinery/utilities.rb

@@ -1,33 +0,0 @@
-module Refinery #:nodoc:
-  # Utilities that can be mixed into a class
-  module Utilities
-    # Camelize the given word.
-    def camelize(word, first_letter_in_uppercase = true)
-      if first_letter_in_uppercase
-        word.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
-      else
-        word.first.downcase + camelize(word)[1..-1]
-      end
-    end
-    
-    # Decode the message_body from Base 64 and then parse from JSON.
-    def decode_message(message_body)
-      JSON.parse(Base64.decode64(message_body))
-    end
-    
-    # Convert the given message_data object to JSON and then Base 64
-    # encode it
-    def encode_message(message_data)
-      Base64.encode64(message_data.to_json)
-    end
-    
-    # Get a Hash of useful host information that can be sent with
-    # messages to the monitoring system.
-    def host_info
-      {
-        'hostname' => Socket.gethostname,
-        'pid' => $$
-      }
-    end
-  end
-end