refinery 0.12.2 → 1.0

data/lib/refinery/beanstalk_queue_provider.rb

@@ -1,18 +0,0 @@
-module Refinery #:nodoc:
-  # A queue provider for beanstalk
-  class BeanstalkQueueProvider
-    include Refinery::Loggable
-    
-    attr_reader :queues
-    
-    # Initialize the queue provider
-    def initialize(hosts=nil)
-      @hosts = hosts
-      @queues = {}
-    end
-    # Get the named queue
-    def queue(name)
-      queues[name] ||= Refinery::BeanstalkQueue.new(name, @hosts)
-    end
-  end
-end

data/lib/refinery/config.rb

@@ -1,48 +0,0 @@
-module Refinery #:nodoc:
-  # Configuration class.
-  class Config
-    # Get a shared configuration
-    def self.default
-      @default ||= new({
-        'aws' => {
-          'credentials' => {}
-        },
-        'processors' => {}
-      })
-    end
-    
-    # Initialize the config with the given data
-    def initialize(data={})
-      @data = data
-    end
-    
-    # Get the configuration value
-    def [](key)
-      data[key.to_s]
-    end
-    
-    # Set the configuration value
-    def []=(key, value)
-      data[key.to_s] = value
-    end
-    
-    # Load configuration from a YAML file
-    def load_file(file)
-      @file = file
-      @data = YAML::load_file(@file)
-      @last_load = File.mtime(@file)
-    end
-    
-    # Refresh the configuration from the YAML file if necessary.
-    def refresh
-      if File.mtime(@file) != @last_load
-        @data = YAML::load_file(@file)
-      end
-    end
-    
-    private
-    def data
-      @data ||= {}
-    end
-  end
-end

data/lib/refinery/configurable.rb

@@ -1,15 +0,0 @@
-module Refinery #:nodoc:
-  # Include this module to get access to a shared configuration
-  module Configurable
-    # Get the configuration. If the config is nil then this will use
-    # the default shared configuration.
-    def config
-      @config ||= Refinery::Config.default
-    end
-    
-    # Set the configuration.
-    def config=(config)
-      @config = config
-    end
-  end
-end

data/lib/refinery/daemon.rb

@@ -1,148 +0,0 @@
-module Refinery #:nodoc:
-  # A daemon provides a thread to run workers in.
-  class Daemon < Thread
-    include Refinery::Loggable
-    include Refinery::Configurable
-    include Refinery::Utilities
-    include Refinery::Queueable
-    
-    RUNNING = 'running'
-    STOPPED = 'stopped'
-    
-    # The name of the daemon
-    attr_reader :name
-    # The settings for the daemon
-    attr_reader :settings
-    # The base queue name
-    attr_reader :queue_name
-    
-    # Stop the daemon
-    def stop
-      self.state = STOPPED
-    end
-    
-    # Return the daemon state
-    def state
-      @state ||= RUNNING
-    end
-    
-    # Set the daemon state.
-    def state=(state)
-      @state = state
-    end
-    protected :state
-    
-    # Return true if the daemon state is running.
-    def running?
-      state == RUNNING
-    end
-    
-    # Initialize the daemon.
-    #
-    # * <tt>processor</tt>: The processor instance
-    # * <tt>name</tt>: The processor name
-    # * <tt>waiting_queue</tt>: The waiting queue that provides messages to be processed
-    # * <tt>error_queue</tt>: The queue where errors are posted.
-    # * <tt>done_queue</tt>: The queue for messages that have been processed.
-    # * <tt>settings</tt>: The settings hash from the config.
-    #
-    # The settings hash may contain the following options:
-    # * <tt>visibility</tt>: The time in seconds that the message is hidden
-    # in the queue.
-    def initialize(processor, name, queue_prefix='', settings={})
-      logger.debug "Starting daemon"
-      
-      @processor = processor
-      @name = name
-      @settings = settings
-      
-      queue_name = settings['queue'] || name
-      queue_name = "#{queue_prefix}#{queue_name}"
-      logger.debug "Using queue #{queue_name}"
-      @queue_name = queue_name
-      
-      super do
-        begin
-          execute
-        rescue Exception => e
-          logger.error e
-        end
-      end
-    end
-    
-    private
-    def execute
-      logger.debug "Running daemon thread: #{name} (settings: #{settings.inspect})"
-      
-      if defined?(java.lang.Thread)
-        java.lang.Thread.current_thread.name = "#{name} Daemon"
-      end
-      
-      while(running?)
-        #logger.debug "Checking #{queue_name}_waiting"
-        with_queue("#{queue_name}_waiting") do |waiting_queue|
-          while (message = waiting_queue.receive(settings['visibility']))
-            worker = load_worker_class(name).new(self)
-            begin
-              result, run_time = worker.run(decode_message(message.body))
-              if result
-                with_queue("#{queue_name}_done") do |done_queue|
-                  done_message = {
-                    'host_info' => host_info,
-                    'original' => message.body,
-                    'run_time' => run_time
-                  }
-                  logger.debug "Sending 'done' message to #{done_queue.name}"
-                  done_queue.send_message(encode_message(done_message))
-                end
-              
-                logger.debug "Deleting message from queue"
-                message.delete()
-              end
-            rescue Exception => e
-              with_queue("#{queue_name}_error") do |error_queue|
-                error_message = {
-                  'error' => {
-                    'message' => e.message, 
-                    'class' => e.class.name,
-                    'backtrace' => e.backtrace
-                  }, 
-                  'host_info' => host_info,
-                  'original' => message.body
-                }
-                logger.error "Sending 'error' message to #{error_queue.name}: #{e.message}"
-                error_queue.send_message(encode_message(error_message))
-              end
-              message.delete()
-            end
-          end
-          sleep(settings['sleep'] || 5)
-        end
-      end
-      logger.debug "Exiting daemon thread"
-    end
-    
-    # A hash of worker classes
-    def workers
-      @workers ||= {}
-    end
-    
-    private
-    # Load the appropriate worker class
-    def load_worker_class(name)
-      source_file = "#{@processor.server.workers_directory}/#{name}.rb"
-      if File.exist?(source_file)
-        modified_at = File.mtime(source_file)
-        if workers[name] != modified_at
-          logger.debug "Loading #{source_file}"
-          load(source_file)
-          workers[name] = modified_at
-        end
-      else
-        raise SourceFileNotFound, "Source file not found: #{source_file}"
-      end
-      
-      Object.const_get(camelize("#{name}_worker"))
-    end
-  end
-end

data/lib/refinery/event_publisher.rb

@@ -1,131 +0,0 @@
-module Refinery #:nodoc:
-  # Publish events.
-  class EventPublisher
-    include Refinery::Loggable
-    include Refinery::Configurable
-    include Refinery::Queueable
-    include Refinery::Utilities
-    
-    STARTING = 'starting' #:nodoc:
-    RUNNING = 'running' #:nodoc:
-    STOPPED = 'stopped' #:nodoc:
-    
-    attr_accessor :publishers_directory
-    
-    # Initialize the event publisher
-    #
-    # Options:
-    # * <tt>:debug</tt>: Set to true to enable debug logging
-    # * <tt>:config</tt>: Provide a file path to load that config
-    def initialize(options={})
-      logger.level = Logger::INFO if options[:verbose]
-      logger.level = Logger::DEBUG if options[:debug]
-      config.load_file(options[:config]) if options[:config]
-      self.publishers_directory = options[:publishers] if options[:publishers]
-    end
-    
-    # Get the event publisher state
-    def state
-      @state ||= STARTING
-    end
-    
-    # Return true if the event publisher is running
-    def running?
-      state == RUNNING
-    end
-    
-    # The directory where publishers are found. Defaults to ./publishers
-    def publishers_directory
-      @publishers_directory ||= './publishers'
-    end
-    
-    # A hash of all publisher classes mapped to last modified timestamps.
-    def publishers
-      @publishers ||= {}
-    end
-    
-    # Run the specified publisher once and return
-    def run_once(key)
-      prefix = config['prefix'] || ''
-      settings = config['processors'][key]
-      raise RuntimeError, "No processor configuration found for #{key}" unless settings
-      queue_name = settings['queue'] || key
-      queue_name = "#{prefix}#{queue_name}"
-      waiting_queue_name = "#{queue_name}_waiting"
-      logger.debug "Using queue #{waiting_queue_name}"
-      load_publisher_class(key).new(waiting_queue_name).execute
-    end
-    
-    # Run the event publisher
-    def run
-      @state = RUNNING
-      logger.info "Starting event publisher"
-      config['processors'].each do |key, settings|
-        run_publisher(key, settings)
-      end
-      
-      begin
-        threads.each { |thread| thread.join }
-      rescue Interrupt => e
-      end
-      
-      logger.info "Exiting event publisher"
-    end
-    
-    private
-    # An array of threads, one for each publisher instance
-    def threads
-      @threads ||= []
-    end
-    
-    # Run the publisher for the given key
-    def run_publisher(key, settings)
-      if File.exists?(source_file(key))
-        prefix = config['prefix'] || ''
-        logger.info "Creating publisher for #{key}"
-        queue_name = settings['queue'] || key
-        queue_name = "#{prefix}#{queue_name}"
-        waiting_queue_name = "#{queue_name}_waiting"
-        logger.debug "Using queue #{waiting_queue_name}"
-      
-        threads << Thread.new(waiting_queue_name, settings) do |waiting_queue_name, settings|
-          while(running?)
-            begin
-              load_publisher_class(key).new(waiting_queue_name).execute
-            rescue Exception => e
-              logger.error "Error running publisher: #{e.message}"
-            end
-          
-            delay = settings['publishers']['delay'] || 60
-            logger.debug "Sleeping #{delay} seconds"
-            sleep delay
-          
-          end
-        end
-      else
-        logger.warn "No publisher found for #{key}"
-      end
-    end
-    
-    def source_file(key)
-      source_file = "#{publishers_directory}/#{key}.rb"
-    end
-    
-    def load_publisher_class(key)
-      source_file = source_file(key)
-      if File.exist?(source_file)
-        modified_at = File.mtime(source_file)
-        if publishers[key] != modified_at
-          logger.debug "Loading #{source_file}"
-          load(source_file)
-          publishers[key] = modified_at
-        end
-      else
-        raise SourceFileNotFound, "Source file not found: #{source_file}"
-      end
-      
-      Object.const_get(camelize("#{key}_publisher"))
-    end
-    
-  end
-end

data/lib/refinery/heartbeat.rb

@@ -1,33 +0,0 @@
-module Refinery #:nodoc:
-  # A heartbeat publisher that indicates a server is alive.
-  class Heartbeat
-    include Refinery::Loggable
-    include Refinery::Configurable
-    include Refinery::Queueable
-    include Refinery::Utilities
-    
-    # Initialize the heartbeat for the given server.
-    def initialize(server)
-      queue_prefix = config['prefix'] || ''
-      @server = server
-      @thread = Thread.new do
-        if defined?(java.lang.Thread)
-          java.lang.Thread.current_thread.name = 'Heartbeat'
-        end
-        
-        loop do
-          with_queue("#{queue_prefix}heartbeat") do |heartbeat_queue|
-            logger.debug "Send heartbeat"
-            message = {
-              'host_info' => host_info,
-              'timestamp' => Time.now.utc,
-              'running_daemons' => @server.daemons.length
-            }
-            heartbeat_queue.send_message(Base64.encode64(message.to_json))
-            sleep(60)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/refinery/loggable.rb

@@ -1,9 +0,0 @@
-module Refinery
-  # Include this module to get access to the server logger
-  module Loggable
-    # Get the logger.
-    def logger
-      @logger ||= Refinery::Server.logger
-    end
-  end
-end

data/lib/refinery/monitor.rb

@@ -1,113 +0,0 @@
-module Refinery #:nodoc:
-  # The monitor is responsible for monitoring the health of the various
-  # components of refinery.
-  class Monitor
-    include Refinery::Loggable
-    include Refinery::Configurable
-    include Refinery::Queueable
-    include Refinery::Utilities
-    
-    # Initialize the monitor.
-    #
-    # Options:
-    # * <tt>:verbose</tt>: Enable INFO level logging
-    # * <tt>:debug</tt>: Enable DEBUG level logging
-    # * <tt>:config</tt>: The config file
-    def initialize(options)
-      logger.level = Logger::INFO if options[:verbose]
-      logger.level = Logger::DEBUG if options[:debug]
-      config.load_file(options[:config]) if options[:config]
-      @queue_prefix = config['prefix'] || ''
-    end
-    
-    # Execute the monitor. The monitor will start one heartbeat 
-    # monitor thread and one thread for each done queue and error
-    # queue as specified in the configuration.
-    def run
-      logger.info "Starting up monitor"
-      heartbeat_monitor_thread = run_heartbeat_monitor
-      done_monitor_threads = run_done_monitors
-      error_monitor_threads = run_error_monitors
-      
-      logger.info "Monitor running"
-      
-      Refinery::StatsServer.new.run
-      
-      begin
-        heartbeat_monitor_thread.join
-        done_monitor_threads.each { |t| t.join }
-        error_monitor_threads.each { |t| t.join }
-      rescue Interrupt => e
-      end
-      
-      logger.info "Monitor is exiting"
-    end
-    
-    private
-    def statistics
-      @statistics ||= Refinery::Statistics.new
-    end
-    
-    def run_heartbeat_monitor
-      logger.info "Starting #{@queue_prefix}heartbeat monitor"
-      Thread.new("#{@queue_prefix}heartbeat") do |heartbeat_queue_name|
-        loop do
-          with_queue(heartbeat_queue_name) do |heartbeat_queue|
-            while (message = heartbeat_queue.receive)
-              logger.debug "#{heartbeat_queue.name}: #{decode_message(message.body).inspect}"
-              message.delete()
-            end
-          end
-          sleep(5)
-        end
-      end
-    end
-    
-    def run_done_monitors
-      config['processors'].collect do |key, settings|
-        queue_name = settings['queue'] || key
-        done_queue_name = "#{@queue_prefix}#{queue_name}_done"
-        logger.debug "Starting monitor for queue #{done_queue_name}"
-        Thread.new(done_queue_name) do |done_queue_name|
-          loop do
-            with_queue(done_queue_name) do |done_queue|
-              while (message = done_queue.receive)
-                done_message = decode_message(message.body)
-                logger.debug "#{done_queue.name}: #{done_message.pretty_inspect}"
-                processed = decode_message(done_message['original'])
-                logger.info "Done: #{queue_name} #{processed.inspect}"
-                message.delete()
-                statistics.record_done(done_message)
-              end
-              sleep(5)
-            end
-          end
-        end
-      end
-    end
-    
-    def run_error_monitors
-      config['processors'].collect do |key, settings|
-        queue_name = settings['queue'] || key
-        error_queue_name = "#{@queue_prefix}#{queue_name}_error"
-        logger.info "Starting error monitor for queue #{error_queue_name}"
-        Thread.new(error_queue_name) do |error_queue_name|
-          loop do
-            with_queue(error_queue_name) do |error_queue|
-              while (message = error_queue.receive)
-                error_message = decode_message(message.body)
-                logger.debug "#{error_queue.name}: #{error_message.pretty_inspect}"
-                processed = decode_message(error_message['original'])
-                logger.info "Error: #{queue_name} #{processed.inspect}"
-                message.delete()
-                statistics.record_error(error_message)
-              end
-            end
-            sleep(5)
-          end
-        end
-      end
-    end
-    
-  end
-end