refinery 0.9.0

data/.autotest

@@ -0,0 +1,10 @@
+module Autotest::CustomTestMatch
+  Autotest.add_hook :initialize do |at|
+    at.add_mapping(/test/) do |f, _|
+      at.files_matching(/_test\.rb$/)
+    end
+    at.add_mapping(/lib\/.*/) do |f, _|
+      at.files_matching(/_test\.rb$/)
+    end
+  end
+end

data/.gitignore

@@ -0,0 +1,5 @@
+logs/*.log
+config/config.yml
+rdoc/*
+refinery-*.gem
+stats.db

data/CHANGELOG

@@ -0,0 +1 @@
+v0.9.0. Initial release.

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2009 Anthony Eden
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,50 @@
+= Refinery
+
+Refinery is a distributed processing framework written in the Ruby
+programming language. It is designed to work with Amazon's Web
+Services such as SQS and S3 to distribute image and data processing
+across multiple severs to alleviate the need for heavy-duty data and
+image processing on web application servers.
+
+Interprocess messaging is accomplished through a distributed queue system such as Amazon SQS and data storage is accomplished through a distributed data store
+such as Amazon's S3.
+
+== Required Libraries
+
+* RightScale AWS
+* JSON
+* Moneta
+
+== Optional Libraries
+
+If you want to run the monitor then you'll need to install Sequel and have SQLite3 installed.
+
+If you want to run the stats server to view operational statistics
+about Refinery then you'll need to install Sequel, SQLite3 and Ramaze.
+
+== Executing
+
+Refinery have several executable scripts that are used to launch the various components:
+
+=== bin/refinery
+
+This command executes the refinery execution server. This is where jobs
+are run.
+
+Example: bin/refinery -c config/config.yml
+
+=== bin/epub
+
+This command executes all publishers that fire on a regular basis.
+
+Example: bin/epub -c config/config.yml
+
+=== bin/pubnow
+
+This command executes a single publisher once.
+
+Example: bin/pubnow -c config/config.yml sample
+
+=== bin/monitor
+
+This command is used to run the monitor system. The monitor system tracks the health of the refinery. The monitor should only be run on a single host.

data/README.textile

@@ -0,0 +1,50 @@
+h1. Refinery
+
+Refinery is a distributed processing framework written in the Ruby
+programming language. It is designed to work with Amazon's Web
+Services such as SQS and S3 to distribute image and data processing
+across multiple severs to alleviate the need for heavy-duty data and
+image processing on web application servers.
+
+Interprocess messaging is accomplished through a distributed queue system such as Amazon SQS and data storage is accomplished through a distributed data store
+such as Amazon's S3.
+
+h2. Required Libraries
+
+* RightScale AWS
+* JSON
+* Moneta
+
+h2. Optional Libraries
+
+If you want to run the monitor then you'll need to install Sequel and have SQLite3 installed.
+
+If you want to run the stats server to view operational statistics
+about Refinery then you'll need to install Sequel, SQLite3 and Ramaze.
+
+h2. Executing
+
+Refinery have several executable scripts that are used to launch the various components:
+
+h3. bin/refinery
+
+This command executes the refinery execution server. This is where jobs
+are run.
+
+Example: bin/refinery -c config/config.yml
+
+h3. bin/epub
+
+This command executes all publishers that fire on a regular basis.
+
+Example: bin/epub -c config/config.yml
+
+h3. bin/pubnow
+
+This command executes a single publisher once.
+
+Example: bin/pubnow -c config/config.yml sample
+
+h3. bin/monitor
+
+This command is used to run the monitor system. The monitor system tracks the health of the refinery. The monitor should only be run on a single host.

data/Rakefile

@@ -0,0 +1,39 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+desc 'Default: run tests.'
+task :default => [:test]
+
+desc 'Run tests.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Refinery'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/*.rb')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "refinery"
+    gemspec.summary = "Refinery processes data in a distributed environment."
+    gemspec.email = "anthonyeden@gmail.com"
+    gemspec.homepage = "http://github.com/aeden/refinery"
+    gemspec.description = "Process data in a distributed fashion."
+    gemspec.authors = ["Anthony Eden"]
+    gemspec.files.exclude 'docs/**/*'
+    gemspec.rubyforge_project = 'refinery'
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION

@@ -0,0 +1 @@
+0.9.0

data/bin/epub

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+# == Synopsis
+#
+# epub: run the event publisher
+#
+# == Usage
+#
+# epub [OPTION]
+#
+# -h, --help:
+#    show help
+#
+# -d, --debug:
+#    turn on debug logging
+#
+# -c, --config filename
+#    specify a configuration file
+#
+# -p, --publishers directory
+#    specify the directory for finding publishers
+#
+# -v, --verbose
+#    print info to the standard output
+
+require 'getoptlong'
+require 'rdoc/usage'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+options = {}
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--publishers', '-p', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT]
+)
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    RDoc::usage
+  when '--debug'
+    options[:debug] = true
+  when '--config'
+    options[:config] = arg
+  when '--publishers'
+    options[:publishers] = arg
+  when '--verbose'
+    options[:verbose] = true
+  end
+end
+
+Refinery::EventPublisher.new(options).run

data/bin/monitor

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+# == Synopsis
+#
+# monitor: monitor the refinery system
+#
+# == Usage
+#
+# monitor [OPTION]
+#
+# -h, --help:
+#    show help
+#
+# -d, --debug:
+#    turn on debug logging
+#
+# -c, --config filename
+#    specify a configuration file
+#
+# -v, --verbose
+#    print info to the standard output
+
+require 'getoptlong'
+require 'rdoc/usage'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+options = {}
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT]
+)
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    RDoc::usage
+  when '--debug'
+    options[:debug] = true
+  when '--config'
+    options[:config] = arg
+  when '--verbose'
+    options[:verbose] = true
+  end
+end
+
+Refinery::Monitor.new(options).run

data/bin/pubnow

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+# == Synopsis
+#
+# epub: run the event publisher
+#
+# == Usage
+#
+# pubnow [OPTION] KEY
+#
+# -h, --help:
+#    show help
+#
+# -d, --debug:
+#    turn on debug logging
+#
+# -c, --config filename
+#    specify a configuration file
+#
+# -p, --publishers directory
+#    specify the directory for finding publishers
+#
+# -v, --verbose
+#    print info to the standard output
+#
+# KEY: the name of the publisher to run. For example
+# to run the sample publisher use 'sample'
+
+require 'getoptlong'
+require 'rdoc/usage'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+options = {}
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--publishers', '-p', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT ]
+)
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    RDoc::usage
+  when '--debug'
+    options[:debug] = true
+  when '--config'
+    options[:config] = arg
+  when '--publishers'
+    options[:publishers] = arg
+  when '--verbose'
+    options[:verbose] = true
+  end
+end
+
+if ARGV.length != 1
+  puts "Missing key argument (try --help)"
+  exit 0
+end
+
+Refinery::EventPublisher.new(options).run_once(ARGV.shift)

data/bin/refinery

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+# == Synopsis
+#
+# refinery: run the Refinery server
+#
+# == Usage
+#
+# refinery [OPTION]
+#
+# -h, --help:
+#    show help
+#
+# -d, --debug:
+#    turn on debug logging
+#
+# -c, --config filename
+#    specify a configuration file
+#
+# -w, --workers directory
+#    specify the directory for finding workers
+#
+# -v, --verbose
+#    print info to the standard output
+
+require 'getoptlong'
+require 'rdoc/usage'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+options = {}
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--workers', '-w', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT]
+)
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    RDoc::usage
+  when '--debug'
+    options[:debug] = true
+  when '--config'
+    options[:config] = arg
+  when '--workers'
+    options[:workers] = arg
+  when '--verbose'
+    options[:verbose] = true
+  end
+end
+
+Refinery::Server.new(options).run

data/config/config.example.yml

@@ -0,0 +1,18 @@
+aws:
+  credentials:
+    access_key_id: "access_key_id"
+    secret_access_key: "secret_access_key"
+processors:
+  sample:
+    queue: 'sample' # can be omitted
+    publishers:
+      delay: 30
+    workers:
+      initial: 3
+      data_store:
+        class: s3
+  error:
+    publishers:
+      delay: 30
+    workers:
+      initial: 1

data/lib/refinery/config.rb

@@ -0,0 +1,48 @@
+module Refinery #:nodoc:
+  # Configuration class.
+  class Config
+    # Get a shared configuration
+    def self.default
+      @default ||= new({
+        'aws' => {
+          'credentials' => {}
+        },
+        'processors' => {}
+      })
+    end
+    
+    # Initialize the config with the given data
+    def initialize(data={})
+      @data = data
+    end
+    
+    # Get the configuration value
+    def [](key)
+      data[key.to_s]
+    end
+    
+    # Set the configuration value
+    def []=(key, value)
+      data[key.to_s] = value
+    end
+    
+    # Load configuration from a YAML file
+    def load_file(file)
+      @file = file
+      @data = YAML::load_file(@file)
+      @last_load = File.mtime(@file)
+    end
+    
+    # Refresh the configuration from the YAML file if necessary.
+    def refresh
+      if File.mtime(@file) != @last_load
+        @data = YAML::load_file(@file)
+      end
+    end
+    
+    private
+    def data
+      @data ||= {}
+    end
+  end
+end

data/lib/refinery/configurable.rb

@@ -0,0 +1,15 @@
+module Refinery #:nodoc:
+  # Include this module to get access to a shared configuration
+  module Configurable
+    # Get the configuration. If the config is nil then this will use
+    # the default shared configuration.
+    def config
+      @config ||= Refinery::Config.default
+    end
+    
+    # Set the configuration.
+    def config=(config)
+      @config = config
+    end
+  end
+end

data/lib/refinery/daemon.rb

@@ -0,0 +1,124 @@
+module Refinery #:nodoc:
+  # A daemon provides a thread to run workers in.
+  class Daemon
+    include Refinery::Loggable
+    include Refinery::Configurable
+    include Refinery::Utilities
+    
+    RUNNING = 'running'
+    STOPPED = 'stopped'
+    
+    # The daemon's thread
+    attr_reader :thread
+    # The name of the daemon
+    attr_reader :name
+    # The queue for incoming messages to process
+    attr_reader :waiting_queue
+    # The queue for outgoing messages once they've been processed
+    attr_reader :done_queue
+    # The queue for error messages
+    attr_reader :error_queue
+    
+    # Stop the daemon
+    def stop
+      self.state = STOPPED
+    end
+    
+    # Return the daemon state
+    def state
+      @state ||= RUNNING
+    end
+    
+    # Set the daemon state.
+    def state=(state)
+      @state = state
+    end
+    protected :state
+    
+    # Return true if the daemon state is running.
+    def running?
+      state == RUNNING
+    end
+    
+    # Initialize the daemon.
+    #
+    # * <tt>server</tt>: The server instance
+    # * <tt>name</tt>: The processor name
+    # * <tt>waiting_queue</tt>: The waiting queue that provides messages to be processed
+    # * <tt>error_queue</tt>: The queue where errors are posted.
+    # * <tt>done_queue</tt>: The queue for messages that have been processed.
+    def initialize(server, name, waiting_queue, error_queue, done_queue)
+      Refinery::Server.logger.debug "Starting daemon"
+      
+      @server = server
+      @name = name
+      @waiting_queue = waiting_queue
+      @error_queue = error_queue
+      @done_queue = done_queue
+      
+      @thread = Thread.new(self) do |daemon|
+        logger.debug "Running daemon thread"
+        while(running?)
+          begin
+            while (message = waiting_queue.receive)
+              worker = load_worker_class(name).new(self)
+              begin
+                result, run_time = worker.run(decode_message(message.body))
+                if result
+                  done_message = {
+                    'host_info' => host_info,
+                    'original' => message.body,
+                    'run_time' => run_time
+                  }
+                  logger.debug "Sending 'done' message to #{done_queue.name}"
+                  done_queue.send_message(encode_message(done_message))
+                
+                  logger.debug "Deleting message from queue"
+                  message.delete()
+                end
+              rescue Exception => e
+                error_message = {
+                  'error' => {
+                    'message' => e.message, 
+                    'class' => e.class.name
+                  }, 
+                  'host_info' => host_info,
+                  'original' => message.body
+                }
+                error_queue.send_message(encode_message(error_message))
+                message.delete()
+              end
+            end
+            sleep(1)
+          rescue Exception => e
+            logger.error "An error occurred while receiving from the waiting queue: #{e.message}"
+          end
+        end
+        logger.debug "Exiting daemon thread"
+      end
+    end
+    
+    # A hash of worker classes
+    def workers
+      @workers ||= {}
+    end
+    
+    private
+    # Load the appropriate worker class
+    def load_worker_class(name)
+      source_file = "#{@server.workers_directory}/#{name}.rb"
+      if File.exist?(source_file)
+        modified_at = File.mtime(source_file)
+        if workers[name] != modified_at
+          logger.debug "Loading #{source_file}"
+          load(source_file)
+          workers[name] = modified_at
+        end
+      else
+        raise SourceFileNotFound, "Source file not found: #{source_file}"
+      end
+      
+      Object.const_get(camelize(name))
+    end
+  end
+end