refinery 0.12.2 → 1.0

data/README.rdoc

@@ -1,58 +0,0 @@
-= Refinery
-
-Refinery is a distributed processing framework written in the Ruby
-programming language. It is designed to work with Amazon's Web
-Services such as SQS and S3 to distribute image and data processing
-across multiple severs to alleviate the need for heavy-duty data and
-image processing on web application servers.
-
-Interprocess messaging is accomplished through a distributed queue system such as Amazon SQS and data storage is accomplished through a distributed data store
-such as Amazon's S3.
-
-== Required Libraries
-
-* RightScale AWS
-* JSON
-* Moneta
-
-== Optional Libraries
-
-If you want to run the monitor then you'll need to install Sequel and have SQLite3 installed.
-
-If you want to run the stats server to view operational statistics
-about Refinery then you'll need to install Sequel, SQLite3 and Ramaze.
-
-== Executing
-
-Refinery have several executable scripts that are used to launch the various components:
-
-=== bin/refinery
-
-This command executes the refinery execution server. This is where jobs
-are run.
-
-Example: bin/refinery -c config/config.yml
-
-=== bin/epub
-
-This command executes all publishers that fire on a regular basis.
-
-Example: bin/epub -c config/config.yml
-
-=== bin/pubnow
-
-This command executes a single publisher once.
-
-Example: bin/pubnow -c config/config.yml sample
-
-== Tests
-
-To run the tests, do one of the following:
-
-* Use autotest
-* Use the command `rake`
-* Run by hand with `ruby -Ilib -Itest unit/test_to_run.rb`
-
-=== bin/monitor
-
-This command is used to run the monitor system. The monitor system tracks the health of the refinery. The monitor should only be run on a single host.

data/README.textile

@@ -1,58 +0,0 @@
-h1. Refinery
-
-Refinery is a distributed processing framework written in the Ruby
-programming language. It is designed to work with Amazon's Web
-Services such as SQS and S3 to distribute image and data processing
-across multiple severs to alleviate the need for heavy-duty data and
-image processing on web application servers.
-
-Interprocess messaging is accomplished through a distributed queue system such as Amazon SQS and data storage is accomplished through a distributed data store
-such as Amazon's S3.
-
-h2. Required Libraries
-
-* RightScale AWS
-* JSON
-* Moneta
-
-h2. Optional Libraries
-
-If you want to run the monitor then you'll need to install Sequel and have SQLite3 installed.
-
-If you want to run the stats server to view operational statistics
-about Refinery then you'll need to install Sequel, SQLite3 and Ramaze.
-
-h2. Executing
-
-Refinery have several executable scripts that are used to launch the various components:
-
-h3. bin/refinery
-
-This command executes the refinery execution server. This is where jobs
-are run.
-
-Example: bin/refinery -c config/config.yml
-
-h3. bin/epub
-
-This command executes all publishers that fire on a regular basis.
-
-Example: bin/epub -c config/config.yml
-
-h3. bin/pubnow
-
-This command executes a single publisher once.
-
-Example: bin/pubnow -c config/config.yml sample
-
-h3. bin/monitor
-
-This command is used to run the monitor system. The monitor system tracks the health of the refinery. The monitor should only be run on a single host.
-
-h2. Tests
-
-To run the tests, do one of the following:
-
-* Use autotest
-* Use the command `rake`
-* Run by hand with `ruby -Ilib -Itest unit/test_to_run.rb`

data/Rakefile

@@ -1,43 +0,0 @@
-require 'rake'
-require 'rake/testtask'
-require 'rake/rdoctask'
-
-desc 'Default: run tests.'
-task :default => [:test]
-
-desc 'Run tests.'
-Rake::TestTask.new(:test) do |t|
-  t.libs << 'lib' << 'test'
-  t.pattern = 'test/**/*_test.rb'
-  t.verbose = true
-end
-
-desc 'Generate documentation.'
-Rake::RDocTask.new(:rdoc) do |rdoc|
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title    = 'Refinery'
-  rdoc.options << '--line-numbers' << '--inline-source'
-  rdoc.rdoc_files.include('README.rdoc')
-  rdoc.rdoc_files.include('lib/*.rb')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gemspec|
-    gemspec.name = "refinery"
-    gemspec.summary = "Refinery processes data in a distributed environment."
-    gemspec.email = "anthonyeden@gmail.com"
-    gemspec.homepage = "http://github.com/aeden/refinery"
-    gemspec.description = "Process data in a distributed fashion."
-    gemspec.authors = ["Anthony Eden"]
-    gemspec.files.exclude 'docs/**/*'
-    gemspec.files.exclude '.autotest'
-    gemspec.rubyforge_project = 'refinery'
-    gemspec.add_dependency "right_aws"
-    gemspec.add_dependency "moneta"
-    gemspec.add_dependency "addressable"
-  end
-rescue LoadError
-  puts "Jeweler not available. Install it with: sudo gem install jeweler"
-end

data/VERSION

@@ -1 +0,0 @@
-0.12.2

data/bin/epub

@@ -1,64 +0,0 @@
-#!/usr/bin/env ruby
-
-# == Synopsis
-#
-# epub: run the event publisher
-#
-# == Usage
-#
-# epub [OPTION]
-#
-# -h, --help:
-#    show help
-#
-# -d, --debug:
-#    turn on debug logging
-#
-# -c, --config filename
-#    specify a configuration file
-#
-# -p, --publishers directory
-#    specify the directory for finding publishers
-#
-# -v, --verbose
-#    print info to the standard output
-
-require 'getoptlong'
-require 'rdoc/usage'
-require 'refinery'
-
-pidfile = 'epub.pid'
-
-options = {}
-opts = GetoptLong.new(
-  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
-  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
-  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
-  [ '--publishers', '-p', GetoptLong::REQUIRED_ARGUMENT],
-  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT],
-  [ '--pidfile', GetoptLong::REQUIRED_ARGUMENT]
-)
-opts.each do |opt, arg|
-  case opt
-  when '--help'
-    RDoc::usage
-  when '--debug'
-    options[:debug] = true
-  when '--config'
-    options[:config] = arg
-  when '--publishers'
-    options[:publishers] = arg
-  when '--verbose'
-    options[:verbose] = true
-  when '--pidfile'
-    pidfile = arg
-  end
-end
-
-open(pidfile, 'w') do |f|
-  f << $$
-end
-
-Refinery::EventPublisher.new(options).run
-
-File.delete(pidfile)

data/bin/monitor

@@ -1,47 +0,0 @@
-#!/usr/bin/env ruby
-
-# == Synopsis
-#
-# monitor: monitor the refinery system
-#
-# == Usage
-#
-# monitor [OPTION]
-#
-# -h, --help:
-#    show help
-#
-# -d, --debug:
-#    turn on debug logging
-#
-# -c, --config filename
-#    specify a configuration file
-#
-# -v, --verbose
-#    print info to the standard output
-
-require 'getoptlong'
-require 'rdoc/usage'
-require 'refinery'
-
-options = {}
-opts = GetoptLong.new(
-  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
-  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
-  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
-  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT]
-)
-opts.each do |opt, arg|
-  case opt
-  when '--help'
-    RDoc::usage
-  when '--debug'
-    options[:debug] = true
-  when '--config'
-    options[:config] = arg
-  when '--verbose'
-    options[:verbose] = true
-  end
-end
-
-Refinery::Monitor.new(options).run

data/bin/pubnow

@@ -1,61 +0,0 @@
-#!/usr/bin/env ruby
-
-# == Synopsis
-#
-# pubnow: run the event publisher
-#
-# == Usage
-#
-# pubnow [OPTION] KEY
-#
-# -h, --help:
-#    show help
-#
-# -d, --debug:
-#    turn on debug logging
-#
-# -c, --config filename
-#    specify a configuration file
-#
-# -p, --publishers directory
-#    specify the directory for finding publishers
-#
-# -v, --verbose
-#    print info to the standard output
-#
-# KEY: the name of the publisher to run. For example
-# to run the sample publisher use 'sample'
-
-require 'getoptlong'
-require 'rdoc/usage'
-require 'refinery'
-
-options = {}
-opts = GetoptLong.new(
-  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
-  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
-  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
-  [ '--publishers', '-p', GetoptLong::REQUIRED_ARGUMENT],
-  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT ]
-)
-opts.each do |opt, arg|
-  case opt
-  when '--help'
-    RDoc::usage
-  when '--debug'
-    options[:debug] = true
-  when '--config'
-    options[:config] = arg
-  when '--publishers'
-    options[:publishers] = arg
-  when '--verbose'
-    options[:verbose] = true
-  end
-end
-
-if ARGV.length != 1
-  puts "Missing key argument (try --help)"
-  exit 0
-end
-
-Refinery::EventPublisher.new(options).run_once(ARGV.shift)

data/bin/refinery

@@ -1,64 +0,0 @@
-#!/usr/bin/env ruby
-
-# == Synopsis
-#
-# refinery: run the Refinery server
-#
-# == Usage
-#
-# refinery [OPTION]
-#
-# -h, --help:
-#    show help
-#
-# -d, --debug:
-#    turn on debug logging
-#
-# -c, --config filename
-#    specify a configuration file
-#
-# -w, --workers directory
-#    specify the directory for finding workers
-#
-# -v, --verbose
-#    print info to the standard output
-
-require 'getoptlong'
-require 'rdoc/usage'
-require 'refinery'
-
-pidfile = 'refinery.pid'
-
-options = {}
-opts = GetoptLong.new(
-  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
-  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
-  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
-  [ '--workers', '-w', GetoptLong::REQUIRED_ARGUMENT],
-  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT],
-  [ '--pidfile', GetoptLong::REQUIRED_ARGUMENT]
-)
-opts.each do |opt, arg|
-  case opt
-  when '--help'
-    RDoc::usage
-  when '--debug'
-    options[:debug] = true
-  when '--config'
-    options[:config] = arg
-  when '--workers'
-    options[:workers] = arg
-  when '--verbose'
-    options[:verbose] = true
-  when '--pidfile'
-    pidfile = arg
-  end
-end
-
-open(pidfile, 'w') do |f|
-  f << $$
-end
-
-Refinery::Server.new(options).run
-
-File.delete(pidfile)

data/config/config.example.yml

@@ -1,21 +0,0 @@
-aws:
-  credentials:
-    access_key_id: "access_key_id"
-    secret_access_key: "secret_access_key"
-# queue_engine: 
-#   provider: 'beanstalk'
-prefix: 'example_'
-processors:
-  sample:
-    queue: 'sample' # can be omitted
-    publishers:
-      delay: 30
-    workers:
-      initial: 3
-      data_store:
-        class: s3
-  error:
-    publishers:
-      delay: 30
-    workers:
-      initial: 1

data/lib/refinery/beanstalk_queue.rb

@@ -1,36 +0,0 @@
-module Refinery #:nodoc:
-  # An interface to beanstalk using SQS-compatible methods.
-  class BeanstalkQueue
-    include Refinery::Loggable
-    attr_reader :name
-    
-    # Construct a BeanstalkQueue instance.
-    #
-    # *<tt>name</tt>: if specified then that "tube" will be used.
-    # *<tt>host</tt>: if specified then those host:port combos will be used.
-    def initialize(name=nil, hosts=nil)
-      @name = name.gsub(/_/, '-') # beanstalk does not like underscores in tube names
-      @hosts = hosts || ['localhost:11300']
-    end
-    
-    # Get the next message from the queue
-    def receive(visibility=nil)
-      beanstalk.reserve(visibility)
-    end
-    
-    # Get the approximate queue size
-    def size
-      beanstalk.stats_tube(name)['current-jobs-ready']
-    end
-    
-    # Send a message
-    def send_message(message)
-      beanstalk.put(message)
-    end
-    
-    protected
-    def beanstalk
-      @beanstalk ||= Beanstalk::Pool.new(@hosts, name)
-    end
-  end
-end