aeden-refinery 0.9.0

data/CHANGELOG

@@ -0,0 +1 @@
+v0.9.0. Initial release.

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2009 Anthony Eden
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,50 @@
+= Refinery
+
+Refinery is a distributed processing framework written in the Ruby
+programming language. It is designed to work with Amazon's Web
+Services such as SQS and S3 to distribute image and data processing
+across multiple severs to alleviate the need for heavy-duty data and
+image processing on web application servers.
+
+Interprocess messaging is accomplished through a distributed queue system such as Amazon SQS and data storage is accomplished through a distributed data store
+such as Amazon's S3.
+
+== Required Libraries
+
+* RightScale AWS
+* JSON
+* Moneta
+
+== Optional Libraries
+
+If you want to run the monitor then you'll need to install Sequel and have SQLite3 installed.
+
+If you want to run the stats server to view operational statistics
+about Refinery then you'll need to install Sequel, SQLite3 and Ramaze.
+
+== Executing
+
+Refinery have several executable scripts that are used to launch the various components:
+
+=== bin/refinery
+
+This command executes the refinery execution server. This is where jobs
+are run.
+
+Example: bin/refinery -c config/config.yml
+
+=== bin/epub
+
+This command executes all publishers that fire on a regular basis.
+
+Example: bin/epub -c config/config.yml
+
+=== bin/pubnow
+
+This command executes a single publisher once.
+
+Example: bin/pubnow -c config/config.yml sample
+
+=== bin/monitor
+
+This command is used to run the monitor system. The monitor system tracks the health of the refinery. The monitor should only be run on a single host.

data/README.textile

@@ -0,0 +1,50 @@
+h1. Refinery
+
+Refinery is a distributed processing framework written in the Ruby
+programming language. It is designed to work with Amazon's Web
+Services such as SQS and S3 to distribute image and data processing
+across multiple severs to alleviate the need for heavy-duty data and
+image processing on web application servers.
+
+Interprocess messaging is accomplished through a distributed queue system such as Amazon SQS and data storage is accomplished through a distributed data store
+such as Amazon's S3.
+
+h2. Required Libraries
+
+* RightScale AWS
+* JSON
+* Moneta
+
+h2. Optional Libraries
+
+If you want to run the monitor then you'll need to install Sequel and have SQLite3 installed.
+
+If you want to run the stats server to view operational statistics
+about Refinery then you'll need to install Sequel, SQLite3 and Ramaze.
+
+h2. Executing
+
+Refinery have several executable scripts that are used to launch the various components:
+
+h3. bin/refinery
+
+This command executes the refinery execution server. This is where jobs
+are run.
+
+Example: bin/refinery -c config/config.yml
+
+h3. bin/epub
+
+This command executes all publishers that fire on a regular basis.
+
+Example: bin/epub -c config/config.yml
+
+h3. bin/pubnow
+
+This command executes a single publisher once.
+
+Example: bin/pubnow -c config/config.yml sample
+
+h3. bin/monitor
+
+This command is used to run the monitor system. The monitor system tracks the health of the refinery. The monitor should only be run on a single host.

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+desc 'Default: run tests.'
+task :default => [:test]
+
+desc 'Run tests.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Refinery'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/*.rb')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/bin/epub

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+# == Synopsis
+#
+# epub: run the event publisher
+#
+# == Usage
+#
+# epub [OPTION]
+#
+# -h, --help:
+#    show help
+#
+# -d, --debug:
+#    turn on debug logging
+#
+# -c, --config filename
+#    specify a configuration file
+#
+# -p, --publishers directory
+#    specify the directory for finding publishers
+#
+# -v, --verbose
+#    print info to the standard output
+
+require 'getoptlong'
+require 'rdoc/usage'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+options = {}
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--publishers', '-p', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT]
+)
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    RDoc::usage
+  when '--debug'
+    options[:debug] = true
+  when '--config'
+    options[:config] = arg
+  when '--publishers'
+    options[:publishers] = arg
+  when '--verbose'
+    options[:verbose] = true
+  end
+end
+
+Refinery::EventPublisher.new(options).run

data/bin/monitor

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+# == Synopsis
+#
+# monitor: monitor the refinery system
+#
+# == Usage
+#
+# monitor [OPTION]
+#
+# -h, --help:
+#    show help
+#
+# -d, --debug:
+#    turn on debug logging
+#
+# -c, --config filename
+#    specify a configuration file
+#
+# -v, --verbose
+#    print info to the standard output
+
+require 'getoptlong'
+require 'rdoc/usage'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+options = {}
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT]
+)
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    RDoc::usage
+  when '--debug'
+    options[:debug] = true
+  when '--config'
+    options[:config] = arg
+  when '--verbose'
+    options[:verbose] = true
+  end
+end
+
+Refinery::Monitor.new(options).run

data/bin/pubnow

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+# == Synopsis
+#
+# epub: run the event publisher
+#
+# == Usage
+#
+# pubnow [OPTION] KEY
+#
+# -h, --help:
+#    show help
+#
+# -d, --debug:
+#    turn on debug logging
+#
+# -c, --config filename
+#    specify a configuration file
+#
+# -p, --publishers directory
+#    specify the directory for finding publishers
+#
+# -v, --verbose
+#    print info to the standard output
+#
+# KEY: the name of the publisher to run. For example
+# to run the sample publisher use 'sample'
+
+require 'getoptlong'
+require 'rdoc/usage'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+options = {}
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--publishers', '-p', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT ]
+)
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    RDoc::usage
+  when '--debug'
+    options[:debug] = true
+  when '--config'
+    options[:config] = arg
+  when '--publishers'
+    options[:publishers] = arg
+  when '--verbose'
+    options[:verbose] = true
+  end
+end
+
+if ARGV.length != 1
+  puts "Missing key argument (try --help)"
+  exit 0
+end
+
+Refinery::EventPublisher.new(options).run_once(ARGV.shift)

data/bin/refinery

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+# == Synopsis
+#
+# refinery: run the Refinery server
+#
+# == Usage
+#
+# refinery [OPTION]
+#
+# -h, --help:
+#    show help
+#
+# -d, --debug:
+#    turn on debug logging
+#
+# -c, --config filename
+#    specify a configuration file
+#
+# -w, --workers directory
+#    specify the directory for finding workers
+#
+# -v, --verbose
+#    print info to the standard output
+
+require 'getoptlong'
+require 'rdoc/usage'
+require File.dirname(__FILE__) + '/../lib/refinery'
+
+options = {}
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--debug', '-d', GetoptLong::NO_ARGUMENT ],
+  [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--workers', '-w', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--verbose', '-v', GetoptLong::NO_ARGUMENT]
+)
+opts.each do |opt, arg|
+  case opt
+  when '--help'
+    RDoc::usage
+  when '--debug'
+    options[:debug] = true
+  when '--config'
+    options[:config] = arg
+  when '--workers'
+    options[:workers] = arg
+  when '--verbose'
+    options[:verbose] = true
+  end
+end
+
+Refinery::Server.new(options).run

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification 
+name: aeden-refinery
+version: !ruby/object:Gem::Version 
+  version: 0.9.0
+platform: ruby
+authors: 
+- Anthony Eden
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-05-16 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Process data in a distributed fashion.
+email: anthonyeden@gmail.com
+executables: 
+- refinery
+- epub
+- pubnow
+- monitor
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- logs
+- publishers
+- Rakefile
+- README.rdoc
+- README.textile
+- LICENSE
+- CHANGELOG
+- workers
+has_rdoc: true
+homepage: http://github.com/aeden/refinery
+post_install_message: 
+rdoc_options: 
+- --title
+- Refinery - Distributed Processing
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: refinery
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Refinery processes data in a distributed environment.
+test_files: []
+