disbatch 0.0.8 → 0.0.9

data/bin/disbatchd

@@ -11,6 +11,9 @@ opts = Trollop.options do
 	opt 'config', 'the disbatchd config file', :default => '/etc/disbatch/disbatchd.conf'
 	opt 'max', 'maximum number of threads per queue', :default => 10
 	opt 'plugins', 'path(s) to dispatch plugins', :multi => true, :type => :string
+    opt 'mongo_host', 'mongo host'
+    opt 'mongo_port', 'mongo port'
+    opt 'mongo_db', 'mongo db'
 	opt 'force',  'force node registration'
 end
 
@@ -20,25 +23,27 @@ config = opts.merge(config)
 force = config['force'] || false
 
 node = Disbatch.node.register(force)
-max  = config['max']
+max  = config.has_key?('max') ? config['max'] : 10
 
-config.has_key?('plugins') && config['plugins'].each { |dir| Disbatch::Plugin.init_all(dir) }
 Disbatch::Plugin.init_all(File.dirname(File.expand_path(__FILE__)) + '/../lib/disbatch/plugin/**/*.rb')
+config.has_key?('plugins') && config['plugins'].each { |dir| Disbatch::Plugin.init_all(dir) }
+
+config.has_key?('mongo_host') && Disbatch.mongo_host = config['mongo_host']
+config.has_key?('mongo_port') && Disbatch.mongo_port = config['mongo_port']
+config.has_key?('mongo_db') && Disbatch.mongo_db = config['mongo_db']
 
 trap('TERM') { node.release; exit }
 trap('INT')  { node.release; exit }
 trap('QUIT') { node.release; exit }
 
-runners = {}
+@runners = {}
 
-EventMachine::run do
-
-	EventMachine::add_periodic_timer(5) do
+def update_queues
 
 		Disbatch::Queue.get_all.each do |queue|
 
 			# runner already exists
-			next if runners.has_key?(queue.id)
+			next if @runners.has_key?(queue.id)
 
 			# node configured to ignore
 			# next if queue.nodes_ignore.include?(node.id)
@@ -51,20 +56,29 @@ EventMachine::run do
 
 			puts "Adding #{queue.plugin} runner for #{queue.id}"
 
-			runners[queue.id] = { 'tg' => ThreadGroup.new, 'queue' => queue }
+			@runners[queue.id] = { 'tg' => ThreadGroup.new, 'queue' => queue }
 
 		end
 
+end
+
+update_queues
+
+EventMachine::run do
+
+	EventMachine::add_periodic_timer(5) do
+
+		update_queues
 
-		runners.each do |id, runner|
+		@runners.each do |id, runner|
 			puts "#{id}: #{runner['queue'].plugin}: #{runner['tg'].list.length}"
 		end
 
 	end
 
-	EventMachine::add_periodic_timer(1) do
+	EventMachine::add_periodic_timer(0.5) do
 
-		runners.each do |id, runner|
+		@runners.each do |id, runner|
 
 			tg = runner['tg']
 			queue = runner['queue']

data/lib/disbatch/error.rb

@@ -0,0 +1,24 @@
+module Disbatch
+
+	# Raised when attempting to open an invalid node
+	class NoNodeError < RuntimeError; end
+
+	# Raised when attempting to open an owned node
+	class RegisteredNodeError < RuntimeError; end
+
+	# Raised when attempting to open a node twice
+	class AlreadyRegisteredNodeError < RuntimeError; end
+
+	# Raised when attempting to release an unregistered node
+	class NotRegisteredNodeError < RuntimeError; end
+
+	# Raised when specifying a non-existant plugin
+	class NoPluginError < RuntimeError; end
+
+	# Raised when specifying an invalid plugin
+	class InvalidPluginError < RuntimeError; end
+
+	# Raised when specifying a non-existent queue
+	class NoQueueError < RuntimeError; end
+
+end

data/lib/disbatch/node.rb

@@ -9,7 +9,7 @@ class Disbatch::Node
 	#
 	# @param [String] id 
 	def initialize(id)
-		@open = false
+		@registered = false
 		@id   = id
 	end
 
@@ -18,7 +18,8 @@ class Disbatch::Node
 	# @param [String] id
 	def self.get(id = Disbatch.node_id)
 		Mongo.try do
-			Disbatch.db[:nodes].find_one({:_id => id}) ? new(id) : nil
+			raise Disbatch::NoNodeError unless Disbatch.db[:nodes].find_one({:_id => id})
+			new(id)
 		end
 	end
 
@@ -45,32 +46,38 @@ class Disbatch::Node
 
 	# Register node
 	def register(force = false)
-		return true if @open
-
-		# FIXME - check for existing registration seperately, throw exception
-
-		doc = Mongo.try do
-			Disbatch.db[:nodes].find_and_modify({
-				:query  => force ? { :_id => @id } : { :_id => @id, :pid => nil },
-				:update => { :$set => {
-					:pid          => Process.pid,
-					:version      => Disbatch::VERSION,
-					:spec_version => Disbatch::SPEC_VERSION
-				} }
-			})
+		raise Disbatch::AlreadyRegisteredNodeError if @registered
+
+		begin
+			doc = Mongo.try do
+				Disbatch.db[:nodes].find_and_modify({
+					:query  => force ? { :_id => @id } : { :_id => @id, :pid => nil },
+					:update => { :$set => {
+						:pid          => Process.pid,
+						:version      => Disbatch::VERSION,
+						:spec_version => Disbatch::SPEC_VERSION
+					} }
+				})
+			end
+		rescue
+			doc = Mongo.try do
+				Disbatch.db[:nodes].find({ :_id => @id })
+			end
+
+			if doc.count == 0
+				raise Disbatch::NoNodeError
+			else
+				raise Disbatch::RegisteredNodeError
+			end
 		end
 
-		if doc.nil?
-			false
-		else
-			@open = true
-			self
-		end
+		@registered = true
+		self
 	end
 
 	# Release ownership of node
 	def release
-		return false unless @open
+		raise Disbatch::NotRegisteredNodeError unless @registered
 
 		doc = Mongo.try do
 			Disbatch.db[:nodes].find_and_modify({
@@ -86,13 +93,14 @@ class Disbatch::Node
 		if doc.nil? 
 			false
 		else
-			@open = false
+			@registered = false
 			true
 		end
 	end
 
-	def open?
-		@open
+	# Check if process is registered as this node.
+	def registered?
+		@registered
 	end
 
 	# Check equality with another node object
@@ -102,4 +110,7 @@ class Disbatch::Node
 		@id == node.id
 	end
 
+	# Deprecated.
+	alias :open? :registered?
+
 end

data/lib/disbatch/plugin.rb

@@ -1,6 +1,5 @@
 module Disbatch::Plugin
 
-
 	attr_reader :plugins
 
 	@plugins = {}
@@ -9,7 +8,7 @@ module Disbatch::Plugin
 	def register(plugin)
 		name = plugin.to_s
 
-		return false unless plugin.respond_to?(:execute)
+		raise Disbatch::InvalidPluginError unless plugin.respond_to?(:execute)
 
 		@plugins[name] = plugin
 		puts "Registered #{name}"
@@ -17,6 +16,7 @@ module Disbatch::Plugin
 
 	# Return a plugin by name
 	def [](name)
+		raise Disbatch::NoPluginError unless @plugins.has_key?(name)
 		@plugins[name]
 	end
 
@@ -24,7 +24,6 @@ module Disbatch::Plugin
 	def init(file)
 		begin
 			load file
-			puts "Loaded #{file}"
 		rescue
 			puts "Error loading #{file}"
 		end
@@ -37,4 +36,6 @@ module Disbatch::Plugin
 
 	extend self
 
+	init_all('disbatch/plugin/**/*')
+
 end

data/lib/disbatch/queue.rb

@@ -1,7 +1,7 @@
 # Represents a Disbatch queue
 class Disbatch::Queue
 
-	require 'disbatch/queue/task'
+	require 'disbatch/task'
 
 	attr_reader :plugin, :id
 
@@ -24,9 +24,7 @@ class Disbatch::Queue
 			Disbatch.db[:queues].find_one({:_id => id})
 		end
 
-		unless doc.nil?
-			new(doc['class'], id)
-		end
+		raise Disbatch::NoQueueError if doc.nil?
 	end
 
 	# Get all existing queues
@@ -46,8 +44,7 @@ class Disbatch::Queue
 	# @option opts [Array] nodes_pin
 	# @option opts [Array] nodes_ignore
 	def self.create(plugin, opts={})
-		
-		# Throw exception if plugin doesn't exist?
+		raise Disbatch::NoPluginError unless Disbatch::Plugin[plugin]
 		
 		id           = opts[:id]           || BSON::ObjectId.new.to_s
 		maxthreads   = opts[:maxthreads]   || 10
@@ -70,7 +67,6 @@ class Disbatch::Queue
 		unless doc.nil?
 			new(plugin, id)
 		end
-
 	end
 
 	# Number of pending tasks
@@ -80,13 +76,13 @@ class Disbatch::Queue
 
 	# Push a new task onto the queue
 	def push(parameters)
-		Disbatch::Queue::Task.create(self, parameters)
+		Disbatch::Task.create(self, parameters)
 		self
 	end
 		
 	# Pop a task off the queue
 	def pop
-		Disbatch::Queue::Task.take(self)
+		Disbatch::Task.take(self)
 	end
 
 	# Check equality with another queue object

data/lib/disbatch/{queue/task.rb → task.rb}

@@ -1,12 +1,19 @@
 # Represents a Disbatch task
-class Disbatch::Queue::Task
-
-	require 'pp'
+class Disbatch::Task
 
 	attr_reader :queue, :parameters, :id
 
 	private_class_method :new
 
+	module Status
+		BLOCKED    = -4
+		TERMINATED = -3
+		CREATED    = -2
+		CLAIMED    = -1
+		RUNNING    =  0
+		CONCLUDED  =  1
+	end
+
 	# Create a task object
 	#
 	# @param [Disbatch::Queue] queue
@@ -24,8 +31,8 @@ class Disbatch::Queue::Task
 	def self.take(queue)
 		doc = Mongo.try do
 			Disbatch.db[:tasks].find_and_modify({
-				:query => { :queue => queue.id, :status => -2 },
-				:update => { :$set => { :node => Disbatch.node.id, :status => -1, } }
+				:query => { :queue => queue.id, :status => Status::CREATED },
+				:update => { :$set => { :node => Disbatch.node.id, :status => Status::CLAIMED, } }
 			})
 		end
 
@@ -46,7 +53,7 @@ class Disbatch::Queue::Task
 			:ctime => Time.now,
 			:mtime => Time.now,
 			:node => -1,
-			:status => -2,
+			:status => Status::CREATED,
 			:stdout => '',
 			:stderr => '',
 			:log => []

data/lib/disbatch.rb

@@ -1,11 +1,12 @@
 module Disbatch
 
 	# engine version
-	VERSION = 'rdisbatch 0.0.8'
+	VERSION = 'rdisbatch 0.0.9'
 	# specification version
 	SPEC_VERSION   = '1.9'
 
-	attr_accessor :mongo_hosts
+	attr_accessor :mongo_host
+	attr_accessor :mongo_port
 	attr_accessor :mongo_db
 	attr_accessor :mongo_opts
 
@@ -16,6 +17,7 @@ module Disbatch
 	require 'mongo'
 	require 'socket'
 
+	require 'disbatch/error'
 	require 'disbatch/node'
 	require 'disbatch/queue'
 	require 'disbatch/plugin'
@@ -33,7 +35,11 @@ module Disbatch
 
 	# Return and cache the local node object
 	def node
-		@node ||= Disbatch::Node.get(@node_id) || Disbatch::Node.create(@node_id)
+		begin
+			@node ||= Disbatch::Node.get(@node_id)
+		rescue Disbatch::NoNodeError
+			Disbatch::Node.create(@node_id)
+		end
 	end
 
 	extend self

metadata

@@ -2,7 +2,7 @@
 name: disbatch
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors: 
   - Matthew Berg
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-12-15 00:00:00 -05:00
+date: 2012-01-17 00:00:00 -05:00
 default_executable: 
 dependencies: 
   - !ruby/object:Gem::Dependency 
@@ -85,10 +85,10 @@ files:
   - lib/disbatch.rb
   - lib/disbatch/queue.rb
   - lib/disbatch/node.rb
-  - lib/disbatch/errors.rb
+  - lib/disbatch/error.rb
+  - lib/disbatch/task.rb
   - lib/disbatch/plugin.rb
   - lib/disbatch/plugin/hello.rb
-  - lib/disbatch/queue/task.rb
   - doc/class_list.html
   - doc/frames.html
   - doc/_index.html

data/lib/disbatch/errors.rb

@@ -1,6 +0,0 @@
-module Disbatch
-
-	# Raised when attempting to open an invalid node
-	class NoNodeError < RuntimeError; end
-
-end