disbatch 0.0.4

data/lib/disbatch/plugin.rb

@@ -0,0 +1,40 @@
+module Disbatch::Plugin
+
+
+	attr_reader :plugins
+
+	@plugins = {}
+
+	# Register a class as a plugin
+	def register(plugin)
+		name = plugin.to_s
+
+		return false unless plugin.respond_to?(:execute)
+
+		@plugins[name] = plugin
+		puts "Registered #{name}"
+	end
+
+	# Return a plugin by name
+	def [](name)
+		@plugins[name]
+	end
+
+	# Attempt to load a plugin file
+	def init(file)
+		begin
+			load file
+			puts "Loaded #{file}"
+		rescue
+			puts "Error loading #{file}"
+		end
+	end
+
+	# Attempt to load all plugin files in a directory
+	def init_all(path)
+		Dir[path].each { |file| init(file) }
+	end
+
+	extend self
+
+end

data/lib/disbatch/queue/task.rb

@@ -0,0 +1,142 @@
+# Represents a Disbatch task
+class Disbatch::Queue::Task
+
+	require 'pp'
+
+	attr_reader :queue, :parameters, :id
+
+	private_class_method :new
+
+	# Create a task object
+	#
+	# @param [Disbatch::Queue] queue
+	# @param [Hash] parameters
+	# @param [String] id
+	def initialize(queue, parameters, id)
+		@queue  = queue
+		@parameters = parameters
+		@id     = id
+	end
+
+	# Take a task
+	#
+	# @param [Disbatch::Queue] queue
+	def self.take(queue)
+		doc = Mongo.try do
+			Disbatch.db[:tasks].find_and_modify({
+				:query => { :queue => queue.id, :status => -2 },
+				:update => { :$set => { :node => Disbatch.node.id, :status => -1, } }
+			})
+		end
+
+		unless doc.nil?
+			new(queue, doc['parameters'], doc['_id'])
+		end
+	end
+
+	# Create a new task
+	#
+	# @param [Disbatch::Queue queue
+	# @param [Hash] parameters
+	def self.create(queue, parameters)
+		id = Mongo.try do
+			Disbatch.db[:tasks].insert({
+			:queue => queue.id,
+			:parameters => parameters,
+			:ctime => Time.now,
+			:mtime => Time.now,
+			:node => -1,
+			:status => -2,
+			:stdout => '',
+			:stderr => '',
+			:log => []
+			})
+		end
+
+		unless id.nil?
+			new(queue, parameters, id)
+		end
+	end
+
+	# Log a message 
+	#
+	# @param [String] message
+	def log(message)
+		Disbatch.db[:tasks].update({:_id => @id}, {'$push' => {'log'=>message} })
+	end
+
+	# Query a task attribute
+	#
+	# @param [String] attribute
+	def query(attribute)
+		doc = Mongo.try { Disbatch.db[:tasks].find_one({:_id => @id}, { :fields => { attribute => 1 } }) }
+		doc.nil? ? nil : doc[attribute]
+	end
+
+	# Update a task attribute
+	#
+	# @param [Symbol] attribute
+	# @param [Object] value
+	def update(attribute, value)
+		Disbatch.db[:tasks].update({:_id => @id}, {:$set => { attribute => value } })
+	end
+
+	# Query task status
+	def status
+		query('status')
+	end
+
+	# Set task status
+	#
+	# @param [Integer] status
+	def status=(status)
+		update(:status,status)
+	end
+
+	# Query stdout
+	def stdout 
+		query('stdout')
+	end
+
+	# Set stdout
+	#
+	# @param [String] message
+	def stdout=(message)
+		update(:stdout,message)
+	end
+
+	# Query stderr
+	def stderr 
+		query('stderr')
+	end
+
+	# Set stdout
+	#
+	# @param [String] message
+	def stderr=(message)
+		update(:stdout,message)
+	end
+
+	# Fail task
+	def fail
+		self.status=-3
+	end
+
+	# Release task
+	def release
+		self.status=-2
+	end
+
+	# Conclude task
+	def conclude
+		self.status=1
+	end
+
+	# Execute the task
+	def execute!
+		Disbatch::Plugin[queue.plugin].execute(self)
+	end
+
+	alias :params :parameters
+
+end

data/lib/disbatch/queue.rb

@@ -0,0 +1,101 @@
+# Represents a Disbatch queue
+class Disbatch::Queue
+
+	require 'disbatch/queue/task'
+
+	attr_reader :plugin, :id
+
+	private_class_method :new
+
+	# Create a queue object 
+	# 
+	# @param [String] plugin
+	# @param [String] id
+	def initialize(plugin, id)
+		@id = id
+		@plugin = plugin
+	end
+
+	# Get an existing queue
+	#
+	# @param [String] id
+	def self.get(id)
+		doc = Mongo.try do
+			Disbatch.db[:queues].find_one({:_id => id})
+		end
+
+		unless doc.nil?
+			new(doc['class'], id)
+		end
+	end
+
+	# Get all existing queues
+	def self.get_all
+		Mongo.try do
+			Disbatch.db[:queues].find.map { |doc| new(doc['class'], doc['_id']) } 
+		end
+	end
+
+	# Create a new queue
+	#
+	# @param [String] plugin
+	# @param [Hash] opts
+	# @option opts [String] id
+	# @option opts [Integer] maxthreads
+	# @option opts [String] description
+	# @option opts [Array] nodes_pin
+	# @option opts [Array] nodes_ignore
+	def self.create(plugin, opts={})
+		
+		# Throw exception if plugin doesn't exist?
+		
+		id           = opts[:id]           || BSON::ObjectId.new.to_s
+		maxthreads   = opts[:maxthreads]   || 10
+		description  = opts[:description]  || ''
+		nodes_pin    = opts[:nodes_pin]    || []
+		nodes_ignore = opts[:nodes_ignore] || []
+	
+		doc = Mongo.try do
+			Disbatch.db[:queues].insert({
+				:_id             => id,
+				:class           => plugin,
+				:maxthreads      => maxthreads,
+				:description     => description,
+				:nodes_pin       => nodes_pin,
+				:nodes_ignore    => nodes_ignore,
+				:ctime           => Time.now
+			})
+		end
+
+		unless doc.nil?
+			new(plugin, id)
+		end
+
+	end
+
+	# Number of pending tasks
+	def length
+		Disbatch.db[:tasks].find({:queue => @id, :status=> -2}).count
+	end
+
+	# Push a new task onto the queue
+	def push(parameters)
+		Disbatch::Queue::Task.create(self, parameters)
+		self
+	end
+		
+	# Pop a task off the queue
+	def pop
+		Disbatch::Queue::Task.take(self)
+	end
+
+	# Check equality with another queue object
+	#
+	# @param[Disbatch::Queue] queue another queue to compare against
+	def ==(queue)
+		@id == queue.id
+	end
+
+	alias :size :length
+
+end

data/lib/disbatch.rb

@@ -0,0 +1,61 @@
+module Disbatch
+
+	# engine version
+	VERSION = 'rdisbatch 0.0.4'
+	# specification version
+	SPEC_VERSION   = '1.9'
+
+	attr_accessor :mongo_hosts
+	attr_accessor :mongo_db
+	attr_accessor :mongo_opts
+
+	attr_accessor :node_id
+
+	require 'bson'
+	require 'json'
+	require 'mongo'
+	require 'socket'
+
+	require 'disbatch/node'
+	require 'disbatch/queue'
+	require 'disbatch/plugin'
+	
+	@node_id = Socket.gethostname
+	@mongo_host = 'localhost'
+	@mongo_port = 27017
+	@mongo_db   = 'disbatch'
+	@mongo_opts = { :pool_size => 50, :safe => 1 }
+
+	# Return and cache the MongoDB connection pool
+	def db
+		@db ||= Mongo::Connection.new(@mongo_host, @mongo_port).db(@mongo_db)
+	end
+
+	# Return and cache the local node object
+	def node
+		@node ||= Disbatch::Node.get(@node_id) || Disbatch::Node.create(@node_id)
+	end
+
+	extend self
+
+end
+
+module Mongo
+
+	# Wrap a MongoDB operation for retry
+	# 
+	# @param [Integer] max_retries maximum number of times to attempt to connect
+	# @param [Float] interval frequency of retry attempts
+	def self.try(max_retries=4,interval=0.5)
+		retries = 0
+		begin
+			yield
+		rescue Mongo::ConnectionFailure => ex
+			retries +=1
+			raise ex if retries > max_retries
+			sleep(interval)
+			retry
+		end
+	end
+
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification 
+name: disbatch
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.4
+platform: ruby
+authors: 
+  - Matthew Berg
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-12-15 00:00:00 -05:00
+default_executable: 
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: bson
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: json
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :runtime
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: mongo
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :runtime
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: trollop
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :runtime
+    version_requirements: *id004
+  - !ruby/object:Gem::Dependency 
+    name: jruby-openssl
+    prerelease: false
+    requirement: &id005 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :runtime
+    version_requirements: *id005
+description: "\t\tDisbatch is a distributed, multi-language batch processing\n\
+  \t\tplatform.\tThis package provides both a client library and\n\
+  \t\texecution node implementation.\n"
+email: mberg@synacor.com
+executables: 
+  - disbatchd
+extensions: []
+
+extra_rdoc_files: 
+  - README
+  - LICENSE
+  - disbatch_specification.txt
+files: 
+  - bin/disbatchd
+  - lib/disbatch.rb
+  - lib/disbatch/queue.rb
+  - lib/disbatch/node.rb
+  - lib/disbatch/errors.rb
+  - lib/disbatch/plugin.rb
+  - lib/disbatch/plugin/hello.rb
+  - lib/disbatch/queue/task.rb
+  - doc/class_list.html
+  - doc/frames.html
+  - doc/_index.html
+  - doc/top-level-namespace.html
+  - doc/Mongo.html
+  - doc/index.html
+  - doc/file.README.html
+  - doc/method_list.html
+  - doc/file_list.html
+  - doc/Disbatch.html
+  - doc/css/full_list.css
+  - doc/css/style.css
+  - doc/css/common.css
+  - doc/Disbatch/Node.html
+  - doc/Disbatch/Queue.html
+  - doc/Disbatch/Plugin.html
+  - doc/Disbatch/NoNodeError.html
+  - doc/Disbatch/Plugins.html
+  - doc/Disbatch/Plugin/Hello.html
+  - doc/Disbatch/Queue/Task.html
+  - doc/js/app.js
+  - doc/js/jquery.js
+  - doc/js/full_list.js
+  - README
+  - LICENSE
+  - disbatch_specification.txt
+has_rdoc: true
+homepage: http://disbatch.org/projects/disbatch-ruby
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: Distributed batch processing package
+test_files: []
+