mongojob 0.0.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Michal Frackowiak
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# MongoJob
+
+MongoJob is a job queuing system inspired by [Resque](http://github.com/defunkt/resque), and using [MongoDB](http://mongodb.org) as a backend.
+
+MongoJob is specifically designed to handle both long and short-term jobs.
+
+# Current features and status
+
+- Persistent, database-backed queues and jobs
+- Worker based on EventMachine
+- Multiple ways to invoke jobs by the worker: process forking, fiber (for non-blocking jobs), blocking (in-line)
+- Pinging and reporting - workers report status every few seconds
+- Jobs with status - jobs can report progress and set custom status
+- Web interface with current workers, jobs etc.
+
+## Still TODO
+
+- MongoJob-deamon that monitors workers and jobs, kills timed-out ones
+- Job rescheduling upon failure
+- Cron-like job scheduling
+- More complete web interface
+- Command-line interface
+- Documentation
+
+# Warning
+
+The library is in early stage of development. If you are looking for a robust job scheduling system, I bet [Resque](http://github.com/defunkt/resque) is much more stable now and I highly recommend it to anyone.
+
+

data/Rakefile

@@ -0,0 +1,5 @@
+# This is the project's main Rakefile. All tasks are bundled in the tasks/ directory.
+
+require File.expand_path(File.join(File.dirname(__FILE__), "lib", "mongojob"))
+
+Dir['tasks/*.rb'].each { |task| load(task) }

data/bin/mongojob-deamon

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'mongojob'
+
+options = MongoJob::Deamon.parse_options
+worker = MongoJob::Deamon.new(options)
+worker.run

data/bin/mongojob-web

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'vegas'
+require 'mongojob/web'
+
+Vegas::Runner.new(MongoJob::Web, 'mongojob-web') do |runner, opts, app|
+  opts.on('-h HOST', "--host HOST", "set the MongoDB host") {|host|
+    MongoJob.host = host
+  }
+  opts.on('-d DATABASE_NAME', "--database-name DATABASE_NAME", "set the MongoDB database name") {|database_name|
+    MongoJob.database_name = database_name
+  }
+end

data/bin/mongojob-worker

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'mongojob'
+
+options = MongoJob::Worker.parse_options
+worker = MongoJob::Worker.new(options)
+worker.run

data/lib/mongojob.rb

@@ -0,0 +1,69 @@
+require "mongo_mapper"
+
+$: << File.dirname(__FILE__)
+MJ_ROOT = File.expand_path(File.join(File.dirname(__FILE__),'..'))
+
+require "mongojob/helpers"
+require "mongojob/version"
+require "mongojob/mixins/document"
+require "mongojob/mixins/fiber_runner"
+require "mongojob/worker"
+require "mongojob/deamon"
+require "mongojob/job"
+
+module MongoJob
+  
+  def self.host=(host)
+    @host = host
+  end
+  
+  def self.host
+    @host || 'localhost'
+  end
+  
+  def self.connection
+    split = host.split ':'
+    @connection ||= Mongo::Connection.new(split[0], (split[1] || 27017).to_i)
+  end
+  
+  def self.database_name=(database_name)
+    @database_name = database_name
+  end
+  
+  def self.database_name
+    @database_name
+  end
+  
+  def self.enqueue(klass, options = {})
+    queue_name = klass.is_a?(Class) ? queue_from_class(klass) : klass.to_s
+    raise "Given class does not return any queue name" unless queue_name
+    job = Model::Job.create({
+      klass: klass.is_a?(Class) ? klass.to_s : nil,
+      options: options,
+      queue_name: queue_name
+    })
+    job.id.to_s
+  end
+  
+  def self.reserve(queue_name, worker_id)
+    Model::Job.reserve(queue_name, worker_id)
+  end
+  
+  def self.find_job(job_id)
+    Model::Job.find job_id
+  end
+  
+  def self.dequeue(job_id)
+    Model::Job.delete(job_id)
+  end
+  
+  # Given a class, try to extrapolate an appropriate queue based on a
+  # class instance variable or `queue` method.
+  def self.queue_from_class(klass)
+    klass.instance_variable_get(:@queue) ||
+      (klass.respond_to?(:queue) and klass.queue)
+  end
+  
+end
+
+Dir[::File.join(::File.dirname(__FILE__), 'mongojob/model/*.rb')].each { |model| require(model) }

data/lib/mongojob/deamon.rb

@@ -0,0 +1,17 @@
+module MongoJob
+  class Deamon
+    
+    extend Mixins::FiberRunner::ClassMethods
+    include Mixins::FiberRunner::InstanceMethods
+    
+    
+    
+    # Runs the worker
+    def run
+      EM.run do
+        run_defined_tasks
+      end
+    end
+    
+  end
+end

data/lib/mongojob/helpers.rb

@@ -0,0 +1,32 @@
+module MongoJob
+  module Helpers
+    
+    # Given a word with dashes, returns a camel cased version of it.
+    #
+    # classify('job-name') # => 'JobName'
+    def classify(dashed_word)
+      dashed_word.split('-').each { |part| part[0] = part[0].chr.upcase }.join
+    end
+
+    # Given a camel cased word, returns the constant it represents
+    #
+    # constantize('JobName') # => JobName
+    def constantize(camel_cased_word)
+      camel_cased_word = camel_cased_word.to_s
+
+      if camel_cased_word.include?('-')
+        camel_cased_word = classify(camel_cased_word)
+      end
+
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        constant = constant.const_get(name) || constant.const_missing(name)
+      end
+      constant
+    end
+    
+  end
+end

data/lib/mongojob/job.rb

@@ -0,0 +1,63 @@
+module MongoJob
+  # You should extend this class to handle jobs
+  class Job
+    
+    attr_accessor :log
+    
+    def self.threading threading
+      @threading = threading
+    end
+    def self.fork?
+      @threading == :fork
+    end
+    def self.fiber?
+      @threading == :fiber
+    end
+    
+    def self.queue(queue = nil)
+      @queue ||= queue
+    end
+    
+    def initialize(job_object, logger = nil)
+      @job = job_object
+      @log = logger || Logger.new(STDOUT)
+    end
+    
+    def options
+      @job.options
+    end
+    
+    def id
+      @job.id
+    end
+    
+    # Please implement this method to perform any actual work.
+    def perform
+      
+    end
+    
+    # Convenience methods
+    
+    # Set the status of the job for the current itteration. <tt>num</tt> and 
+    # <tt>total</tt> are passed to the status as well as any messages. 
+    #
+    # Usage:
+    # at(0.29) - at 29%
+    # at(0.29,1.0) - at 29%
+    # at(29,100) - at 29%
+    # at(2,7) - at 2 of 7
+    # at(2,7, {status: {foo: 'bar'}}) - at 2 of 7, and set custom_status
+    def at *args
+      @job.at *args
+    end
+    
+    # Set custom status for the job. Accepts a hash as a parameter.
+    def update_status status
+      @job.set({
+        custom_status: status,
+        pinged_at: Time.now
+      })
+    end
+    
+  end
+end

data/lib/mongojob/mixins/document.rb

@@ -0,0 +1,13 @@
+module MongoJob
+  module Mixins
+    module Document
+      def connection
+        MongoJob.connection
+      end
+      
+      def database_name
+        MongoJob.database_name
+      end
+    end
+  end
+end

data/lib/mongojob/mixins/fiber_runner.rb

@@ -0,0 +1,51 @@
+module MongoJob
+  module Mixins
+    module FiberRunner
+
+      module ClassMethods
+        def task method_name, period, *args
+          @tasks ||= []
+          @tasks << {method_name: method_name, period: period, args: args}
+        end
+      end
+
+      module InstanceMethods
+        
+        def run_em_fiber period, &blk
+          # log.info "Starting tick #{method_name} with period #{period} seconds"
+          Fiber.new do
+            loop do
+              f = Fiber.current
+              begin
+                # log.debug "Running method #{method_name}"
+                blk.call
+              rescue Exception => e
+                # do something
+                # log.error "Caught exception when running #{method_name}"
+                # log.error e
+                p e
+              end
+              EM.add_timer period do
+                f.resume
+              end
+              Fiber.yield
+            end
+          end.resume
+        end
+
+        def run_defined_tasks
+          tasks = nil
+          self.class.class_eval do
+            tasks = @tasks || []
+          end
+          tasks.each do |task|
+            run_em_fiber task[:period] do
+              self.send task[:method_name], *task[:args]
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/mongojob/model/job.rb

@@ -0,0 +1,121 @@
+module MongoJob
+  module Model
+    class Job
+      include MongoMapper::Document
+      extend MongoJob::Mixins::Document
+      
+      extend Helpers
+      include Helpers
+      
+      key :queue_name, String
+      key :options, Hash
+      key :klass, String
+      
+      key :worker_id, String
+      key :status, String, default: 'queued' # one of 'queued', 'working', 'done', 'failed'
+      key :error # if failed
+      
+      key :progress, Hash # :at of :total
+      key :custom_status, Hash # Any custom status set by the worker
+      key :pinged_at, Time # Should be updated frequently by the worker
+      
+      key :release_at, Time
+      key :started_at, Time
+      key :completed_at, Time
+      timestamps!
+      
+      belongs_to :queue, class_name: 'MongoJob::Model::Queue', foreign_key: :queue_name
+      belongs_to :worker, class_name: 'MongoJob::Model::Worker', foreign_key: :worker_id
+      
+      before_create :setup_queue
+      before_create :set_release_at
+      
+      # Make sure the queue exists for a given queue name. The usual way to create a job is to provide a queue_name
+      # without caring about the unmet reference, so we need to fix it here.
+      def setup_queue
+        queue = Queue.find self.queue_name
+        unless queue
+          queue = Queue.create _id: self.queue_name
+        end
+      end
+      
+      def set_release_at
+        self.release_at ||= Time.now
+      end
+      
+      def job_class
+        @job_class ||= constantize klass
+      end
+      
+      def job_object
+        @job_object ||= job_class.new(self)
+      end
+      
+      def fail error
+        error_text = error.is_a?(Exception) ? "#{error.message}\n\n#{error.backtrace}" : error.to_s
+        self.set({
+          status: 'failed',
+          error: error_text
+        })
+        reload
+      end
+      
+      def complete
+        set({
+          status: 'done',
+          completed_at: Time.now
+        })
+      end
+      
+      # Usage:
+      # at(0.29) - at 29%
+      # at(0.29,1.0) - at 29%
+      # at(29,100) - at 29%
+      # at(2,7) - at 2 of 7
+      # at(2,7, {status: {foo: 'bar'}}) - at 2 of 7, and set custom_status
+      def at *args
+        options = args.last.is_a?(Hash) ? args.pop : {}
+        num = args[0]
+        total = args[1] || 1.0
+        custom_status = options[:status]
+        data = { pinged_at: Time.now }
+        data[:progress] = {
+          at: num,
+          total: total
+        } if num
+        data[:custom_status] = custom_status if custom_status
+        set data
+        reload
+        
+        # TODO: stop the job if cancelled, e.g. by raising an exception.
+      end
+      
+      def percent_done
+        progress['at'].to_f/ progress['total'].to_f if progress['at']
+      end
+      
+      # Pop the first unassigned job from the queue
+      def self.reserve(queue_name, worker_id)
+        begin
+          job = self.first conditions: {
+            queue_name: queue_name,
+            status: 'queued'
+          }, order: 'release_at'
+          return nil unless job
+          if job
+            # Might be free. Update it with the new status
+            self.set({id: job.id, status: 'queued'}, {status: 'working', worker_id: worker_id, started_at: Time.now})
+          end
+          job.reload
+        end while job.worker_id != worker_id
+        job
+      end
+      
+      def ping
+        set pinged_at: Time.now
+        reload
+      end
+      
+    end
+  end
+end