navvy 0.0.0 → 0.1.0

data/lib/navvy/job/sequel.rb

@@ -0,0 +1,200 @@
+require 'rubygems'
+require 'sequel'
+require 'yaml'
+
+module Navvy
+  class Job < Sequel::Model
+    class << self
+      attr_writer :limit
+      attr_accessor :keep
+    end
+
+    ##
+    # Default limit of jobs to be fetched
+    #
+    # @return [Integer] limit
+
+    def self.limit
+      @limit || 100
+    end
+
+    ##
+    # Should the job be kept?
+    #
+    # @return [true, false] keep
+
+    def self.keep?
+      keep = (@keep || false)
+      return (Time.now + keep) >= Time.now if keep.is_a? Fixnum
+      keep
+    end
+
+    ##
+    # Add a job to the job queue.
+    #
+    # @param [Object] object the object you want to run a method from
+    # @param [Symbol, String] method_name the name of the method you want to 
+    # run
+    # @param [*] arguments optional arguments you want to pass to the method
+    #
+    # @return [true, false]
+
+    def self.enqueue(object, method_name, *args)
+      create(
+        :object =>      object.to_s,
+        :method_name => method_name.to_s,
+        :arguments =>   YAML::dump(args),
+        :run_at =>      Time.now,
+        :created_at =>  Time.now
+      )
+    end
+
+    ##
+    # Find the next available jobs in the queue. This will not include failed
+    # jobs (where :failed_at is not nil) and jobs that should run in the future
+    # (where :run_at is greater than the current time).
+    #
+    # @param [Integer] limit the limit of jobs to be fetched. Defaults to
+    # Navvy::Job.limit
+    #
+    # @return [array, nil] the next available jobs in an array or nil if no
+    # jobs were found.
+
+    def self.next(limit = self.limit)
+      filter(
+        '`failed_at` IS NULL AND `completed_at` IS NULL AND `run_at` <= ?',
+        Time.now
+      ).order(:created_at).first(limit)
+    end
+
+    ##
+    # Clean up jobs that we don't need to keep anymore. If Navvy::Job.keep is
+    # false it'll delete every completed job, if it's a timestamp it'll only
+    # delete completed jobs that have passed their keeptime.
+    #
+    # @return [true, false] delete_all the result of the delete_all call
+
+    def self.cleanup
+      if keep.is_a? Fixnum
+        filter('`completed_at` <= ?', (Time.now - keep)).delete
+      else
+        filter('`completed_at` IS NOT NULL').delete unless keep?
+      end
+    end
+    
+    ##
+    # Run the job. Will delete the Navvy::Job record and return its return
+    # value if it runs successfully unless Navvy::Job.keep is set. If a job
+    # fails, it'll update the Navvy::Job record to include the exception
+    # message it sent back and set the :failed_at date. Failed jobs never get
+    # deleted.
+    #
+    # @example
+    #   job = Navvy::Job.next # finds the next available job in the queue
+    #   job.run               # runs the job and returns the job's return value
+    #
+    # @return [String] return value of the called method.
+
+    def run
+      begin
+        update(:started_at => Time.now)
+        result = Kernel.const_get(object).send(method_name, *args)
+        Navvy::Job.keep? ? completed : destroy
+        result
+      rescue Exception => exception
+        failed(exception.message)
+      end
+    end
+    
+    ##
+    # Mark the job as completed. Will set completed_at to the current time and 
+    # optionally add the return value if provided.
+    #
+    # @param [String] return_value the return value you want to store.
+    #
+    # @return [true, false] update_attributes the result of the
+    # update_attributes call
+    
+    def completed(return_value = nil)
+      update({
+        :completed_at =>  Time.now,
+        :return =>        return_value
+      })
+    end
+    
+    ##
+    # Mark the job as failed. Will set failed_at to the current time and 
+    # optionally add the exception message if provided.
+    #
+    # @param [String] exception the exception message you want to store.
+    #
+    # @return [true, false] update_attributes the result of the
+    # update_attributes call
+    
+    def failed(message = nil)
+      update({
+        :failed_at => Time.now,
+        :exception => message
+      })
+    end
+    
+    ##
+    # Check if the job has been run. 
+    #
+    # @return [true, false] ran
+    
+    def ran?
+      completed? || failed?
+    end
+    
+    ##
+    # Check how long it took for a job to complete or fail
+    #
+    # @return [Time, Integer] time the time it took
+    
+    def duration
+      ran? ? (completed_at || failed_at) - started_at : 0
+    end
+    
+    ##
+    # Check if completed_at is set
+    #
+    # @return [true, false] set?
+    
+    def completed_at?
+      !completed_at.nil?
+    end
+
+    ##
+    # Check if failed_at is set
+    #
+    # @return [true, false] set?
+    
+    def failed_at?
+      !failed_at.nil?
+    end
+    
+    ##
+    # Get the job arguments as an array
+    #
+    # @return [array] arguments
+    
+    def args
+      arguments.first.is_a?(Array) ? arguments : YAML.load(arguments)
+    end
+    
+    ##
+    # Get the job status
+    #
+    # @return [:pending, :completed, :failed] status
+    
+    def status
+      return :completed if completed?
+      return :failed if failed?
+      :pending
+    end
+    
+    alias_method :completed?, :completed_at?
+    alias_method :failed?,    :failed_at?
+  end
+end

data/lib/navvy/log.rb

@@ -0,0 +1,60 @@
+module Navvy
+  class Log  
+    class << self
+      attr_writer :logger
+      attr_accessor :quiet
+    end
+    
+    class LoggerNotFound < StandardError; end
+
+    def self.logger
+      @logger
+    end
+
+    ##
+    # Pass a log to the logger. It will check if self.logger is an array. If it
+    # is, it'll loop through it and log to every logger. If it's not, it'll
+    # just log once.
+    #
+    # @param [String] message the message you want to log
+    # @param [Integer] color an optional color code to use in the terminal
+    # output
+
+    def self.info(message, color = nil)
+      if logger.is_a? Array
+        logger.each do |logger|
+          write(logger, message, color)
+        end
+      else
+        write(logger, message, color)
+      end
+    end
+    
+    ##
+    # Actually write the log to the logger. It'll check self.logger and use
+    # that to define a logger
+    #
+    # @param [Symbol] logger the logger you want to use
+    # @param [String] message the message you want to log
+    # @param [Integer] color an optional color code to use in the terminal
+    # output
+    
+    def self.write(logger, message, color = nil)
+      puts "\e[#{color}m#{message}\e[0m" unless quiet
+      case logger
+      when :justlogging
+        raise(
+          LoggerNotFound,
+          'JustLogging could not be found. No logs were created.'
+        ) unless defined? Justlogging.log
+        Justlogging.log(message)  
+      when :rails
+        raise(
+          LoggerNotFound,
+          'RAILS_DEFAULT_LOGGER could not be found. No logs were created.'
+        ) unless defined? RAILS_DEFAULT_LOGGER.info
+        RAILS_DEFAULT_LOGGER.info(message)
+      end
+    end
+  end
+end

data/lib/navvy/tasks.rb

@@ -0,0 +1,13 @@
+task :environment
+ 
+namespace :navvy do
+  desc "Clear the Navvy queue."
+  task :clear => :environment do
+    Navvy::Job.delete_all
+  end
+ 
+  desc "Start a Navvy worker."
+  task :work => :environment do
+    Navvy::Worker.start
+  end
+end

data/lib/navvy/worker.rb

@@ -1,22 +1,37 @@
 module Navvy
   class Worker
-    
+
     ##
     # Start the worker.
-    
+
     def self.start
+      Navvy::Log.info '*** Starting ***'
+      trap('TERM') { Navvy::Log.info '*** Exiting ***'; $exit = true }
+      trap('INT')  { Navvy::Log.info '*** Exiting ***'; $exit = true }
+
       loop do
         fetch_and_run_jobs
+
+        if $exit
+          Navvy::Log.info '*** Cleaning up ***'
+          Navvy::Job.cleanup
+          break
+        end
         sleep 5
       end
     end
-    
+
     ##
-    # Fetch jobs an run them.
-    
+    # Fetch jobs and run them.
+
     def self.fetch_and_run_jobs
       Job.next.each do |job|
-        job.run
+        result = job.run
+        Navvy::Log.info(
+          "* #{job.object.to_s}.#{job.method_name}" <<
+          "(#{job.args.join(', ')}) => #{(job.exception || result).to_s}",
+          job.failed? ? 31 : 32
+        )
       end
     end
   end

data/navvy.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{navvy}
-  s.version = "0.0.0"
+  s.version = "0.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jeff Kreeftmeijer"]
-  s.date = %q{2010-01-11}
+  s.date = %q{2010-01-21}
   s.description = %q{Simple background job processor inspired by delayed_job, but aiming for database agnosticism.}
   s.email = %q{jeff@kreeftmeijer.nl}
   s.extra_rdoc_files = [
@@ -23,14 +23,24 @@ Gem::Specification.new do |s|
      "README.textile",
      "Rakefile",
      "VERSION",
-     "lib/job/mongo_mapper.rb",
+     "generators/navvy/navvy_generator.rb",
+     "generators/navvy/templates/migration.rb",
+     "lib/generators/navvy_generator.rb",
      "lib/navvy.rb",
-     "lib/navvy/job.rb",
+     "lib/navvy/job/active_record.rb",
      "lib/navvy/job/mongo_mapper.rb",
+     "lib/navvy/job/sequel.rb",
+     "lib/navvy/log.rb",
+     "lib/navvy/tasks.rb",
      "lib/navvy/worker.rb",
      "navvy.gemspec",
+     "spec/job/active_record_spec.rb",
      "spec/job/mongo_mapper_spec.rb",
+     "spec/job/sequel_spec.rb",
+     "spec/log_spec.rb",
+     "spec/setup/active_record.rb",
      "spec/setup/mongo_mapper.rb",
+     "spec/setup/sequel.rb",
      "spec/spec_helper.rb",
      "spec/worker_spec.rb"
   ]
@@ -40,8 +50,13 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.5}
   s.summary = %q{Simple background job processor inspired by delayed_job, but aiming for database agnosticism.}
   s.test_files = [
-    "spec/job/mongo_mapper_spec.rb",
+    "spec/job/active_record_spec.rb",
+     "spec/job/mongo_mapper_spec.rb",
+     "spec/job/sequel_spec.rb",
+     "spec/log_spec.rb",
+     "spec/setup/active_record.rb",
      "spec/setup/mongo_mapper.rb",
+     "spec/setup/sequel.rb",
      "spec/spec_helper.rb",
      "spec/worker_spec.rb"
   ]
@@ -51,26 +66,29 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<mongo_mapper>, [">= 0.6.10"])
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_development_dependency(%q<yard>, [">= 0.5.2"])
       s.add_development_dependency(%q<metric_fu>, [">= 1.1.6"])
       s.add_development_dependency(%q<machinist>, [">= 1.0.6"])
+      s.add_development_dependency(%q<mongo_mapper>, [">= 0.6.10"])
       s.add_development_dependency(%q<machinist_mongomapper>, [">= 0.9.7"])
+      s.add_development_dependency(%q<sequel>, [">= 3.8.0"])
     else
-      s.add_dependency(%q<mongo_mapper>, [">= 0.6.10"])
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_dependency(%q<yard>, [">= 0.5.2"])
       s.add_dependency(%q<metric_fu>, [">= 1.1.6"])
       s.add_dependency(%q<machinist>, [">= 1.0.6"])
+      s.add_dependency(%q<mongo_mapper>, [">= 0.6.10"])
       s.add_dependency(%q<machinist_mongomapper>, [">= 0.9.7"])
+      s.add_dependency(%q<sequel>, [">= 3.8.0"])
     end
   else
-    s.add_dependency(%q<mongo_mapper>, [">= 0.6.10"])
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
     s.add_dependency(%q<yard>, [">= 0.5.2"])
     s.add_dependency(%q<metric_fu>, [">= 1.1.6"])
     s.add_dependency(%q<machinist>, [">= 1.0.6"])
+    s.add_dependency(%q<mongo_mapper>, [">= 0.6.10"])
     s.add_dependency(%q<machinist_mongomapper>, [">= 0.9.7"])
+    s.add_dependency(%q<sequel>, [">= 3.8.0"])
   end
 end

data/spec/job/active_record_spec.rb

@@ -0,0 +1,325 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe 'Navvy::Job' do
+  before do
+    require File.expand_path(File.dirname(__FILE__) + '/../setup/active_record')
+  end
+
+  describe '.enqueue' do
+    before(:each) do
+      Navvy::Job.delete_all
+    end
+
+    it 'should enqueue a job' do
+      Navvy::Job.enqueue(Cow, :speak)
+      Navvy::Job.count.should == 1
+    end
+
+    it 'should set the object and the method_name' do
+      Navvy::Job.enqueue(Cow, :speak)
+      job = Navvy::Job.first
+      job.object.should == 'Cow'
+      job.method_name.should == 'speak'
+    end
+
+    it 'should turn the method_name into a symbol' do
+      Navvy::Job.enqueue(Cow, 'speak')
+      job = Navvy::Job.first
+      job.method_name.should == 'speak'
+    end
+
+    it 'should set the arguments' do
+      Navvy::Job.enqueue(Cow, :speak, true, false)
+      job = Navvy::Job.first
+      YAML.load(job.arguments).should == [true, false]
+    end
+
+    it 'should set the created_at date' do
+      Navvy::Job.enqueue(Cow, :speak, true, false)
+      job = Navvy::Job.first
+      job.created_at.should be_instance_of Time
+      job.created_at.should <= Time.now
+      job.created_at.should > 10.seconds.ago
+    end
+
+    it 'should set the run_at date' do
+      Navvy::Job.enqueue(Cow, :speak, true, false)
+      job = Navvy::Job.first
+      job.run_at.should be_instance_of Time
+      job.run_at.should <= Time.now
+      job.created_at.should > 10.seconds.ago
+    end
+
+    it 'should return the enqueued job' do
+      Navvy::Job.enqueue(Cow, :speak, true, false).
+        should be_instance_of Navvy::Job
+    end
+  end
+
+  describe '.next' do
+    before(:each) do
+      Navvy::Job.delete_all
+      Navvy::Job.create(
+        :object =>      'Cow',
+        :method_name => :last,
+        :created_at =>  Time.now + 1.day,
+        :run_at =>        Time.now
+      )
+      Navvy::Job.create(
+        :object =>        'Cow',
+        :method_name =>   :break,
+        :completed_at =>  Time.now,
+        :run_at =>        Time.now
+      )
+      Navvy::Job.create(
+        :object =>      'Cow',
+        :method_name => :break,
+        :failed_at =>   Time.now,
+        :run_at =>      Time.now
+      )
+      Navvy::Job.create(
+        :object =>      'Cow',
+        :method_name => :tomorrow,
+        :run_at =>      Time.now + 1.day
+      )
+      120.times do
+        Navvy::Job.enqueue(Cow, :speak)
+      end
+    end
+
+    it 'should find the next 10 available jobs' do
+      jobs = Navvy::Job.next
+      jobs.count.should == 100
+      jobs.each do |job|
+        job.should be_instance_of Navvy::Job
+        job.method_name.should == 'speak'
+      end
+    end
+
+    it 'should find the next 2 available jobs' do
+      Navvy::Job.next(2).count.should == 2
+    end
+
+    it 'should find the next 4 available jobs' do
+      Navvy::Job.limit = 4
+      Navvy::Job.next.count.should == 4
+    end
+  end
+
+  describe '.cleanup' do
+    before(:each) do
+      Navvy::Job.delete_all
+      Navvy::Job.create(
+        :object =>        'Cow',
+        :method_name =>   :speak,
+        :completed_at =>  Time.now - 2.days
+      )
+      Navvy::Job.create(
+        :object =>        'Cow',
+        :method_name =>   :speak,
+        :completed_at =>  Time.now
+      )
+      Navvy::Job.create(
+        :object =>      'Cow',
+        :method_name => :speak
+      )
+    end
+
+    it 'should delete all complete jobs when "keep" is false' do
+      Navvy::Job.cleanup
+      Navvy::Job.count.should == 1
+    end
+
+    it 'should not delete any complete jobs when "keep" is true' do
+      Navvy::Job.keep = true
+      Navvy::Job.cleanup
+      Navvy::Job.count.should == 3
+    end
+
+    it 'should delete all complete jobs where "keep" has passed' do
+      Navvy::Job.keep = 1.day
+      Navvy::Job.cleanup
+      Navvy::Job.count.should == 2
+    end
+  end
+
+  describe '#run' do
+
+    it 'should pass the arguments' do
+      Navvy::Job.delete_all
+      job = Navvy::Job.enqueue(Cow, :name, 'Betsy')
+      Cow.should_receive(:name).with('Betsy')
+      job.run
+    end
+
+    describe 'when everything goes well' do
+      before(:each) do
+        Navvy::Job.delete_all
+        Navvy::Job.enqueue(Cow, :speak)
+        Navvy::Job.keep = false
+      end
+
+      it 'should run the job and delete it' do
+        jobs = Navvy::Job.next
+        jobs.first.run.should == 'moo'
+        Navvy::Job.count.should == 0
+      end
+
+      describe 'when Navvy::Job.keep is set' do
+        it 'should mark the job as complete when keep is true' do
+          Navvy::Job.keep = true
+          jobs = Navvy::Job.next
+          jobs.first.run
+          Navvy::Job.count.should == 1
+          jobs.first.started_at.should be_instance_of Time
+          jobs.first.completed_at.should be_instance_of Time
+        end
+
+        it 'should mark the job as complete when keep has not passed yer' do
+          Navvy::Job.keep = 1.day
+          jobs = Navvy::Job.next
+          jobs.first.run
+          Navvy::Job.count.should == 1
+          jobs.first.started_at.should be_instance_of Time
+          jobs.first.completed_at.should be_instance_of Time
+        end
+
+        it 'should delete the job when the "keep" flag has passed' do
+          Navvy::Job.keep = -1.day
+          jobs = Navvy::Job.next
+          jobs.first.run
+          Navvy::Job.count.should == 0
+        end
+      end
+    end
+
+    describe 'when a job fails' do
+      before(:each) do
+        Navvy::Job.delete_all
+        Navvy::Job.enqueue(Cow, :broken)
+      end
+
+      it 'should store the exception and current time' do
+        jobs = Navvy::Job.next
+        jobs.first.run
+        Navvy::Job.count.should == 1
+        jobs.first.exception.should == 'this method is broken'
+        jobs.first.started_at.should be_instance_of Time
+        jobs.first.failed_at.should be_instance_of Time
+      end
+    end
+  end
+
+  describe '#completed' do
+    before(:each) do
+      Navvy::Job.delete_all
+      Navvy::Job.enqueue(Cow, :speak)
+    end
+
+    it 'should update the jobs completed_at date' do
+      jobs = Navvy::Job.next
+      jobs.first.completed
+      jobs.first.completed_at.should_not be_nil
+    end
+
+    it 'should set the return if provided' do
+      jobs = Navvy::Job.next
+      jobs.first.completed('woo!')
+      jobs.first.return.should == 'woo!'
+    end
+  end
+
+  describe '#failed' do
+    before(:each) do
+      Navvy::Job.delete_all
+      Navvy::Job.enqueue(Cow, :speak)
+    end
+
+    it 'should update the jobs failed_at date' do
+      jobs = Navvy::Job.next
+      jobs.first.failed
+      jobs.first.failed_at.should_not be_nil
+    end
+
+    it 'should set the exception message if provided' do
+      jobs = Navvy::Job.next
+      jobs.first.failed('broken')
+      jobs.first.exception.should == 'broken'
+    end
+  end
+
+  describe '#ran?' do
+    it 'should return false when failed_at? and completed_at? are false' do
+      job = Navvy::Job.create
+      job.ran?.should be_false
+    end
+
+    it 'should return true when failed_at? or completed_at? is true' do
+      [
+        Navvy::Job.create(:failed_at => Time.now),
+        Navvy::Job.create(:completed_at => Time.now)
+      ].each do |job|
+        job.ran?.should be_true
+      end
+    end
+  end
+
+  describe '#duration' do
+    it 'should return a duration if started_at and completed_at are set' do
+      job = Navvy::Job.create(
+        :started_at =>    2.seconds.ago,
+        :completed_at =>  Time.now
+      )
+
+      job.duration.should >= 2.seconds
+    end
+
+    it 'should return a duration if started_at and failed_at are set' do
+      job = Navvy::Job.create(
+        :started_at =>  3.seconds.ago,
+        :failed_at =>   Time.now
+      )
+
+      job.duration.should >= 3.seconds
+    end
+
+    it 'should return 0 if only started_at is set' do
+      job = Navvy::Job.create(
+        :started_at => 4.seconds.ago
+      )
+
+      job.duration.should == 0
+    end
+  end
+
+  describe '#args' do
+    it 'should return an array of arguments' do
+      job = Navvy::Job.enqueue(Cow, :speak, true, false)
+      job.args.should be_instance_of Array
+      job.args.count.should == 2
+    end
+  end
+
+  describe '#status' do
+    before(:each) do
+      Navvy::Job.delete_all
+    end
+
+    it 'should return :pending' do
+      job = Navvy::Job.enqueue(Cow, :speak)
+      job.status.should == :pending
+    end
+
+    it 'should return :completed' do
+      job = Navvy::Job.enqueue(Cow, :speak)
+      job.completed
+      job.status.should == :completed
+    end
+
+    it 'should return :failed' do
+      job = Navvy::Job.enqueue(Cow, :speak)
+      job.failed
+      job.status.should == :failed
+    end
+  end
+end