navvy-sequelhooks 0.3.3

data/lib/navvy/job/sequelhooks.rb

@@ -0,0 +1,144 @@
+require 'sequel'
+require 'yaml'
+
+##
+# Use this adapter if you want Sequel::Model hooks, like before_create, around_update or after_destroy.
+
+module Navvy
+  class Job < Sequel::Model
+
+    ##
+    # Add a job to the job queue.
+    #
+    # @param [Object] object the object you want to run a method from
+    # @param [Symbol, String] method_name the name of the method you want to
+    # run
+    # @param [*] arguments optional arguments you want to pass to the method
+    #
+    # @return [Job, false] created Job or false if failed
+
+    def self.enqueue(object, method_name, *args)
+      options = {}
+      if args.last.is_a?(Hash)
+        options = args.last.delete(:job_options) || {}
+        args.pop if args.last.empty?
+      end
+
+      Job[create(
+        :object =>      object.to_s,
+        :method_name => method_name.to_s,
+        :arguments =>   args.to_yaml,
+        :priority =>    options[:priority] || 0,
+        :parent_id =>   options[:parent_id],
+        :run_at =>      options[:run_at] || Time.now,
+        :created_at =>  Time.now
+      ).id]
+    end
+
+    ##
+    # Find the next available jobs in the queue. This will not include failed
+    # jobs (where :failed_at is not nil) and jobs that should run in the future
+    # (where :run_at is greater than the current time).
+    #
+    # @param [Integer] limit the limit of jobs to be fetched. Defaults to
+    # Navvy::Job.limit
+    #
+    # @return [array, nil] the next available jobs in an array or nil if no
+    # jobs were found.
+
+    def self.next(limit = self.limit)
+      filter(:failed_at => nil).
+        filter(:completed_at => nil).
+        filter{run_at <= Time.now}.
+        order(:priority.desc, :created_at).
+        first(limit)
+    end
+
+    ##
+    # Clean up jobs that we don't need to keep anymore. If Navvy::Job.keep is
+    # false it'll delete every completed job, if it's a timestamp it'll only
+    # delete completed jobs that have passed their keeptime.
+    #
+    # @return [true, false] delete_all the result of the delete_all call
+
+    def self.cleanup
+      if keep.is_a? Fixnum
+        time = Time.now - keep
+        filter{completed_at <= time}.destroy
+      else
+        filter(~{:completed_at => nil}).destroy unless keep?
+      end
+    end
+
+    ##
+    # Deletes all jobs.
+    #
+    # @return [Integer] amount the amount of jobs that were deleted
+
+    def self.delete_all
+      Navvy::Job.destroy
+    end
+
+    ##
+    # Mark the job as started. Will set started_at to the current time.
+    #
+    # @return [true, false] update_attributes the result of the
+    # update_attributes call
+
+    def started
+      update(:started_at =>  Time.now)
+      save()
+    end
+
+    ##
+    # Mark the job as completed. Will set completed_at to the current time and
+    # optionally add the return value if provided.
+    #
+    # @param [String] return_value the return value you want to store.
+    #
+    # @return [true, false] update_attributes the result of the
+    # update_attributes call
+
+    def completed(return_value = nil)
+      update(
+        :completed_at =>  Time.now,
+        :return =>        return_value
+      )
+      save()
+    end
+
+    ##
+    # Mark the job as failed. Will set failed_at to the current time and
+    # optionally add the exception message if provided. Also, it will retry
+    # the job unless max_attempts has been reached.
+    #
+    # @param [String] exception the exception message you want to store.
+    #
+    # @return [true, false] update_attributes the result of the
+    # update_attributes call
+
+    def failed(message = nil)
+      self.retry unless times_failed >= self.class.max_attempts
+      update(
+        :failed_at => Time.now,
+        :exception => message
+      )
+      save()
+    end
+
+    ##
+    # Check how many times the job has failed. Will try to find jobs with a
+    # parent_id that's the same as self.id and count them
+    #
+    # @return [Integer] count the amount of times the job has failed
+
+    def times_failed
+      i = parent_id || id
+      self.class.filter({:id => i} | {:parent_id => i}).
+        filter(~{:failed_at => nil}).
+        count
+    end
+  end
+end
+
+require 'navvy/job'

data/lib/navvy/logger.rb

@@ -0,0 +1,38 @@
+require 'logger'
+
+module Navvy
+  class Logger < Logger
+    ##
+    # Create a new logger. Works like Logger from Ruby's standard library, but
+    # defaults to STDOUT instead of failing. You can pass a filename to log to.
+    #
+    # @param [String] logdev a filename to log to, defaults to STDOUT
+    #
+    # @example
+    #   logger = Navvy::Logger.new
+    #   logger = Navvy::Logger.new('~/file.log')
+
+    def initialize(logdev = STDOUT)
+      super logdev
+    end
+
+    ##
+    # Send colored logs to the logger. Will only colorize output sent to
+    # STDOUT and will call the regular info method when writing to file.
+    #
+    # @param [String] message the message you want to log
+    # @param [String] color the color code you want to use to color your
+    # message
+    #
+    # @example
+    #   logger = Navvy::Logger.new
+    #   logger.colorized_info "I'm green!", 32
+
+    def colorized_info(message, color)
+      unless @logdev.filename
+        return info("\e[#{color}m#{message}\e[0m")
+      end
+      info(message)
+    end
+  end
+end

data/lib/navvy/tasks.rb

@@ -0,0 +1,16 @@
+task :environment
+
+namespace :navvy do
+  desc "Clear the Navvy queue."
+  task :clear => :environment do
+    Navvy::Job.delete_all
+  end
+
+  desc "Start a Navvy worker."
+  task :work => :environment do
+    Navvy::Worker.start
+  end
+end
+
+# heroku background jobs use jobs:work
+task 'jobs:work' => 'navvy:work'

data/lib/navvy/worker.rb

@@ -0,0 +1,50 @@
+module Navvy
+  class Worker
+    class << self
+      attr_writer :sleep_time
+    end
+
+    ##
+    # Sleep time of the worker.
+    #
+    # @return [Integer] sleep
+
+    def self.sleep_time
+      @sleep_time || Navvy.configuration.sleep_time
+    end
+
+    ##
+    # Start the worker.
+
+    def self.start
+      Navvy.logger.info '*** Starting ***'
+      trap('TERM') { Navvy.logger.info '*** Exiting ***'; $exit = true }
+      trap('INT')  { Navvy.logger.info '*** Exiting ***'; $exit = true }
+
+      loop do
+        fetch_and_run_jobs
+
+        if $exit
+          Navvy.logger.info '*** Cleaning up ***'
+          Navvy::Job.cleanup
+          break
+        end
+        sleep sleep_time
+      end
+    end
+
+    ##
+    # Fetch jobs and run them.
+
+    def self.fetch_and_run_jobs
+      Job.next.each do |job|
+        result = job.run
+        Navvy.logger.colorized_info(
+          "* #{job.object.to_s}.#{job.method_name}" <<
+          "(#{job.args.join(', ')}) => #{(job.exception || result).to_s}",
+          job.failed? ? 31 : 32
+        )
+      end
+    end
+  end
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+describe Navvy::Configuration do
+  after do
+    Navvy.configure do |config|
+      config.job_limit =  100
+      config.keep_jobs =  false
+      config.logger =     Navvy::Logger.new('/dev/null')
+      config.sleep_time = 5
+    end
+  end
+
+  it 'should have a job limit of 100 by default' do
+    Navvy::Job.limit.should == 100
+  end
+
+  it 'should set the job limit' do
+    Navvy.configure do |config|
+      config.job_limit = 10
+    end
+
+    Navvy::Job.limit.should == 10
+  end
+
+  it 'should have keep_jobs off by default' do
+    Navvy::Job.keep.should == false
+  end
+
+  it 'should set keep_jobs' do
+    Navvy.configure do |config|
+      config.keep_jobs = 10
+    end
+
+    Navvy::Job.keep.should == 10
+  end
+
+  it 'should set the logger' do
+    Navvy.configure do |config|
+      config.logger = Navvy::Logger.new
+    end
+
+    Navvy.logger.instance_variable_get(:@logdev).filename.should == nil
+  end
+
+  it 'should have a default sleep time of 5' do
+    Navvy::Worker.sleep_time.should == 5
+  end
+
+  it 'should turn quiet off' do
+    Navvy.configure do |config|
+      config.sleep_time = 10
+    end
+
+    Navvy::Worker.sleep_time.should == 10
+  end
+
+  it 'should have a default max_attempts of 25' do
+    Navvy::Job.max_attempts.should == 25
+  end
+
+  it 'should set max_attempts to 15' do
+    Navvy.configure do |config|
+      config.max_attempts = 15
+    end
+
+    Navvy::Job.max_attempts.should == 15
+  end
+end

data/spec/job_spec.rb

@@ -0,0 +1,505 @@
+require 'spec_helper'
+
+describe 'Navvy::Job' do
+  before do
+    Timecop.freeze(Time.local(2010, 1, 1))
+  end
+
+  after do
+    Timecop.return
+  end
+
+  describe '.keep?' do
+    after(:each) do
+      Navvy::Job.keep = false
+      Navvy.configure do |config|
+        config.keep_jobs = false
+      end
+    end
+
+    describe 'when configured using Navvy::Job.keep=' do
+      it 'should return false' do
+        Navvy::Job.keep = false
+        Navvy::Job.keep?.should == false
+      end
+
+      it 'should return true' do
+        Navvy::Job.keep = true
+        Navvy::Job.keep?.should == true
+      end
+    end
+
+    describe 'when configured with Navvy.configure' do
+      it 'should return false' do
+        Navvy.configure do |config|
+          config.keep_jobs = false
+        end
+        Navvy::Job.keep?.should == false
+      end
+
+      it 'should return true' do
+        Navvy.configure do |config|
+          config.keep_jobs = true
+        end
+        Navvy::Job.keep?.should == true
+      end
+    end
+  end
+
+  describe '.enqueue' do
+    before(:each) do
+      Navvy::Job.delete_all
+    end
+
+    it 'should enqueue a job' do
+      Navvy::Job.enqueue(Cow, :speak)
+      job_count.should == 1
+    end
+
+    it 'should set the object and the method_name' do
+      Navvy::Job.enqueue(Cow, :speak)
+      job = first_job
+      job.object.should == 'Cow'
+      job.method_name.to_s.should == 'speak'
+    end
+
+    it 'should turn the method_name into a symbol' do
+      Navvy::Job.enqueue(Cow, 'speak')
+      job = first_job
+      job.method_name.to_s.should == 'speak'
+    end
+
+    it 'should set the arguments' do
+      Navvy::Job.enqueue(Cow, :speak, true, false)
+      job = first_job
+      job.args.should == [true, false]
+    end
+
+    it 'should set the created_at date' do
+      Navvy::Job.enqueue(Cow, :speak, true, false)
+      job = first_job
+      job.created_at.should == Time.now
+    end
+
+    it 'should set the run_at date' do
+      Navvy::Job.enqueue(Cow, :speak, true, false)
+      job = first_job
+      job.run_at.should == Time.now
+      job.created_at.should == Time.now
+    end
+
+    it 'should return the enqueued job' do
+      Navvy::Job.enqueue(Cow, :speak, true, false).
+        should be_instance_of Navvy::Job
+    end
+  end
+
+  describe '.next' do
+    before(:each) do
+      Navvy::Job.delete_all
+      Navvy::Job.create(
+        :object =>      'Cow',
+        :method_name => :last,
+        :created_at =>  Time.now + (60 * 60),
+        :run_at =>      Time.now
+      )
+      Navvy::Job.create(
+        :object =>        'Cow',
+        :method_name =>   :break,
+        :completed_at =>  Time.now,
+        :run_at =>        Time.now
+      )
+      Navvy::Job.create(
+        :object =>      'Cow',
+        :method_name => :break,
+        :failed_at =>   Time.now,
+        :run_at =>      Time.now
+      )
+      Navvy::Job.create(
+        :object =>      'Cow',
+        :method_name => :tomorrow,
+        :run_at =>      Time.now + (60 * 60)
+      )
+      120.times do
+        Navvy::Job.enqueue(Cow, :speak)
+      end
+    end
+
+    it 'should find the next 10 available jobs' do
+      jobs = Navvy::Job.next
+      jobs.length.should == 100
+      jobs.each do |job|
+        job.should be_instance_of Navvy::Job
+        job.method_name.to_s.should == 'speak'
+      end
+    end
+
+    it 'should find the next 2 available jobs' do
+      Navvy::Job.next(2).length.should == 2
+    end
+
+    it 'should find the next 4 available jobs' do
+      Navvy::Job.limit = 4
+      Navvy::Job.next.length.should == 4
+    end
+  end
+
+  describe '.cleanup' do
+    before(:each) do
+      Navvy::Job.delete_all
+      Navvy::Job.create(
+        :object =>        'Cow',
+        :method_name =>   :speak,
+        :completed_at =>  Time.now - (2 * 60 * 60)
+      )
+      Navvy::Job.create(
+        :object =>        'Cow',
+        :method_name =>   :speak,
+        :completed_at =>  Time.now
+      )
+      Navvy::Job.create(
+        :object =>      'Cow',
+        :method_name => :speak
+      )
+    end
+
+    it 'should delete all complete jobs when "keep" is false' do
+      Navvy::Job.cleanup
+      job_count.should == 1
+    end
+
+    it 'should not delete any complete jobs when "keep" is true' do
+      Navvy::Job.keep = true
+      Navvy::Job.cleanup
+      job_count.should == 3
+    end
+
+    it 'should delete all complete jobs where "keep" has passed' do
+      Navvy::Job.keep = (60 * 60)
+      Navvy::Job.cleanup
+      job_count.should == 2
+    end
+  end
+
+  describe '.delete_all' do
+    it 'should delete all jobs' do
+      3.times do; Navvy::Job.create; end
+      Navvy::Job.delete_all
+      job_count.should == 0
+    end
+  end
+
+  describe '#run' do
+    it 'should pass the arguments' do
+      Navvy::Job.delete_all
+      job = Navvy::Job.enqueue(Cow, :name, 'Betsy')
+      Cow.should_receive(:name).with('Betsy')
+      job.run
+    end
+
+    describe 'when everything goes well' do
+      before(:each) do
+        Navvy::Job.delete_all
+        Navvy::Job.enqueue(Cow, :speak)
+        Navvy::Job.keep = false
+      end
+
+      it 'should run the job and delete it' do
+        jobs = Navvy::Job.next
+        jobs.first.run.should == '"moo"'
+        job_count.should == 0
+      end
+
+      describe 'when Navvy::Job.keep is set' do
+        it 'should call #completed with the return value after processing the job' do
+          Navvy::Job.keep = true
+          jobs = Navvy::Job.next
+          jobs.first.should_receive(:completed).with('"moo"')
+          jobs.first.run
+        end
+
+        it 'should mark the job as complete when keep is true' do
+          Navvy::Job.keep = true
+          jobs = Navvy::Job.next
+          jobs.first.run
+          job_count.should == 1
+          jobs.first.started_at.should == Time.now
+          jobs.first.completed_at.should == Time.now
+        end
+
+        it 'should mark the job as complete when keep has not passed yet' do
+          Navvy::Job.keep = (60 * 60)
+          jobs = Navvy::Job.next
+          jobs.first.run
+          job_count.should == 1
+          jobs.first.started_at.should == Time.now
+          jobs.first.completed_at.should == Time.now
+        end
+
+        it 'should delete the job when the "keep" flag has passed' do
+          Navvy::Job.keep = -(60 * 60)
+          jobs = Navvy::Job.next
+          jobs.first.run
+          job_count.should == 0
+        end
+      end
+    end
+
+    describe 'when a job fails' do
+      before(:each) do
+        Navvy::Job.delete_all
+        Navvy::Job.enqueue(Cow, :broken)
+      end
+
+      it 'should store the exception and current time' do
+        jobs = Navvy::Job.next
+        jobs.first.run
+        jobs.first.exception.should == 'this method is broken'
+        jobs.first.started_at.should == Time.now
+        jobs.first.failed_at.should == Time.now
+      end
+    end
+  end
+
+  describe '#started' do
+    before(:each) do
+      Navvy::Job.delete_all
+      Navvy::Job.enqueue(Cow, :speak)
+    end
+
+    it 'should update the jobs started_at date' do
+      jobs = Navvy::Job.next
+      jobs.first.started
+      jobs.first.started_at.should_not be_nil
+    end
+  end
+
+  describe '#completed' do
+    before(:each) do
+      Navvy::Job.delete_all
+      Navvy::Job.enqueue(Cow, :speak)
+    end
+
+    it 'should update the jobs completed_at date' do
+      jobs = Navvy::Job.next
+      jobs.first.completed
+      jobs.first.completed_at.should_not be_nil
+    end
+
+    it 'should set the return if provided' do
+      jobs = Navvy::Job.next
+      jobs.first.completed('woo!')
+      jobs.first.return.should == 'woo!'
+    end
+  end
+
+  describe '#failed' do
+    before(:each) do
+      Navvy::Job.delete_all
+      Navvy::Job.enqueue(Cow, :speak)
+    end
+
+    it 'should update the jobs failed_at date' do
+      jobs = Navvy::Job.next
+      jobs.first.failed
+      jobs.first.failed_at.should_not be_nil
+    end
+
+    it 'should set the exception message if provided' do
+      jobs = Navvy::Job.next
+      jobs.first.failed('broken')
+      jobs.first.exception.should == 'broken'
+    end
+
+    it 'should retry' do
+      jobs = Navvy::Job.next
+      jobs.first.should_receive(:retry)
+      jobs.first.failed('broken')
+    end
+
+    it 'should not retry when the job has failed 25 times already' do
+      jobs = Navvy::Job.next
+      jobs.first.stub!(:times_failed).and_return 25
+      jobs.first.should_not_receive(:retry)
+      jobs.first.failed('broken')
+    end
+
+    it 'should not retry when the job has failed 10 times' do
+      Navvy::Job.max_attempts = 10
+      jobs = Navvy::Job.next
+      jobs.first.stub!(:times_failed).and_return 10
+      jobs.first.should_not_receive(:retry)
+      jobs.first.failed('broken')
+    end
+  end
+
+  describe '#retry' do
+    before(:each) do
+      Navvy::Job.delete_all
+    end
+
+    it 'should enqueue a child for the failed job' do
+      failed_job = Navvy::Job.enqueue(Cow, :speak, true, false)
+      job = failed_job.retry
+      job.object.should ==            'Cow'
+      job.method_name.to_s.should ==  'speak'
+      job.args.should ==              [true, false]
+      job.parent_id.should ==         failed_job.id
+    end
+
+    it 'should handle hashes correctly' do
+      failed_job = Navvy::Job.enqueue(Cow, :speak, 'name' => 'Betsy')
+      job = failed_job.retry
+      job.args.should ==      [{'name' => 'Betsy'}]
+      job.parent_id.should == failed_job.id
+    end
+
+    it 'should set the priority' do
+      failed_job = Navvy::Job.enqueue(
+        Cow,
+        :speak,
+        'name' => 'Betsy',
+        :job_options => {
+          :priority => 2
+        }
+      )
+      job = failed_job.retry
+      job.priority.should == 2
+    end
+
+    it 'should set the run_at date to about 16 seconds from now' do
+      failed_job = Navvy::Job.enqueue(Cow, :speak, 'name' => 'Betsy')
+      failed_job.stub!(:times_failed).and_return 2
+      job = failed_job.retry
+      job.run_at.to_i.should == (Time.now + 16).to_i
+    end
+
+    it 'should set the run_at date to about 256 seconds from now' do
+      failed_job = Navvy::Job.enqueue(Cow, :speak, 'name' => 'Betsy')
+      failed_job.stub!(:times_failed).and_return 4
+      job = failed_job.retry
+      job.run_at.to_i.should == (Time.now + 256).to_i
+    end
+
+    it 'should set the run_at date to about 4096 seconds from now' do
+      failed_job = Navvy::Job.enqueue(Cow, :speak, 'name' => 'Betsy')
+      failed_job.stub!(:times_failed).and_return 8
+      job = failed_job.retry
+      job.run_at.to_i.should == (Time.now + 4096).to_i
+    end
+
+    it 'should set the parent_id to the master job id' do
+      failed_job = Navvy::Job.enqueue(Cow, :speak, 'name' => 'Betsy')
+      failed_child = failed_job.retry
+      failed_child.retry.parent_id.should == failed_job.id
+    end
+  end
+
+  describe '#times_failed' do
+    before(:each) do
+      Navvy::Job.delete_all
+      @failed_job = Navvy::Job.create(
+        :failed_at => Time.now
+      )
+    end
+
+    it 'should return 1' do
+      @failed_job.times_failed.should == 1
+    end
+
+    it 'should return 3 when having 2 failed children' do
+      2.times do
+        Navvy::Job.create(
+          :failed_at => Time.now,
+          :parent_id => @failed_job.id
+        )
+      end
+
+      @failed_job.times_failed.should == 3
+    end
+
+    it 'should return 2 when having 1 failed and one pending child' do
+      Navvy::Job.create(
+        :failed_at => Time.now,
+        :parent_id => @failed_job.id
+      )
+
+      Navvy::Job.create(
+        :parent_id => @failed_job.id
+      )
+
+      Navvy::Job.create(
+        :parent_id => @failed_job.id
+      )
+
+      @failed_job.times_failed.should == 2
+    end
+
+    it 'should return 2 when having failed and having a failed parent' do
+      failed_child =  Navvy::Job.create(
+        :failed_at => Time.now,
+        :parent_id => @failed_job.id
+      )
+      failed_child.times_failed.should == 2
+    end
+  end
+
+  describe '#ran?' do
+    it 'should return false when failed_at? and completed_at? are false' do
+      job = Navvy::Job.create
+      job.ran?.should be_false
+    end
+
+    it 'should return true when failed_at? or completed_at? is true' do
+      [
+        Navvy::Job.create(:failed_at => Time.now),
+        Navvy::Job.create(:completed_at => Time.now)
+      ].each do |job|
+        job.ran?.should be_true
+      end
+    end
+  end
+
+  describe '#duration' do
+    it 'should return a duration if started_at and completed_at are set' do
+      job = Navvy::Job.create(
+        :started_at =>    Time.now - 2,
+        :completed_at =>  Time.now
+      )
+
+      job.duration.should == 2
+    end
+
+    it 'should return a duration if started_at and failed_at are set' do
+      job = Navvy::Job.create(
+        :started_at =>  Time.now - 3,
+        :failed_at =>   Time.now
+      )
+
+      job.duration.should == 3
+    end
+
+    it 'should return 0 if only started_at is set' do
+      job = Navvy::Job.create(
+        :started_at => Time.now - 4
+      )
+
+      job.duration.should == 0
+    end
+  end
+
+  describe '#args' do
+    it 'should return an array of arguments' do
+      job = Navvy::Job.enqueue(Cow, :speak, true, false)
+      job.args.should be_instance_of Array
+      job.args.count.should == 2
+    end
+  end
+
+  describe '#namespaced' do
+    it 'should accept a namespaced class name' do
+      job = Navvy::Job.enqueue(Animals::Cow, :speak)
+      job.run.should == '"moo"'
+    end
+  end
+end