inst-jobs 0.11.0

data/spec/active_record_job_spec.rb

@@ -0,0 +1,246 @@
+require File.expand_path("../spec_helper", __FILE__)
+
+describe 'Delayed::Backed::ActiveRecord::Job' do
+  before :all do
+    Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
+  end
+
+  after :all do
+    Delayed.send(:remove_const, :Job)
+  end
+
+  before do
+    Delayed::Testing.clear_all!
+  end
+
+  include_examples 'a delayed_jobs implementation'
+
+  it "should recover as well as possible from a failure failing a job" do
+    allow(Delayed::Job::Failed).to receive(:create).and_raise(RuntimeError)
+    job = "test".send_later_enqueue_args :reverse, no_delay: true
+    job_id = job.id
+    proc { job.fail! }.should raise_error(RuntimeError)
+    proc { Delayed::Job.find(job_id) }.should raise_error(ActiveRecord::RecordNotFound)
+    Delayed::Job.count.should == 0
+  end
+
+  context "when another worker has worked on a task since the job was found to be available, it" do
+    before :each do
+      @job = Delayed::Job.create :payload_object => SimpleJob.new
+      @job_copy_for_worker_2 = Delayed::Job.find(@job.id)
+    end
+
+    it "should not allow a second worker to get exclusive access if already successfully processed by worker1" do
+      @job.destroy
+      @job_copy_for_worker_2.send(:lock_exclusively!, 'worker2').should == false
+    end
+
+    it "should not allow a second worker to get exclusive access if failed to be processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
+      @job.update_attributes(:attempts => 1, :run_at => 1.day.from_now)
+      @job_copy_for_worker_2.send(:lock_exclusively!, 'worker2').should == false
+    end
+
+    it "should select the next job at random if enabled" do
+      begin
+        Delayed::Settings.select_random_from_batch = true
+        15.times { "test".send_later :length }
+        founds = []
+        15.times do
+          job = Delayed::Job.get_and_lock_next_available('tester')
+          founds << job
+          job.unlock
+          job.save!
+        end
+        founds.uniq.size.should > 1
+      ensure
+        Delayed::Settings.select_random_from_batch = false
+      end
+    end
+  end
+
+  it "should unlock a successfully locked job and persist the job's unlocked state" do
+    job = Delayed::Job.create :payload_object => SimpleJob.new
+    job.send(:lock_exclusively!, 'worker1').should == true
+    job.unlock
+    job.save!
+    job.reload
+    job.locked_by.should == nil
+    job.locked_at.should == nil
+  end
+
+  describe "bulk_update failed jobs" do
+    context "holding/unholding failed jobs" do
+      before :each do
+        @job = Delayed::Job.create :payload_object => SimpleJob.new
+        Delayed::Job.get_and_lock_next_available('worker1').should == @job
+        @job.fail!
+      end
+
+      it "should raise error when holding failed jobs" do
+        expect { Delayed::Job.bulk_update('hold', :flavor => 'failed', :query => @query) }.to raise_error(RuntimeError)
+      end
+
+      it "should raise error unholding failed jobs" do
+        expect { Delayed::Job.bulk_update('unhold', :flavor => 'failed', :query => @query) }.to raise_error(RuntimeError)
+      end
+    end
+
+    context "deleting failed jobs" do
+      before :each do
+        2.times {
+          j = Delayed::Job.create(:payload_object => SimpleJob.new)
+          j.send(:lock_exclusively!, 'worker1').should == true
+          j.fail!
+        }
+      end
+
+      it "should delete failed jobs by id" do
+        target_ids = Delayed::Job::Failed.all[0..2].map { |j| j.id }
+        Delayed::Job.bulk_update('destroy', :ids => target_ids, :flavor => 'failed', :query => @query).should == target_ids.length
+      end
+
+      it "should delete all failed jobs" do
+        failed_count = Delayed::Job::Failed.count
+        Delayed::Job.bulk_update('destroy', :flavor => 'failed', :query => @query).should == failed_count
+      end
+    end
+  end
+
+  context 'n_strand' do
+    it "should default to 1" do
+      expect(Delayed::Job).to receive(:rand).never
+      job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
+      job.strand.should == "njobs"
+    end
+
+    it "should set max_concurrent based on num_strands" do
+      change_setting(Delayed::Settings, :num_strands, ->(strand_name) { expect(strand_name).to eql "njobs"; "3" }) do
+        job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
+        job.strand.should == "njobs"
+        job.max_concurrent.should == 3
+      end
+    end
+
+    context "with two parameters" do
+      it "should use the first param as the setting to read" do
+        job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
+        job.strand.should == "njobs/123"
+        change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
+          case strand_name
+          when "njobs"; 3
+          else nil
+          end
+        }) do
+          job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
+          job.strand.should == "njobs/123"
+          job.max_concurrent.should == 3
+        end
+      end
+
+      it "should allow overridding the setting based on the second param" do
+        change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
+          case strand_name
+          when "njobs/123"; 5
+          else nil
+          end
+        }) do
+          job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
+          job.strand.should == "njobs/123"
+          job.max_concurrent.should == 5
+          job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
+          job.strand.should == "njobs/456"
+          job.max_concurrent.should == 1
+        end
+
+        change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
+          case strand_name
+          when "njobs/123"; 5
+          when "njobs"; 3
+          else nil
+          end
+        }) do
+          job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
+          job.strand.should == "njobs/123"
+          job.max_concurrent.should == 5
+          job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
+          job.strand.should == "njobs/456"
+          job.max_concurrent.should == 3
+        end
+      end
+    end
+
+    context "max_concurrent triggers" do
+      before do
+        skip("postgres specific") unless ActiveRecord::Base.connection.adapter_name == 'PostgreSQL'
+      end
+
+      it "should set one job as next_in_strand at a time with max_concurrent of 1" do
+        job1 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
+        job1.reload
+        job1.next_in_strand.should == true
+        job2 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
+        job2.reload
+        job2.next_in_strand.should == false
+        run_job(job1)
+        job2.reload
+        job2.next_in_strand.should == true
+      end
+
+      it "should set multiple jobs as next_in_strand at a time based on max_concurrent" do
+        change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
+          case strand_name
+          when "njobs"; 2
+          else nil
+          end
+        }) do
+          job1 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
+          job1.reload
+          job1.next_in_strand.should == true
+          job2 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
+          job2.reload
+          job2.next_in_strand.should == true
+          job3 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
+          job3.reload
+          job3.next_in_strand.should == false
+          run_job(job1)
+          job3.reload
+          job3.next_in_strand.should == true
+        end
+      end
+
+      it "should set multiple jobs as next_in_strand at once if needed" do
+        change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
+          case strand_name
+          when "njobs"; 2
+          else nil
+          end
+        }) do
+          job1 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
+          job1.reload
+          job1.next_in_strand.should == true
+          job2 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
+          job2.reload
+          job2.next_in_strand.should == true
+
+          job3 = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs"])
+          job3.reload
+          job3.next_in_strand.should == false
+
+          # manually unset next_in_strand
+          Delayed::Job.where(:id => job2).update_all(:next_in_strand => false)
+          job2.reload
+          job2.next_in_strand.should == false
+
+          run_job(job1) # should update both jobs
+
+          job3.reload
+          job3.next_in_strand.should == true
+
+          job2.reload
+          job2.next_in_strand.should == true
+
+        end
+      end
+    end
+  end
+end

data/spec/delayed/cli_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::CLI do
+  describe '#parse_cli_options!' do
+    it 'correctly parses the --config option' do
+      cli = described_class.new(%w{run --config /path/to/some/file.yml})
+      options = cli.parse_cli_options!
+      expect(options).to include config_file: '/path/to/some/file.yml'
+    end
+  end
+
+  describe '#run' do
+    before do
+      expect(Delayed::Settings).to receive(:worker_config).and_return({})
+    end
+
+    it 'prints help when no command is given' do
+      cli = described_class.new([])
+      expect(cli).to receive(:puts).with(/Usage/)
+      cli.run
+    end
+  end
+end

data/spec/delayed/daemon_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::Daemon do
+  let(:pid_folder) { "/test/pid/folder" }
+  let(:pid) { 9999 }
+  let(:subject) { described_class.new(pid_folder) }
+
+  before do
+    allow(subject).to receive(:pid).and_return(pid)
+  end
+
+  describe '#stop' do
+    it 'prints status if not running' do
+      expect(subject).to receive(:status).with(print: false, pid: pid).and_return(false)
+      expect(subject).to receive(:status).with(no_args)
+      expect(Process).to receive(:kill).never
+      subject.stop
+    end
+
+    it 'prints status if draining' do
+      expect(subject).to receive(:status).with(print: false, pid: pid).and_return(:draining)
+      expect(subject).to receive(:status).with(no_args)
+      expect(Process).to receive(:kill).never
+      subject.stop
+    end
+
+    it 'sends INT by default' do
+      expect(subject).to receive(:status).with(print: false, pid: pid).and_return(:running)
+      expect(subject).to receive(:puts).with(/Stopping pool/)
+      expect(Process).to receive(:kill).with('INT', pid)
+      expect(subject).to receive(:wait).with(false)
+      subject.stop
+    end
+  end
+end

data/spec/delayed/server_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+require 'delayed/server'
+
+RSpec.describe Delayed::Server, sinatra: true do
+  include Rack::Test::Methods
+
+  def app
+    described_class.new
+  end
+
+  def parsed_body
+    @parsed_body ||= JSON.parse(last_response.body)
+  end
+
+  before :all do
+    Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
+  end
+
+  after :all do
+    Delayed.send(:remove_const, :Job)
+  end
+
+  describe "get '/running'" do
+    before do
+      3.times do |i|
+        Delayed::Job.create!({
+          run_at: Time.now,
+          locked_at: Time.now,
+          locked_by: "dummy-runner-#{i}:${$$}",
+        })
+      end
+      get '/running'
+    end
+
+    it 'must return a json object with the running job data in an array', aggregate_failures: true do
+      expect(last_response).to be_ok
+      expect(parsed_body['data']).to be_an Array
+      expect(parsed_body['data'].size).to eq 3
+    end
+  end
+
+  describe "get '/jobs'" do
+    let!(:job_1) { Delayed::Job.enqueue(SimpleJob.new, strand: 'strand-1') }
+    let!(:job_2) { Delayed::Job.enqueue(SimpleJob.new, strand: 'strand-2') }
+    let!(:job_3) { Delayed::Job.enqueue(SimpleJob.new, strand: 'strand-3') }
+
+    context 'with the flavor param set to id' do
+      before do
+        get "/jobs?flavor=id&search_term=#{job_2.id}"
+      end
+
+      it 'must only return the job with the id specified in the search_term param' do
+        jobs = parsed_body['data']
+        job_ids = jobs.map{ |j| j['id'] }
+        expect(job_ids).to eq [job_2.id]
+      end
+
+      it 'must set recordsFiltered in the response to 1' do
+        expect(parsed_body['recordsFiltered']).to eq 1
+      end
+    end
+  end
+end

data/spec/delayed/settings_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::Settings do
+  let(:configfile) {<<-YAML
+default:
+  workers:
+  - queue: myqueue
+    workers: 2
+  - queue: secondqueue
+    max_priority: 7
+  max_attempts: 1
+    YAML
+  }
+
+  describe '.worker_config' do
+    it 'merges each worker config with the top-level config' do
+      expect(File).to receive(:read).with("fname").and_return(configfile)
+      config = described_class.worker_config("fname")
+      expect(config[:workers]).to eq([
+        {'queue' => 'myqueue', 'workers' => 2, 'max_attempts' => 1},
+        {'queue' => 'secondqueue', 'max_priority' => 7, 'max_attempts' => 1},
+      ])
+    end
+  end
+
+  describe '.apply_worker_config!' do
+    it 'applies global settings from the given config' do
+      expect(described_class).to receive(:last_ditch_logfile=).with(true)
+      described_class.apply_worker_config!('last_ditch_logfile' => true)
+    end
+  end
+end

data/spec/delayed/work_queue/in_process_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::WorkQueue::InProcess do
+  before :all do
+    Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
+  end
+
+  after :all do
+    Delayed.send(:remove_const, :Job)
+  end
+
+  after :each do
+    Delayed::Worker.lifecycle.reset!
+  end
+
+  let(:subject) { described_class.new }
+  let(:args) { ["worker_name", "queue_name", 1, 2] }
+
+  it 'triggers the lifecycle event around the pop' do
+    called = false
+    Delayed::Worker.lifecycle.around(:work_queue_pop) do |queue, &cb|
+      expect(queue).to eq(subject)
+      expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*args).and_return(:job)
+      called = true
+      cb.call(queue)
+    end
+    job = subject.get_and_lock_next_available(*args)
+    expect(job).to eq(:job)
+    expect(called).to eq(true)
+  end
+end

data/spec/delayed/work_queue/parent_process_spec.rb

@@ -0,0 +1,159 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::WorkQueue::ParentProcess do
+  before :all do
+    Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
+  end
+
+  after :all do
+    Delayed.send(:remove_const, :Job)
+  end
+
+  after :each do
+    Delayed::Worker.lifecycle.reset!
+  end
+
+  let(:subject) { described_class.new }
+
+  it 'generates a server listening on a valid unix socket' do
+    server = subject.server
+    expect(server).to be_a(Delayed::WorkQueue::ParentProcess::Server)
+    expect(server.listen_socket.local_address.unix?).to be(true)
+    expect { server.listen_socket.accept_nonblock }.to raise_error(IO::WaitReadable)
+  end
+
+  it 'generates a client connected to the server unix socket' do
+    server = subject.server
+    client = subject.client
+    expect(client).to be_a(Delayed::WorkQueue::ParentProcess::Client)
+    expect(client.addrinfo.unix?).to be(true)
+    expect(client.addrinfo.unix_path).to eq(server.listen_socket.local_address.unix_path)
+  end
+
+  describe Delayed::WorkQueue::ParentProcess::Client do
+    let(:subject) { described_class.new(addrinfo) }
+    let(:addrinfo) { double('Addrinfo') }
+    let(:connection) { double('Socket') }
+    let(:args) { ["worker_name", "queue_name", 1, 2] }
+    let(:job) { Delayed::Job.new(locked_by: "worker_name") }
+
+    it 'marshals the given arguments to the server and returns the response' do
+      expect(addrinfo).to receive(:connect).once.and_return(connection)
+      expect(Marshal).to receive(:dump).with(args, connection).ordered
+      expect(Marshal).to receive(:load).with(connection).and_return(job).ordered
+      response = subject.get_and_lock_next_available(*args)
+      expect(response).to eq(job)
+    end
+
+    it 'returns nil and then reconnects on socket error' do
+      expect(addrinfo).to receive(:connect).once.and_return(connection)
+      expect(Marshal).to receive(:dump).and_raise(SystemCallError.new("failure"))
+      response = subject.get_and_lock_next_available(*args)
+      expect(response).to be_nil
+
+      expect(addrinfo).to receive(:connect).once.and_return(connection)
+      expect(Marshal).to receive(:dump).with(args, connection)
+      expect(Marshal).to receive(:load).with(connection).and_return(job)
+      response = subject.get_and_lock_next_available(*args)
+      expect(response).to eq(job)
+    end
+
+    it 'errors if the response is not a locked job' do
+      expect(addrinfo).to receive(:connect).once.and_return(connection)
+      expect(Marshal).to receive(:dump).with(args, connection)
+      expect(Marshal).to receive(:load).with(connection).and_return(:not_a_job)
+      expect { subject.get_and_lock_next_available(*args) }.to raise_error(Delayed::WorkQueue::ParentProcess::ProtocolError)
+    end
+
+    it 'errors if the response is a job not locked by this worker' do
+      expect(addrinfo).to receive(:connect).once.and_return(connection)
+      expect(Marshal).to receive(:dump).with(args, connection)
+      job.locked_by = "somebody_else"
+      expect(Marshal).to receive(:load).with(connection).and_return(job)
+      expect { subject.get_and_lock_next_available(*args) }.to raise_error(Delayed::WorkQueue::ParentProcess::ProtocolError)
+    end
+  end
+
+  describe Delayed::WorkQueue::ParentProcess::Server do
+    let(:subject) { described_class.new(listen_socket) }
+    let(:listen_socket) { Socket.unix_server_socket(Delayed::WorkQueue::ParentProcess.generate_socket_path) }
+    let(:args) { [1,2,3] }
+    let(:job) { :a_job }
+
+    it 'accepts new clients' do
+      client = Socket.unix(subject.listen_socket.local_address.unix_path)
+      expect { subject.run_once }.to change(subject, :connected_clients).by(1)
+    end
+
+    it 'queries the queue on client request' do
+      client = Socket.unix(subject.listen_socket.local_address.unix_path)
+      subject.run_once
+
+      expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*args).and_return(job)
+      Marshal.dump(args, client)
+      subject.run_once
+      expect(Marshal.load(client)).to eq(job)
+    end
+
+    it 'drops the client on i/o error' do
+      client = Socket.unix(subject.listen_socket.local_address.unix_path)
+      subject.run_once
+
+      Marshal.dump(args, client)
+
+      expect(Marshal).to receive(:load).and_raise(IOError.new("socket went away"))
+      expect { subject.run_once }.to change(subject, :connected_clients).by(-1)
+    end
+
+    it 'drops the client on timeout' do
+      client = Socket.unix(subject.listen_socket.local_address.unix_path)
+      subject.run_once
+
+      Marshal.dump(args, client)
+
+      expect(Marshal).to receive(:load).and_raise(Timeout::Error.new("socket timed out"))
+      expect(Timeout).to receive(:timeout).with(Delayed::Settings.parent_process_client_timeout).and_yield
+      expect { subject.run_once }.to change(subject, :connected_clients).by(-1)
+    end
+
+    it 'tracks when clients are idle' do
+      expect(subject.all_workers_idle?).to be(true)
+
+      client = Socket.unix(subject.listen_socket.local_address.unix_path)
+      subject.run_once
+      expect(subject.all_workers_idle?).to be(true)
+
+      expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*args).and_return(job)
+      Marshal.dump(args, client)
+      subject.run_once
+      expect(subject.all_workers_idle?).to be(false)
+
+      expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*args).and_return(nil)
+      Marshal.dump(args, client)
+      subject.run_once
+      expect(subject.all_workers_idle?).to be(true)
+    end
+
+    it 'triggers the lifecycle event around the pop' do
+      called = false
+      client = Socket.unix(subject.listen_socket.local_address.unix_path)
+      subject.run_once
+
+      Delayed::Worker.lifecycle.around(:work_queue_pop) do |queue, &cb|
+        expect(subject.all_workers_idle?).to be(true)
+        expect(queue).to eq(subject)
+        expect(Delayed::Job).to receive(:get_and_lock_next_available).with(*args).and_return(job)
+        called = true
+        res = cb.call(queue)
+        expect(subject.all_workers_idle?).to be(false)
+        res
+      end
+
+      Marshal.dump(args, client)
+      subject.run_once
+
+      expect(Marshal.load(client)).to eq(job)
+      expect(called).to eq(true)
+    end
+  end
+end

data/spec/delayed/worker_spec.rb

@@ -0,0 +1,16 @@
+require_relative "../spec_helper"
+
+module Delayed
+  describe Worker do
+    describe "#perform" do
+      it "fires off an error callback when a job raises an exception" do
+        fired = false
+        Worker.lifecycle.before(:error) {|worker, exception| fired = true}
+        worker = Worker.new
+        job = double(:last_error= => nil, attempts: 1, reschedule: nil)
+        worker.perform(job)
+        expect(fired).to be_truthy
+      end
+    end
+  end
+end

data/spec/gemfiles/32.gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+gemspec :path=>"../../"
+
+gem "rails", "~> 3.2.19"
+

data/spec/gemfiles/40.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec :path=>"../../"
+
+gem "rails", "~> 4.0.10"

data/spec/gemfiles/41.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec :path=>"../../"
+
+gem "rails", "~> 4.1.6"

data/spec/gemfiles/42.gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec :path=>"../../"
+
+gem "rails", "~> 4.2.5"

data/spec/migrate/20140924140513_add_story_table.rb

@@ -0,0 +1,7 @@
+class AddStoryTable < ActiveRecord::Migration
+  def change
+    create_table :stories do |table|
+      table.string :text
+    end
+  end
+end