inst-jobs 2.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/db/migrate/20101216224513_create_delayed_jobs.rb +9 -7
- data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +8 -13
- data/db/migrate/20110610213249_optimize_delayed_jobs.rb +8 -8
- data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +25 -25
- data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +4 -8
- data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +1 -3
- data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +11 -15
- data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +1 -1
- data/db/migrate/20120608191051_add_jobs_run_at_index.rb +2 -2
- data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +1 -1
- data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +2 -3
- data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +9 -13
- data/db/migrate/20151210162949_improve_max_concurrent.rb +4 -8
- data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +3 -2
- data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +13 -17
- data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +8 -8
- data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +72 -77
- data/db/migrate/20200825011002_add_strand_order_override.rb +93 -97
- data/db/migrate/20210809145804_add_n_strand_index.rb +12 -0
- data/db/migrate/20210812210128_add_singleton_column.rb +200 -0
- data/db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb +27 -0
- data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb +56 -0
- data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb +27 -0
- data/db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb +137 -0
- data/db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb +171 -0
- data/db/migrate/20211220112800_fix_singleton_race_condition_insert.rb +59 -0
- data/db/migrate/20211220113000_fix_singleton_race_condition_delete.rb +207 -0
- data/db/migrate/20220127091200_fix_singleton_unique_constraint.rb +31 -0
- data/db/migrate/20220128084800_update_insert_trigger_for_singleton_unique_constraint_change.rb +60 -0
- data/db/migrate/20220128084900_update_delete_trigger_for_singleton_unique_constraint_change.rb +209 -0
- data/db/migrate/20220203063200_remove_old_singleton_index.rb +31 -0
- data/db/migrate/20220328152900_add_failed_jobs_indicies.rb +12 -0
- data/exe/inst_jobs +3 -2
- data/lib/delayed/backend/active_record.rb +226 -168
- data/lib/delayed/backend/base.rb +119 -72
- data/lib/delayed/batch.rb +11 -9
- data/lib/delayed/cli.rb +98 -84
- data/lib/delayed/core_ext/kernel.rb +4 -2
- data/lib/delayed/daemon.rb +70 -74
- data/lib/delayed/job_tracking.rb +26 -25
- data/lib/delayed/lifecycle.rb +28 -23
- data/lib/delayed/log_tailer.rb +17 -17
- data/lib/delayed/logging.rb +13 -16
- data/lib/delayed/message_sending.rb +43 -52
- data/lib/delayed/performable_method.rb +6 -8
- data/lib/delayed/periodic.rb +72 -68
- data/lib/delayed/plugin.rb +2 -4
- data/lib/delayed/pool.rb +205 -168
- data/lib/delayed/rails_reloader_plugin.rb +30 -0
- data/lib/delayed/server/helpers.rb +6 -6
- data/lib/delayed/server.rb +51 -54
- data/lib/delayed/settings.rb +96 -81
- data/lib/delayed/testing.rb +21 -22
- data/lib/delayed/version.rb +1 -1
- data/lib/delayed/work_queue/in_process.rb +21 -17
- data/lib/delayed/work_queue/parent_process/client.rb +55 -53
- data/lib/delayed/work_queue/parent_process/server.rb +245 -207
- data/lib/delayed/work_queue/parent_process.rb +52 -53
- data/lib/delayed/worker/consul_health_check.rb +32 -33
- data/lib/delayed/worker/health_check.rb +35 -27
- data/lib/delayed/worker/null_health_check.rb +3 -1
- data/lib/delayed/worker/process_helper.rb +11 -12
- data/lib/delayed/worker.rb +257 -244
- data/lib/delayed/yaml_extensions.rb +12 -10
- data/lib/delayed_job.rb +37 -37
- data/lib/inst-jobs.rb +1 -1
- data/spec/active_record_job_spec.rb +152 -139
- data/spec/delayed/cli_spec.rb +7 -7
- data/spec/delayed/daemon_spec.rb +10 -9
- data/spec/delayed/message_sending_spec.rb +16 -9
- data/spec/delayed/periodic_spec.rb +14 -21
- data/spec/delayed/server_spec.rb +38 -38
- data/spec/delayed/settings_spec.rb +26 -25
- data/spec/delayed/work_queue/in_process_spec.rb +8 -9
- data/spec/delayed/work_queue/parent_process/client_spec.rb +17 -12
- data/spec/delayed/work_queue/parent_process/server_spec.rb +118 -42
- data/spec/delayed/work_queue/parent_process_spec.rb +21 -23
- data/spec/delayed/worker/consul_health_check_spec.rb +37 -50
- data/spec/delayed/worker/health_check_spec.rb +60 -52
- data/spec/delayed/worker_spec.rb +53 -24
- data/spec/sample_jobs.rb +45 -15
- data/spec/shared/delayed_batch.rb +74 -67
- data/spec/shared/delayed_method.rb +143 -102
- data/spec/shared/performable_method.rb +39 -38
- data/spec/shared/shared_backend.rb +801 -440
- data/spec/shared/testing.rb +14 -14
- data/spec/shared/worker.rb +157 -149
- data/spec/shared_jobs_specs.rb +13 -13
- data/spec/spec_helper.rb +57 -56
- metadata +183 -103
- data/lib/delayed/backend/redis/bulk_update.lua +0 -50
- data/lib/delayed/backend/redis/destroy_job.lua +0 -2
- data/lib/delayed/backend/redis/enqueue.lua +0 -29
- data/lib/delayed/backend/redis/fail_job.lua +0 -5
- data/lib/delayed/backend/redis/find_available.lua +0 -3
- data/lib/delayed/backend/redis/functions.rb +0 -59
- data/lib/delayed/backend/redis/get_and_lock_next_available.lua +0 -17
- data/lib/delayed/backend/redis/includes/jobs_common.lua +0 -203
- data/lib/delayed/backend/redis/job.rb +0 -535
- data/lib/delayed/backend/redis/set_running.lua +0 -5
- data/lib/delayed/backend/redis/tickle_strand.lua +0 -2
- data/spec/gemfiles/42.gemfile +0 -7
- data/spec/gemfiles/50.gemfile +0 -7
- data/spec/gemfiles/51.gemfile +0 -7
- data/spec/gemfiles/52.gemfile +0 -7
- data/spec/gemfiles/60.gemfile +0 -7
- data/spec/redis_job_spec.rb +0 -148
    
        data/spec/redis_job_spec.rb
    DELETED
    
    | @@ -1,148 +0,0 @@ | |
| 1 | 
            -
            # frozen_string_literal: true
         | 
| 2 | 
            -
             | 
| 3 | 
            -
            require File.expand_path("../spec_helper", __FILE__)
         | 
| 4 | 
            -
             | 
| 5 | 
            -
            describe 'Delayed::Backend::Redis::Job' do
         | 
| 6 | 
            -
              before :all do
         | 
| 7 | 
            -
                Delayed.select_backend(Delayed::Backend::Redis::Job)
         | 
| 8 | 
            -
              end
         | 
| 9 | 
            -
             | 
| 10 | 
            -
              before do
         | 
| 11 | 
            -
                Delayed::Testing.clear_all!
         | 
| 12 | 
            -
              end
         | 
| 13 | 
            -
             | 
| 14 | 
            -
              include_examples 'a delayed_jobs implementation'
         | 
| 15 | 
            -
             | 
| 16 | 
            -
              describe "tickle_strand" do
         | 
| 17 | 
            -
                it "should continue trying to tickle until the strand is empty" do
         | 
| 18 | 
            -
                  jobs = []
         | 
| 19 | 
            -
                  3.times { jobs << "test".delay(ignore_transaction: true, strand: "s1").to_s }
         | 
| 20 | 
            -
                  job = "test".delay(strand: "s1", ignore_transaction: true).to_s
         | 
| 21 | 
            -
                  # manually delete the first jobs, bypassing the strand book-keeping
         | 
| 22 | 
            -
                  jobs.each { |j| Delayed::Job.redis.del(Delayed::Job::Keys::JOB[j.id]) }
         | 
| 23 | 
            -
                  Delayed::Job.redis.llen(Delayed::Job::Keys::STRAND['s1']).should == 4
         | 
| 24 | 
            -
                  job.destroy
         | 
| 25 | 
            -
                  Delayed::Job.redis.llen(Delayed::Job::Keys::STRAND['s1']).should == 0
         | 
| 26 | 
            -
                end
         | 
| 27 | 
            -
             | 
| 28 | 
            -
                it "should tickle until it finds an existing job" do
         | 
| 29 | 
            -
                  jobs = []
         | 
| 30 | 
            -
                  3.times { jobs << "test".delay(strand: "s1", ignore_transaction: true).to_s }
         | 
| 31 | 
            -
                  job = "test".delay(strand: "s1", ignore_transaction: true).to_s
         | 
| 32 | 
            -
                  # manually delete the first jobs, bypassing the strand book-keeping
         | 
| 33 | 
            -
                  jobs[0...-1].each { |j| Delayed::Job.redis.del(Delayed::Job::Keys::JOB[j.id]) }
         | 
| 34 | 
            -
                  Delayed::Job.redis.llen(Delayed::Job::Keys::STRAND['s1']).should == 4
         | 
| 35 | 
            -
                  jobs[-1].destroy
         | 
| 36 | 
            -
                  Delayed::Job.redis.lrange(Delayed::Job::Keys::STRAND['s1'], 0, -1).should == [job.id]
         | 
| 37 | 
            -
                  found = [Delayed::Job.get_and_lock_next_available('test worker'),
         | 
| 38 | 
            -
                           Delayed::Job.get_and_lock_next_available('test worker')]
         | 
| 39 | 
            -
                  found.should =~ [job, nil]
         | 
| 40 | 
            -
                end
         | 
| 41 | 
            -
              end
         | 
| 42 | 
            -
             | 
| 43 | 
            -
              describe "missing jobs in queues" do
         | 
| 44 | 
            -
                before do
         | 
| 45 | 
            -
                  @job = "test".delay(ignore_transaction: true).to_s
         | 
| 46 | 
            -
                  @job2 = "test".delay(ignore_transaction: true).to_s
         | 
| 47 | 
            -
                  # manually delete the job from redis
         | 
| 48 | 
            -
                  Delayed::Job.redis.del(Delayed::Job::Keys::JOB[@job.id])
         | 
| 49 | 
            -
                end
         | 
| 50 | 
            -
             | 
| 51 | 
            -
                it "should discard when trying to lock" do
         | 
| 52 | 
            -
                  found = [Delayed::Job.get_and_lock_next_available("test worker"),
         | 
| 53 | 
            -
                           Delayed::Job.get_and_lock_next_available("test worker")]
         | 
| 54 | 
            -
                  found.should =~ [@job2, nil]
         | 
| 55 | 
            -
                end
         | 
| 56 | 
            -
             | 
| 57 | 
            -
                it "should filter for find_available" do
         | 
| 58 | 
            -
                  found = [Delayed::Job.find_available(1),
         | 
| 59 | 
            -
                           Delayed::Job.find_available(1)]
         | 
| 60 | 
            -
                  found.should be_include([@job2])
         | 
| 61 | 
            -
                end
         | 
| 62 | 
            -
              end
         | 
| 63 | 
            -
             | 
| 64 | 
            -
              describe "delay" do
         | 
| 65 | 
            -
                it "should schedule job on transaction commit" do
         | 
| 66 | 
            -
                  before_count = Delayed::Job.jobs_count(:current)
         | 
| 67 | 
            -
                  ActiveRecord::Base.transaction do
         | 
| 68 | 
            -
                    job = "string".delay.reverse
         | 
| 69 | 
            -
                    job.should be_nil
         | 
| 70 | 
            -
                    Delayed::Job.jobs_count(:current).should == before_count
         | 
| 71 | 
            -
                  end
         | 
| 72 | 
            -
                  Delayed::Job.jobs_count(:current) == before_count + 1
         | 
| 73 | 
            -
                end
         | 
| 74 | 
            -
              end
         | 
| 75 | 
            -
             | 
| 76 | 
            -
              context 'n_strand' do
         | 
| 77 | 
            -
                it "should default to 1" do
         | 
| 78 | 
            -
                  expect(Delayed::Job).to receive(:rand).never
         | 
| 79 | 
            -
                  job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
         | 
| 80 | 
            -
                  job.strand.should == "njobs"
         | 
| 81 | 
            -
                end
         | 
| 82 | 
            -
             | 
| 83 | 
            -
                it "should pick a strand randomly out of N" do
         | 
| 84 | 
            -
                  change_setting(Delayed::Settings, :num_strands, ->(strand_name) { expect(strand_name).to eql "njobs"; "3" }) do
         | 
| 85 | 
            -
                    expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
         | 
| 86 | 
            -
                    job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
         | 
| 87 | 
            -
                    job.strand.should == "njobs:2"
         | 
| 88 | 
            -
                  end
         | 
| 89 | 
            -
                end
         | 
| 90 | 
            -
             | 
| 91 | 
            -
                context "with two parameters" do
         | 
| 92 | 
            -
                  it "should use the first param as the setting to read" do
         | 
| 93 | 
            -
                    job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
         | 
| 94 | 
            -
                    job.strand.should == "njobs/123"
         | 
| 95 | 
            -
                    change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
         | 
| 96 | 
            -
                      case strand_name
         | 
| 97 | 
            -
                      when "njobs"; 3
         | 
| 98 | 
            -
                      else nil
         | 
| 99 | 
            -
                      end
         | 
| 100 | 
            -
                    }) do
         | 
| 101 | 
            -
                      expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
         | 
| 102 | 
            -
                      job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
         | 
| 103 | 
            -
                      job.strand.should == "njobs/123:2"
         | 
| 104 | 
            -
                    end
         | 
| 105 | 
            -
                  end
         | 
| 106 | 
            -
             | 
| 107 | 
            -
                  it "should allow overridding the setting based on the second param" do
         | 
| 108 | 
            -
                    change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
         | 
| 109 | 
            -
                      case strand_name
         | 
| 110 | 
            -
                      when "njobs/123"; 5
         | 
| 111 | 
            -
                      else nil
         | 
| 112 | 
            -
                      end
         | 
| 113 | 
            -
                    }) do
         | 
| 114 | 
            -
                      expect(Delayed::Job).to receive(:rand).with(5).and_return(3)
         | 
| 115 | 
            -
                      job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
         | 
| 116 | 
            -
                      job.strand.should == "njobs/123:4"
         | 
| 117 | 
            -
                      job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
         | 
| 118 | 
            -
                      job.strand.should == "njobs/456"
         | 
| 119 | 
            -
                    end
         | 
| 120 | 
            -
             | 
| 121 | 
            -
                    change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
         | 
| 122 | 
            -
                      case strand_name
         | 
| 123 | 
            -
                      when "njobs/123"; 5
         | 
| 124 | 
            -
                      when "njobs"; 3
         | 
| 125 | 
            -
                      else nil
         | 
| 126 | 
            -
                      end
         | 
| 127 | 
            -
                    }) do
         | 
| 128 | 
            -
                      expect(Delayed::Job).to receive(:rand).with(5).and_return(2)
         | 
| 129 | 
            -
                      expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
         | 
| 130 | 
            -
                      job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
         | 
| 131 | 
            -
                      job.strand.should == "njobs/123:3"
         | 
| 132 | 
            -
                      job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
         | 
| 133 | 
            -
                      job.strand.should == "njobs/456:2"
         | 
| 134 | 
            -
                    end
         | 
| 135 | 
            -
                  end
         | 
| 136 | 
            -
                end
         | 
| 137 | 
            -
              end
         | 
| 138 | 
            -
             | 
| 139 | 
            -
              it "allows the API for fetching multiple jobs at once" do
         | 
| 140 | 
            -
                jobs = 3.times.map { Delayed::Job.create :payload_object => SimpleJob.new }
         | 
| 141 | 
            -
                locked_jobs = Delayed::Job.get_and_lock_next_available(['worker1', 'worker2'])
         | 
| 142 | 
            -
                locked_jobs.length.should == 1
         | 
| 143 | 
            -
                locked_jobs.keys.should == ['worker1']
         | 
| 144 | 
            -
                jobs.map(&:id).should be_include(locked_jobs.values.first.id)
         | 
| 145 | 
            -
                jobs.map { |j| Delayed::Job.find(j.id).locked_by }.compact.should == ['worker1']
         | 
| 146 | 
            -
              end
         | 
| 147 | 
            -
             | 
| 148 | 
            -
            end
         |