inst-jobs 2.3.2 → 2.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. data/db/migrate/20101216224513_create_delayed_jobs.rb +9 -7
  3. data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +8 -13
  4. data/db/migrate/20110610213249_optimize_delayed_jobs.rb +8 -8
  5. data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +25 -25
  6. data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +4 -8
  7. data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +1 -3
  8. data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +11 -15
  9. data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +1 -1
  10. data/db/migrate/20120608191051_add_jobs_run_at_index.rb +2 -2
  11. data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +1 -1
  12. data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +2 -3
  13. data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +9 -13
  14. data/db/migrate/20151210162949_improve_max_concurrent.rb +4 -8
  15. data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +3 -2
  16. data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +13 -17
  17. data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +8 -8
  18. data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +72 -77
  19. data/db/migrate/20200825011002_add_strand_order_override.rb +93 -97
  20. data/db/migrate/20210809145804_add_n_strand_index.rb +12 -0
  21. data/db/migrate/20210812210128_add_singleton_column.rb +203 -0
  22. data/exe/inst_jobs +3 -2
  23. data/lib/delayed/backend/active_record.rb +182 -148
  24. data/lib/delayed/backend/base.rb +79 -74
  25. data/lib/delayed/batch.rb +11 -9
  26. data/lib/delayed/cli.rb +98 -84
  27. data/lib/delayed/core_ext/kernel.rb +4 -2
  28. data/lib/delayed/daemon.rb +70 -74
  29. data/lib/delayed/job_tracking.rb +26 -25
  30. data/lib/delayed/lifecycle.rb +27 -24
  31. data/lib/delayed/log_tailer.rb +17 -17
  32. data/lib/delayed/logging.rb +13 -16
  33. data/lib/delayed/message_sending.rb +42 -51
  34. data/lib/delayed/performable_method.rb +5 -7
  35. data/lib/delayed/periodic.rb +66 -65
  36. data/lib/delayed/plugin.rb +2 -4
  37. data/lib/delayed/pool.rb +198 -193
  38. data/lib/delayed/server/helpers.rb +6 -6
  39. data/lib/delayed/server.rb +51 -54
  40. data/lib/delayed/settings.rb +93 -81
  41. data/lib/delayed/testing.rb +21 -22
  42. data/lib/delayed/version.rb +1 -1
  43. data/lib/delayed/work_queue/in_process.rb +21 -18
  44. data/lib/delayed/work_queue/parent_process/client.rb +54 -55
  45. data/lib/delayed/work_queue/parent_process/server.rb +215 -209
  46. data/lib/delayed/work_queue/parent_process.rb +52 -53
  47. data/lib/delayed/worker/consul_health_check.rb +21 -19
  48. data/lib/delayed/worker/health_check.rb +21 -12
  49. data/lib/delayed/worker/null_health_check.rb +3 -1
  50. data/lib/delayed/worker/process_helper.rb +8 -9
  51. data/lib/delayed/worker.rb +271 -265
  52. data/lib/delayed/yaml_extensions.rb +12 -10
  53. data/lib/delayed_job.rb +37 -38
  54. data/lib/inst-jobs.rb +1 -1
  55. data/spec/active_record_job_spec.rb +128 -135
  56. data/spec/delayed/cli_spec.rb +7 -7
  57. data/spec/delayed/daemon_spec.rb +8 -8
  58. data/spec/delayed/message_sending_spec.rb +16 -9
  59. data/spec/delayed/periodic_spec.rb +13 -12
  60. data/spec/delayed/server_spec.rb +38 -38
  61. data/spec/delayed/settings_spec.rb +26 -25
  62. data/spec/delayed/work_queue/in_process_spec.rb +7 -7
  63. data/spec/delayed/work_queue/parent_process/client_spec.rb +15 -11
  64. data/spec/delayed/work_queue/parent_process/server_spec.rb +43 -40
  65. data/spec/delayed/work_queue/parent_process_spec.rb +21 -21
  66. data/spec/delayed/worker/consul_health_check_spec.rb +22 -22
  67. data/spec/delayed/worker/health_check_spec.rb +51 -49
  68. data/spec/delayed/worker_spec.rb +28 -25
  69. data/spec/gemfiles/52.gemfile +5 -3
  70. data/spec/gemfiles/52.gemfile.lock +240 -0
  71. data/spec/gemfiles/60.gemfile +5 -3
  72. data/spec/gemfiles/60.gemfile.lock +1 -1
  73. data/spec/gemfiles/61.gemfile +5 -3
  74. data/spec/sample_jobs.rb +45 -15
  75. data/spec/shared/delayed_batch.rb +74 -67
  76. data/spec/shared/delayed_method.rb +143 -102
  77. data/spec/shared/performable_method.rb +39 -38
  78. data/spec/shared/shared_backend.rb +534 -441
  79. data/spec/shared/testing.rb +14 -14
  80. data/spec/shared/worker.rb +155 -147
  81. data/spec/shared_jobs_specs.rb +13 -13
  82. data/spec/spec_helper.rb +43 -40
  83. metadata +75 -56
  84. data/lib/delayed/backend/redis/bulk_update.lua +0 -50
  85. data/lib/delayed/backend/redis/destroy_job.lua +0 -2
  86. data/lib/delayed/backend/redis/enqueue.lua +0 -29
  87. data/lib/delayed/backend/redis/fail_job.lua +0 -5
  88. data/lib/delayed/backend/redis/find_available.lua +0 -3
  89. data/lib/delayed/backend/redis/functions.rb +0 -59
  90. data/lib/delayed/backend/redis/get_and_lock_next_available.lua +0 -17
  91. data/lib/delayed/backend/redis/includes/jobs_common.lua +0 -203
  92. data/lib/delayed/backend/redis/job.rb +0 -528
  93. data/lib/delayed/backend/redis/set_running.lua +0 -5
  94. data/lib/delayed/backend/redis/tickle_strand.lua +0 -2
  95. data/spec/redis_job_spec.rb +0 -148
@@ -1,5 +0,0 @@
1
- local job_id = unpack(ARGV)
2
- local locked_at, queue, strand = unpack(redis.call('HMGET', Keys.job(job_id), 'locked_at', 'queue', 'strand'))
3
-
4
- remove_from_queues(job_id, queue, strand)
5
- redis.call('ZADD', Keys.running_jobs(), locked_at, job_id)
@@ -1,2 +0,0 @@
1
- local job_id, strand, now = unpack(ARGV)
2
- tickle_strand(job_id, strand, now)
@@ -1,148 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require File.expand_path("../spec_helper", __FILE__)
4
-
5
- describe 'Delayed::Backend::Redis::Job' do
6
- before :all do
7
- Delayed.select_backend(Delayed::Backend::Redis::Job)
8
- end
9
-
10
- before do
11
- Delayed::Testing.clear_all!
12
- end
13
-
14
- include_examples 'a delayed_jobs implementation'
15
-
16
- describe "tickle_strand" do
17
- it "should continue trying to tickle until the strand is empty" do
18
- jobs = []
19
- 3.times { jobs << "test".delay(ignore_transaction: true, strand: "s1").to_s }
20
- job = "test".delay(strand: "s1", ignore_transaction: true).to_s
21
- # manually delete the first jobs, bypassing the strand book-keeping
22
- jobs.each { |j| Delayed::Job.redis.del(Delayed::Job::Keys::JOB[j.id]) }
23
- Delayed::Job.redis.llen(Delayed::Job::Keys::STRAND['s1']).should == 4
24
- job.destroy
25
- Delayed::Job.redis.llen(Delayed::Job::Keys::STRAND['s1']).should == 0
26
- end
27
-
28
- it "should tickle until it finds an existing job" do
29
- jobs = []
30
- 3.times { jobs << "test".delay(strand: "s1", ignore_transaction: true).to_s }
31
- job = "test".delay(strand: "s1", ignore_transaction: true).to_s
32
- # manually delete the first jobs, bypassing the strand book-keeping
33
- jobs[0...-1].each { |j| Delayed::Job.redis.del(Delayed::Job::Keys::JOB[j.id]) }
34
- Delayed::Job.redis.llen(Delayed::Job::Keys::STRAND['s1']).should == 4
35
- jobs[-1].destroy
36
- Delayed::Job.redis.lrange(Delayed::Job::Keys::STRAND['s1'], 0, -1).should == [job.id]
37
- found = [Delayed::Job.get_and_lock_next_available('test worker'),
38
- Delayed::Job.get_and_lock_next_available('test worker')]
39
- found.should =~ [job, nil]
40
- end
41
- end
42
-
43
- describe "missing jobs in queues" do
44
- before do
45
- @job = "test".delay(ignore_transaction: true).to_s
46
- @job2 = "test".delay(ignore_transaction: true).to_s
47
- # manually delete the job from redis
48
- Delayed::Job.redis.del(Delayed::Job::Keys::JOB[@job.id])
49
- end
50
-
51
- it "should discard when trying to lock" do
52
- found = [Delayed::Job.get_and_lock_next_available("test worker"),
53
- Delayed::Job.get_and_lock_next_available("test worker")]
54
- found.should =~ [@job2, nil]
55
- end
56
-
57
- it "should filter for find_available" do
58
- found = [Delayed::Job.find_available(1),
59
- Delayed::Job.find_available(1)]
60
- found.should be_include([@job2])
61
- end
62
- end
63
-
64
- describe "delay" do
65
- it "should schedule job on transaction commit" do
66
- before_count = Delayed::Job.jobs_count(:current)
67
- ActiveRecord::Base.transaction do
68
- job = "string".delay.reverse
69
- job.should be_nil
70
- Delayed::Job.jobs_count(:current).should == before_count
71
- end
72
- Delayed::Job.jobs_count(:current) == before_count + 1
73
- end
74
- end
75
-
76
- context 'n_strand' do
77
- it "should default to 1" do
78
- expect(Delayed::Job).to receive(:rand).never
79
- job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
80
- job.strand.should == "njobs"
81
- end
82
-
83
- it "should pick a strand randomly out of N" do
84
- change_setting(Delayed::Settings, :num_strands, ->(strand_name) { expect(strand_name).to eql "njobs"; "3" }) do
85
- expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
86
- job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
87
- job.strand.should == "njobs:2"
88
- end
89
- end
90
-
91
- context "with two parameters" do
92
- it "should use the first param as the setting to read" do
93
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
94
- job.strand.should == "njobs/123"
95
- change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
96
- case strand_name
97
- when "njobs"; 3
98
- else nil
99
- end
100
- }) do
101
- expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
102
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
103
- job.strand.should == "njobs/123:2"
104
- end
105
- end
106
-
107
- it "should allow overridding the setting based on the second param" do
108
- change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
109
- case strand_name
110
- when "njobs/123"; 5
111
- else nil
112
- end
113
- }) do
114
- expect(Delayed::Job).to receive(:rand).with(5).and_return(3)
115
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
116
- job.strand.should == "njobs/123:4"
117
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
118
- job.strand.should == "njobs/456"
119
- end
120
-
121
- change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
122
- case strand_name
123
- when "njobs/123"; 5
124
- when "njobs"; 3
125
- else nil
126
- end
127
- }) do
128
- expect(Delayed::Job).to receive(:rand).with(5).and_return(2)
129
- expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
130
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
131
- job.strand.should == "njobs/123:3"
132
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
133
- job.strand.should == "njobs/456:2"
134
- end
135
- end
136
- end
137
- end
138
-
139
- it "allows the API for fetching multiple jobs at once" do
140
- jobs = 3.times.map { Delayed::Job.create :payload_object => SimpleJob.new }
141
- locked_jobs = Delayed::Job.get_and_lock_next_available(['worker1', 'worker2'])
142
- locked_jobs.length.should == 1
143
- locked_jobs.keys.should == ['worker1']
144
- jobs.map(&:id).should be_include(locked_jobs.values.first.id)
145
- jobs.map { |j| Delayed::Job.find(j.id).locked_by }.compact.should == ['worker1']
146
- end
147
-
148
- end