inst-jobs 2.0.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/db/migrate/20101216224513_create_delayed_jobs.rb +9 -7
  3. data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +8 -13
  4. data/db/migrate/20110610213249_optimize_delayed_jobs.rb +8 -8
  5. data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +25 -25
  6. data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +4 -8
  7. data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +1 -3
  8. data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +11 -15
  9. data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +1 -1
  10. data/db/migrate/20120608191051_add_jobs_run_at_index.rb +2 -2
  11. data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +1 -1
  12. data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +2 -3
  13. data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +9 -13
  14. data/db/migrate/20151210162949_improve_max_concurrent.rb +4 -8
  15. data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +3 -2
  16. data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +13 -17
  17. data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +8 -8
  18. data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +72 -77
  19. data/db/migrate/20200825011002_add_strand_order_override.rb +93 -97
  20. data/db/migrate/20210809145804_add_n_strand_index.rb +12 -0
  21. data/db/migrate/20210812210128_add_singleton_column.rb +200 -0
  22. data/db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb +27 -0
  23. data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb +56 -0
  24. data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb +27 -0
  25. data/exe/inst_jobs +3 -2
  26. data/lib/delayed/backend/active_record.rb +211 -168
  27. data/lib/delayed/backend/base.rb +110 -72
  28. data/lib/delayed/batch.rb +11 -9
  29. data/lib/delayed/cli.rb +98 -84
  30. data/lib/delayed/core_ext/kernel.rb +4 -2
  31. data/lib/delayed/daemon.rb +70 -74
  32. data/lib/delayed/job_tracking.rb +26 -25
  33. data/lib/delayed/lifecycle.rb +27 -23
  34. data/lib/delayed/log_tailer.rb +17 -17
  35. data/lib/delayed/logging.rb +13 -16
  36. data/lib/delayed/message_sending.rb +43 -52
  37. data/lib/delayed/performable_method.rb +6 -8
  38. data/lib/delayed/periodic.rb +72 -68
  39. data/lib/delayed/plugin.rb +2 -4
  40. data/lib/delayed/pool.rb +205 -168
  41. data/lib/delayed/server/helpers.rb +6 -6
  42. data/lib/delayed/server.rb +51 -54
  43. data/lib/delayed/settings.rb +94 -81
  44. data/lib/delayed/testing.rb +21 -22
  45. data/lib/delayed/version.rb +1 -1
  46. data/lib/delayed/work_queue/in_process.rb +21 -17
  47. data/lib/delayed/work_queue/parent_process/client.rb +55 -53
  48. data/lib/delayed/work_queue/parent_process/server.rb +245 -207
  49. data/lib/delayed/work_queue/parent_process.rb +52 -53
  50. data/lib/delayed/worker/consul_health_check.rb +32 -33
  51. data/lib/delayed/worker/health_check.rb +34 -26
  52. data/lib/delayed/worker/null_health_check.rb +3 -1
  53. data/lib/delayed/worker/process_helper.rb +8 -9
  54. data/lib/delayed/worker.rb +272 -241
  55. data/lib/delayed/yaml_extensions.rb +12 -10
  56. data/lib/delayed_job.rb +37 -37
  57. data/lib/inst-jobs.rb +1 -1
  58. data/spec/active_record_job_spec.rb +143 -139
  59. data/spec/delayed/cli_spec.rb +7 -7
  60. data/spec/delayed/daemon_spec.rb +10 -9
  61. data/spec/delayed/message_sending_spec.rb +16 -9
  62. data/spec/delayed/periodic_spec.rb +14 -21
  63. data/spec/delayed/server_spec.rb +38 -38
  64. data/spec/delayed/settings_spec.rb +26 -25
  65. data/spec/delayed/work_queue/in_process_spec.rb +7 -8
  66. data/spec/delayed/work_queue/parent_process/client_spec.rb +17 -12
  67. data/spec/delayed/work_queue/parent_process/server_spec.rb +117 -41
  68. data/spec/delayed/work_queue/parent_process_spec.rb +21 -23
  69. data/spec/delayed/worker/consul_health_check_spec.rb +37 -50
  70. data/spec/delayed/worker/health_check_spec.rb +60 -52
  71. data/spec/delayed/worker_spec.rb +44 -21
  72. data/spec/sample_jobs.rb +45 -15
  73. data/spec/shared/delayed_batch.rb +74 -67
  74. data/spec/shared/delayed_method.rb +143 -102
  75. data/spec/shared/performable_method.rb +39 -38
  76. data/spec/shared/shared_backend.rb +550 -437
  77. data/spec/shared/testing.rb +14 -14
  78. data/spec/shared/worker.rb +156 -148
  79. data/spec/shared_jobs_specs.rb +13 -13
  80. data/spec/spec_helper.rb +53 -55
  81. metadata +148 -82
  82. data/lib/delayed/backend/redis/bulk_update.lua +0 -50
  83. data/lib/delayed/backend/redis/destroy_job.lua +0 -2
  84. data/lib/delayed/backend/redis/enqueue.lua +0 -29
  85. data/lib/delayed/backend/redis/fail_job.lua +0 -5
  86. data/lib/delayed/backend/redis/find_available.lua +0 -3
  87. data/lib/delayed/backend/redis/functions.rb +0 -59
  88. data/lib/delayed/backend/redis/get_and_lock_next_available.lua +0 -17
  89. data/lib/delayed/backend/redis/includes/jobs_common.lua +0 -203
  90. data/lib/delayed/backend/redis/job.rb +0 -535
  91. data/lib/delayed/backend/redis/set_running.lua +0 -5
  92. data/lib/delayed/backend/redis/tickle_strand.lua +0 -2
  93. data/spec/gemfiles/42.gemfile +0 -7
  94. data/spec/gemfiles/50.gemfile +0 -7
  95. data/spec/gemfiles/51.gemfile +0 -7
  96. data/spec/gemfiles/52.gemfile +0 -7
  97. data/spec/gemfiles/60.gemfile +0 -7
  98. data/spec/redis_job_spec.rb +0 -148
@@ -1,148 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require File.expand_path("../spec_helper", __FILE__)
4
-
5
- describe 'Delayed::Backend::Redis::Job' do
6
- before :all do
7
- Delayed.select_backend(Delayed::Backend::Redis::Job)
8
- end
9
-
10
- before do
11
- Delayed::Testing.clear_all!
12
- end
13
-
14
- include_examples 'a delayed_jobs implementation'
15
-
16
- describe "tickle_strand" do
17
- it "should continue trying to tickle until the strand is empty" do
18
- jobs = []
19
- 3.times { jobs << "test".delay(ignore_transaction: true, strand: "s1").to_s }
20
- job = "test".delay(strand: "s1", ignore_transaction: true).to_s
21
- # manually delete the first jobs, bypassing the strand book-keeping
22
- jobs.each { |j| Delayed::Job.redis.del(Delayed::Job::Keys::JOB[j.id]) }
23
- Delayed::Job.redis.llen(Delayed::Job::Keys::STRAND['s1']).should == 4
24
- job.destroy
25
- Delayed::Job.redis.llen(Delayed::Job::Keys::STRAND['s1']).should == 0
26
- end
27
-
28
- it "should tickle until it finds an existing job" do
29
- jobs = []
30
- 3.times { jobs << "test".delay(strand: "s1", ignore_transaction: true).to_s }
31
- job = "test".delay(strand: "s1", ignore_transaction: true).to_s
32
- # manually delete the first jobs, bypassing the strand book-keeping
33
- jobs[0...-1].each { |j| Delayed::Job.redis.del(Delayed::Job::Keys::JOB[j.id]) }
34
- Delayed::Job.redis.llen(Delayed::Job::Keys::STRAND['s1']).should == 4
35
- jobs[-1].destroy
36
- Delayed::Job.redis.lrange(Delayed::Job::Keys::STRAND['s1'], 0, -1).should == [job.id]
37
- found = [Delayed::Job.get_and_lock_next_available('test worker'),
38
- Delayed::Job.get_and_lock_next_available('test worker')]
39
- found.should =~ [job, nil]
40
- end
41
- end
42
-
43
- describe "missing jobs in queues" do
44
- before do
45
- @job = "test".delay(ignore_transaction: true).to_s
46
- @job2 = "test".delay(ignore_transaction: true).to_s
47
- # manually delete the job from redis
48
- Delayed::Job.redis.del(Delayed::Job::Keys::JOB[@job.id])
49
- end
50
-
51
- it "should discard when trying to lock" do
52
- found = [Delayed::Job.get_and_lock_next_available("test worker"),
53
- Delayed::Job.get_and_lock_next_available("test worker")]
54
- found.should =~ [@job2, nil]
55
- end
56
-
57
- it "should filter for find_available" do
58
- found = [Delayed::Job.find_available(1),
59
- Delayed::Job.find_available(1)]
60
- found.should be_include([@job2])
61
- end
62
- end
63
-
64
- describe "delay" do
65
- it "should schedule job on transaction commit" do
66
- before_count = Delayed::Job.jobs_count(:current)
67
- ActiveRecord::Base.transaction do
68
- job = "string".delay.reverse
69
- job.should be_nil
70
- Delayed::Job.jobs_count(:current).should == before_count
71
- end
72
- Delayed::Job.jobs_count(:current) == before_count + 1
73
- end
74
- end
75
-
76
- context 'n_strand' do
77
- it "should default to 1" do
78
- expect(Delayed::Job).to receive(:rand).never
79
- job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
80
- job.strand.should == "njobs"
81
- end
82
-
83
- it "should pick a strand randomly out of N" do
84
- change_setting(Delayed::Settings, :num_strands, ->(strand_name) { expect(strand_name).to eql "njobs"; "3" }) do
85
- expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
86
- job = Delayed::Job.enqueue(SimpleJob.new, :n_strand => 'njobs')
87
- job.strand.should == "njobs:2"
88
- end
89
- end
90
-
91
- context "with two parameters" do
92
- it "should use the first param as the setting to read" do
93
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
94
- job.strand.should == "njobs/123"
95
- change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
96
- case strand_name
97
- when "njobs"; 3
98
- else nil
99
- end
100
- }) do
101
- expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
102
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
103
- job.strand.should == "njobs/123:2"
104
- end
105
- end
106
-
107
- it "should allow overridding the setting based on the second param" do
108
- change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
109
- case strand_name
110
- when "njobs/123"; 5
111
- else nil
112
- end
113
- }) do
114
- expect(Delayed::Job).to receive(:rand).with(5).and_return(3)
115
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
116
- job.strand.should == "njobs/123:4"
117
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
118
- job.strand.should == "njobs/456"
119
- end
120
-
121
- change_setting(Delayed::Settings, :num_strands, ->(strand_name) {
122
- case strand_name
123
- when "njobs/123"; 5
124
- when "njobs"; 3
125
- else nil
126
- end
127
- }) do
128
- expect(Delayed::Job).to receive(:rand).with(5).and_return(2)
129
- expect(Delayed::Job).to receive(:rand).with(3).and_return(1)
130
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "123"])
131
- job.strand.should == "njobs/123:3"
132
- job = Delayed::Job.enqueue(SimpleJob.new, n_strand: ["njobs", "456"])
133
- job.strand.should == "njobs/456:2"
134
- end
135
- end
136
- end
137
- end
138
-
139
- it "allows the API for fetching multiple jobs at once" do
140
- jobs = 3.times.map { Delayed::Job.create :payload_object => SimpleJob.new }
141
- locked_jobs = Delayed::Job.get_and_lock_next_available(['worker1', 'worker2'])
142
- locked_jobs.length.should == 1
143
- locked_jobs.keys.should == ['worker1']
144
- jobs.map(&:id).should be_include(locked_jobs.values.first.id)
145
- jobs.map { |j| Delayed::Job.find(j.id).locked_by }.compact.should == ['worker1']
146
- end
147
-
148
- end