que 0.14.3 → 1.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -0
  3. data/CHANGELOG.md +108 -14
  4. data/LICENSE.txt +1 -1
  5. data/README.md +49 -45
  6. data/bin/command_line_interface.rb +239 -0
  7. data/bin/que +8 -82
  8. data/docs/README.md +2 -0
  9. data/docs/active_job.md +6 -0
  10. data/docs/advanced_setup.md +7 -64
  11. data/docs/command_line_interface.md +45 -0
  12. data/docs/error_handling.md +65 -18
  13. data/docs/inspecting_the_queue.md +30 -80
  14. data/docs/job_helper_methods.md +27 -0
  15. data/docs/logging.md +3 -22
  16. data/docs/managing_workers.md +6 -61
  17. data/docs/middleware.md +15 -0
  18. data/docs/migrating.md +4 -7
  19. data/docs/multiple_queues.md +8 -4
  20. data/docs/shutting_down_safely.md +1 -1
  21. data/docs/using_plain_connections.md +39 -15
  22. data/docs/using_sequel.md +5 -3
  23. data/docs/writing_reliable_jobs.md +15 -24
  24. data/lib/que.rb +98 -182
  25. data/lib/que/active_job/extensions.rb +97 -0
  26. data/lib/que/active_record/connection.rb +51 -0
  27. data/lib/que/active_record/model.rb +48 -0
  28. data/lib/que/connection.rb +179 -0
  29. data/lib/que/connection_pool.rb +78 -0
  30. data/lib/que/job.rb +107 -156
  31. data/lib/que/job_cache.rb +240 -0
  32. data/lib/que/job_methods.rb +168 -0
  33. data/lib/que/listener.rb +176 -0
  34. data/lib/que/locker.rb +466 -0
  35. data/lib/que/metajob.rb +47 -0
  36. data/lib/que/migrations.rb +24 -17
  37. data/lib/que/migrations/4/down.sql +48 -0
  38. data/lib/que/migrations/4/up.sql +265 -0
  39. data/lib/que/poller.rb +267 -0
  40. data/lib/que/rails/railtie.rb +14 -0
  41. data/lib/que/result_queue.rb +35 -0
  42. data/lib/que/sequel/model.rb +51 -0
  43. data/lib/que/utils/assertions.rb +62 -0
  44. data/lib/que/utils/constantization.rb +19 -0
  45. data/lib/que/utils/error_notification.rb +68 -0
  46. data/lib/que/utils/freeze.rb +20 -0
  47. data/lib/que/utils/introspection.rb +50 -0
  48. data/lib/que/utils/json_serialization.rb +21 -0
  49. data/lib/que/utils/logging.rb +78 -0
  50. data/lib/que/utils/middleware.rb +33 -0
  51. data/lib/que/utils/queue_management.rb +18 -0
  52. data/lib/que/utils/transactions.rb +34 -0
  53. data/lib/que/version.rb +1 -1
  54. data/lib/que/worker.rb +128 -167
  55. data/que.gemspec +13 -2
  56. metadata +37 -80
  57. data/.rspec +0 -2
  58. data/.travis.yml +0 -64
  59. data/Gemfile +0 -24
  60. data/docs/customizing_que.md +0 -200
  61. data/lib/generators/que/install_generator.rb +0 -24
  62. data/lib/generators/que/templates/add_que.rb +0 -13
  63. data/lib/que/adapters/active_record.rb +0 -40
  64. data/lib/que/adapters/base.rb +0 -133
  65. data/lib/que/adapters/connection_pool.rb +0 -16
  66. data/lib/que/adapters/pg.rb +0 -21
  67. data/lib/que/adapters/pond.rb +0 -16
  68. data/lib/que/adapters/sequel.rb +0 -20
  69. data/lib/que/railtie.rb +0 -16
  70. data/lib/que/rake_tasks.rb +0 -59
  71. data/lib/que/sql.rb +0 -170
  72. data/spec/adapters/active_record_spec.rb +0 -175
  73. data/spec/adapters/connection_pool_spec.rb +0 -22
  74. data/spec/adapters/pg_spec.rb +0 -41
  75. data/spec/adapters/pond_spec.rb +0 -22
  76. data/spec/adapters/sequel_spec.rb +0 -57
  77. data/spec/gemfiles/Gemfile.current +0 -19
  78. data/spec/gemfiles/Gemfile.old +0 -19
  79. data/spec/gemfiles/Gemfile.older +0 -19
  80. data/spec/gemfiles/Gemfile.oldest +0 -19
  81. data/spec/spec_helper.rb +0 -129
  82. data/spec/support/helpers.rb +0 -25
  83. data/spec/support/jobs.rb +0 -35
  84. data/spec/support/shared_examples/adapter.rb +0 -42
  85. data/spec/support/shared_examples/multi_threaded_adapter.rb +0 -46
  86. data/spec/unit/configuration_spec.rb +0 -31
  87. data/spec/unit/connection_spec.rb +0 -14
  88. data/spec/unit/customization_spec.rb +0 -251
  89. data/spec/unit/enqueue_spec.rb +0 -245
  90. data/spec/unit/helper_spec.rb +0 -12
  91. data/spec/unit/logging_spec.rb +0 -101
  92. data/spec/unit/migrations_spec.rb +0 -84
  93. data/spec/unit/pool_spec.rb +0 -365
  94. data/spec/unit/run_spec.rb +0 -14
  95. data/spec/unit/states_spec.rb +0 -50
  96. data/spec/unit/stats_spec.rb +0 -46
  97. data/spec/unit/transaction_spec.rb +0 -36
  98. data/spec/unit/work_spec.rb +0 -596
  99. data/spec/unit/worker_spec.rb +0 -167
  100. data/tasks/benchmark.rb +0 -3
  101. data/tasks/rspec.rb +0 -14
  102. data/tasks/safe_shutdown.rb +0 -67
@@ -1,167 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
-
5
- describe Que::Worker do
6
- it "should work jobs when started until there are none available" do
7
- begin
8
- Que::Job.enqueue
9
- Que::Job.enqueue
10
- DB[:que_jobs].count.should be 2
11
-
12
- @worker = Que::Worker.new
13
- sleep_until { @worker.sleeping? }
14
- DB[:que_jobs].count.should be 0
15
-
16
- $logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_worked job_unavailable)
17
-
18
- json = JSON.load($logger.messages[0])
19
- json['job']['job_class'].should == 'Que::Job'
20
- ensure
21
- if @worker
22
- @worker.stop
23
- @worker.wait_until_stopped
24
- end
25
- end
26
- end
27
-
28
- it "should work jobs without a named queue by default" do
29
- begin
30
- Que::Job.enqueue 1
31
- Que::Job.enqueue 2, :queue => 'my_queue'
32
-
33
- @worker = Que::Worker.new
34
- sleep_until { @worker.sleeping? }
35
- DB[:que_jobs].count.should be 1
36
-
37
- $logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_unavailable)
38
-
39
- json = JSON.load($logger.messages[0])
40
- json['job']['queue'].should == ''
41
- json['job']['job_class'].should == 'Que::Job'
42
- json['job']['args'].should == [1]
43
- ensure
44
- if @worker
45
- @worker.stop
46
- @worker.wait_until_stopped
47
- end
48
- end
49
- end
50
-
51
- it "should accept the name of a single queue to work jobs from" do
52
- begin
53
- Que::Job.enqueue 1
54
- Que::Job.enqueue 2, :queue => 'my_queue'
55
-
56
- @worker = Que::Worker.new(:my_queue)
57
- sleep_until { @worker.sleeping? }
58
- DB[:que_jobs].count.should be 1
59
-
60
- $logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_unavailable)
61
-
62
- json = JSON.load($logger.messages[0])
63
- json['job']['queue'].should == 'my_queue'
64
- json['job']['job_class'].should == 'Que::Job'
65
- json['job']['args'].should == [2]
66
- ensure
67
- if @worker
68
- @worker.stop
69
- @worker.wait_until_stopped
70
- end
71
- end
72
- end
73
-
74
- it "#wake! should return truthy if the worker was asleep and is woken up, at which point it should work until no jobs are available" do
75
- begin
76
- @worker = Que::Worker.new
77
- sleep_until { @worker.sleeping? }
78
-
79
- Que::Job.enqueue
80
- Que::Job.enqueue
81
- DB[:que_jobs].count.should be 2
82
-
83
- @worker.wake!.should be true
84
- sleep_until { @worker.sleeping? }
85
- DB[:que_jobs].count.should be 0
86
- ensure
87
- if @worker
88
- @worker.stop
89
- @worker.wait_until_stopped
90
- end
91
- end
92
- end
93
-
94
- it "#wake! should return falsy if the worker was already working" do
95
- begin
96
- BlockJob.enqueue
97
- @worker = Que::Worker.new
98
-
99
- $q1.pop
100
- DB[:que_jobs].count.should be 1
101
- @worker.wake!.should be nil
102
- $q2.push nil
103
- ensure
104
- if @worker
105
- @worker.stop
106
- @worker.wait_until_stopped
107
- end
108
- end
109
- end
110
-
111
- it "should not be deterred by a job that raises an error" do
112
- begin
113
- ErrorJob.enqueue :priority => 1
114
- Que::Job.enqueue :priority => 5
115
-
116
- @worker = Que::Worker.new
117
-
118
- sleep_until { @worker.sleeping? }
119
-
120
- DB[:que_jobs].count.should be 1
121
- job = DB[:que_jobs].first
122
- job[:job_class].should == 'ErrorJob'
123
- job[:run_at].should be_within(3).of Time.now + 4
124
-
125
- log = JSON.load($logger.messages[0])
126
- log['event'].should == 'job_errored'
127
- log['error']['class'].should == 'RuntimeError'
128
- log['error']['message'].should == "ErrorJob!"
129
- log['job']['job_class'].should == 'ErrorJob'
130
- ensure
131
- if @worker
132
- @worker.stop
133
- @worker.wait_until_stopped
134
- end
135
- end
136
- end
137
-
138
- it "should receive and respect a notification to stop down when it is working, after its current job completes" do
139
- begin
140
- BlockJob.enqueue :priority => 1
141
- Que::Job.enqueue :priority => 5
142
- DB[:que_jobs].count.should be 2
143
-
144
- @worker = Que::Worker.new
145
-
146
- $q1.pop
147
- @worker.stop
148
- $q2.push nil
149
-
150
- @worker.wait_until_stopped
151
-
152
- DB[:que_jobs].count.should be 1
153
- job = DB[:que_jobs].first
154
- job[:job_class].should == 'Que::Job'
155
- end
156
- end
157
-
158
- it "should receive and respect a notification to stop when it is currently asleep" do
159
- begin
160
- @worker = Que::Worker.new
161
- sleep_until { @worker.sleeping? }
162
-
163
- @worker.stop
164
- @worker.wait_until_stopped
165
- end
166
- end
167
- end
@@ -1,3 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # The benchmarking tasks have been merged together and now live at github.com/chanks/queue-shootout.
@@ -1,14 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'rspec/core/rake_task'
4
-
5
- RSpec::Core::RakeTask.new :default do |spec|
6
- spec.pattern = './spec/**/*_spec.rb'
7
- end
8
-
9
- # Shortcut to skip the adapter specs, and run only with the basic PG
10
- # connection. I use this occasionally to make sure ActiveRecord isn't loaded,
11
- # so any accidental Rails-isms are caught.
12
- RSpec::Core::RakeTask.new :pg do |spec|
13
- spec.pattern = './spec/unit/*_spec.rb'
14
- end
@@ -1,67 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This task is used to test Que's behavior when its process is shut down.
4
-
5
- # The situation we're trying to avoid occurs when the process dies while a job
6
- # is in the middle of a transaction - ideally, the transaction would be rolled
7
- # back and the job could just be reattempted later, but if we're not careful,
8
- # the transaction could be committed prematurely. For specifics, see here:
9
-
10
- # http://coderrr.wordpress.com/2011/05/03/beware-of-threadkill-or-your-activerecord-transactions-are-in-danger-of-being-partially-committed/
11
-
12
- # So, this task opens a transaction within a job, makes a write, then prompts
13
- # you to kill it with one of a few signals. You can then run it again to make
14
- # sure that the write was rolled back (if it wasn't, Que isn't functioning
15
- # like it should). This task only explicitly tests Sequel, but the behavior
16
- # for ActiveRecord is very similar.
17
-
18
- task :safe_shutdown do
19
- require 'sequel'
20
- require 'que'
21
-
22
- url = ENV['DATABASE_URL'] || 'postgres://postgres:@localhost/que-test'
23
- DB = Sequel.connect(url)
24
-
25
- if DB.table_exists?(:que_jobs)
26
- puts "Uh-oh! Previous shutdown wasn't clean!" if DB[:que_jobs].where(:job_id => 0).count > 0
27
- DB.drop_table :que_jobs
28
- end
29
-
30
- Que.connection = DB
31
- Que.create!
32
-
33
- $queue = Queue.new
34
-
35
- class SafeJob < Que::Job
36
- def run
37
- DB.transaction do
38
- DB[:que_jobs].insert(:job_id => 0, :job_class => 'Que::Job')
39
- $queue.push nil
40
- sleep
41
- end
42
- end
43
- end
44
-
45
- SafeJob.enqueue
46
- Que.mode = :async
47
- $queue.pop
48
-
49
- puts "From a different terminal window, run one of the following:"
50
- %w(SIGINT SIGTERM SIGKILL).each do |signal|
51
- puts "kill -#{signal} #{Process.pid}"
52
- end
53
-
54
- stop = false
55
- trap('INT'){stop = true}
56
-
57
- at_exit do
58
- $stdout.puts "Finishing Que's current jobs before exiting..."
59
- Que.mode = :off
60
- $stdout.puts "Que's jobs finished, exiting..."
61
- end
62
-
63
- loop do
64
- sleep 0.01
65
- break if stop
66
- end
67
- end