que 0.14.3 → 1.0.0.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -0
  3. data/CHANGELOG.md +108 -14
  4. data/LICENSE.txt +1 -1
  5. data/README.md +49 -45
  6. data/bin/command_line_interface.rb +239 -0
  7. data/bin/que +8 -82
  8. data/docs/README.md +2 -0
  9. data/docs/active_job.md +6 -0
  10. data/docs/advanced_setup.md +7 -64
  11. data/docs/command_line_interface.md +45 -0
  12. data/docs/error_handling.md +65 -18
  13. data/docs/inspecting_the_queue.md +30 -80
  14. data/docs/job_helper_methods.md +27 -0
  15. data/docs/logging.md +3 -22
  16. data/docs/managing_workers.md +6 -61
  17. data/docs/middleware.md +15 -0
  18. data/docs/migrating.md +4 -7
  19. data/docs/multiple_queues.md +8 -4
  20. data/docs/shutting_down_safely.md +1 -1
  21. data/docs/using_plain_connections.md +39 -15
  22. data/docs/using_sequel.md +5 -3
  23. data/docs/writing_reliable_jobs.md +15 -24
  24. data/lib/que.rb +98 -182
  25. data/lib/que/active_job/extensions.rb +97 -0
  26. data/lib/que/active_record/connection.rb +51 -0
  27. data/lib/que/active_record/model.rb +48 -0
  28. data/lib/que/connection.rb +179 -0
  29. data/lib/que/connection_pool.rb +78 -0
  30. data/lib/que/job.rb +107 -156
  31. data/lib/que/job_cache.rb +240 -0
  32. data/lib/que/job_methods.rb +168 -0
  33. data/lib/que/listener.rb +176 -0
  34. data/lib/que/locker.rb +466 -0
  35. data/lib/que/metajob.rb +47 -0
  36. data/lib/que/migrations.rb +24 -17
  37. data/lib/que/migrations/4/down.sql +48 -0
  38. data/lib/que/migrations/4/up.sql +265 -0
  39. data/lib/que/poller.rb +267 -0
  40. data/lib/que/rails/railtie.rb +14 -0
  41. data/lib/que/result_queue.rb +35 -0
  42. data/lib/que/sequel/model.rb +51 -0
  43. data/lib/que/utils/assertions.rb +62 -0
  44. data/lib/que/utils/constantization.rb +19 -0
  45. data/lib/que/utils/error_notification.rb +68 -0
  46. data/lib/que/utils/freeze.rb +20 -0
  47. data/lib/que/utils/introspection.rb +50 -0
  48. data/lib/que/utils/json_serialization.rb +21 -0
  49. data/lib/que/utils/logging.rb +78 -0
  50. data/lib/que/utils/middleware.rb +33 -0
  51. data/lib/que/utils/queue_management.rb +18 -0
  52. data/lib/que/utils/transactions.rb +34 -0
  53. data/lib/que/version.rb +1 -1
  54. data/lib/que/worker.rb +128 -167
  55. data/que.gemspec +13 -2
  56. metadata +37 -80
  57. data/.rspec +0 -2
  58. data/.travis.yml +0 -64
  59. data/Gemfile +0 -24
  60. data/docs/customizing_que.md +0 -200
  61. data/lib/generators/que/install_generator.rb +0 -24
  62. data/lib/generators/que/templates/add_que.rb +0 -13
  63. data/lib/que/adapters/active_record.rb +0 -40
  64. data/lib/que/adapters/base.rb +0 -133
  65. data/lib/que/adapters/connection_pool.rb +0 -16
  66. data/lib/que/adapters/pg.rb +0 -21
  67. data/lib/que/adapters/pond.rb +0 -16
  68. data/lib/que/adapters/sequel.rb +0 -20
  69. data/lib/que/railtie.rb +0 -16
  70. data/lib/que/rake_tasks.rb +0 -59
  71. data/lib/que/sql.rb +0 -170
  72. data/spec/adapters/active_record_spec.rb +0 -175
  73. data/spec/adapters/connection_pool_spec.rb +0 -22
  74. data/spec/adapters/pg_spec.rb +0 -41
  75. data/spec/adapters/pond_spec.rb +0 -22
  76. data/spec/adapters/sequel_spec.rb +0 -57
  77. data/spec/gemfiles/Gemfile.current +0 -19
  78. data/spec/gemfiles/Gemfile.old +0 -19
  79. data/spec/gemfiles/Gemfile.older +0 -19
  80. data/spec/gemfiles/Gemfile.oldest +0 -19
  81. data/spec/spec_helper.rb +0 -129
  82. data/spec/support/helpers.rb +0 -25
  83. data/spec/support/jobs.rb +0 -35
  84. data/spec/support/shared_examples/adapter.rb +0 -42
  85. data/spec/support/shared_examples/multi_threaded_adapter.rb +0 -46
  86. data/spec/unit/configuration_spec.rb +0 -31
  87. data/spec/unit/connection_spec.rb +0 -14
  88. data/spec/unit/customization_spec.rb +0 -251
  89. data/spec/unit/enqueue_spec.rb +0 -245
  90. data/spec/unit/helper_spec.rb +0 -12
  91. data/spec/unit/logging_spec.rb +0 -101
  92. data/spec/unit/migrations_spec.rb +0 -84
  93. data/spec/unit/pool_spec.rb +0 -365
  94. data/spec/unit/run_spec.rb +0 -14
  95. data/spec/unit/states_spec.rb +0 -50
  96. data/spec/unit/stats_spec.rb +0 -46
  97. data/spec/unit/transaction_spec.rb +0 -36
  98. data/spec/unit/work_spec.rb +0 -596
  99. data/spec/unit/worker_spec.rb +0 -167
  100. data/tasks/benchmark.rb +0 -3
  101. data/tasks/rspec.rb +0 -14
  102. data/tasks/safe_shutdown.rb +0 -67
@@ -1,167 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
-
5
- describe Que::Worker do
6
- it "should work jobs when started until there are none available" do
7
- begin
8
- Que::Job.enqueue
9
- Que::Job.enqueue
10
- DB[:que_jobs].count.should be 2
11
-
12
- @worker = Que::Worker.new
13
- sleep_until { @worker.sleeping? }
14
- DB[:que_jobs].count.should be 0
15
-
16
- $logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_worked job_unavailable)
17
-
18
- json = JSON.load($logger.messages[0])
19
- json['job']['job_class'].should == 'Que::Job'
20
- ensure
21
- if @worker
22
- @worker.stop
23
- @worker.wait_until_stopped
24
- end
25
- end
26
- end
27
-
28
- it "should work jobs without a named queue by default" do
29
- begin
30
- Que::Job.enqueue 1
31
- Que::Job.enqueue 2, :queue => 'my_queue'
32
-
33
- @worker = Que::Worker.new
34
- sleep_until { @worker.sleeping? }
35
- DB[:que_jobs].count.should be 1
36
-
37
- $logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_unavailable)
38
-
39
- json = JSON.load($logger.messages[0])
40
- json['job']['queue'].should == ''
41
- json['job']['job_class'].should == 'Que::Job'
42
- json['job']['args'].should == [1]
43
- ensure
44
- if @worker
45
- @worker.stop
46
- @worker.wait_until_stopped
47
- end
48
- end
49
- end
50
-
51
- it "should accept the name of a single queue to work jobs from" do
52
- begin
53
- Que::Job.enqueue 1
54
- Que::Job.enqueue 2, :queue => 'my_queue'
55
-
56
- @worker = Que::Worker.new(:my_queue)
57
- sleep_until { @worker.sleeping? }
58
- DB[:que_jobs].count.should be 1
59
-
60
- $logger.messages.map{|m| JSON.load(m)['event']}.should == %w(job_worked job_unavailable)
61
-
62
- json = JSON.load($logger.messages[0])
63
- json['job']['queue'].should == 'my_queue'
64
- json['job']['job_class'].should == 'Que::Job'
65
- json['job']['args'].should == [2]
66
- ensure
67
- if @worker
68
- @worker.stop
69
- @worker.wait_until_stopped
70
- end
71
- end
72
- end
73
-
74
- it "#wake! should return truthy if the worker was asleep and is woken up, at which point it should work until no jobs are available" do
75
- begin
76
- @worker = Que::Worker.new
77
- sleep_until { @worker.sleeping? }
78
-
79
- Que::Job.enqueue
80
- Que::Job.enqueue
81
- DB[:que_jobs].count.should be 2
82
-
83
- @worker.wake!.should be true
84
- sleep_until { @worker.sleeping? }
85
- DB[:que_jobs].count.should be 0
86
- ensure
87
- if @worker
88
- @worker.stop
89
- @worker.wait_until_stopped
90
- end
91
- end
92
- end
93
-
94
- it "#wake! should return falsy if the worker was already working" do
95
- begin
96
- BlockJob.enqueue
97
- @worker = Que::Worker.new
98
-
99
- $q1.pop
100
- DB[:que_jobs].count.should be 1
101
- @worker.wake!.should be nil
102
- $q2.push nil
103
- ensure
104
- if @worker
105
- @worker.stop
106
- @worker.wait_until_stopped
107
- end
108
- end
109
- end
110
-
111
- it "should not be deterred by a job that raises an error" do
112
- begin
113
- ErrorJob.enqueue :priority => 1
114
- Que::Job.enqueue :priority => 5
115
-
116
- @worker = Que::Worker.new
117
-
118
- sleep_until { @worker.sleeping? }
119
-
120
- DB[:que_jobs].count.should be 1
121
- job = DB[:que_jobs].first
122
- job[:job_class].should == 'ErrorJob'
123
- job[:run_at].should be_within(3).of Time.now + 4
124
-
125
- log = JSON.load($logger.messages[0])
126
- log['event'].should == 'job_errored'
127
- log['error']['class'].should == 'RuntimeError'
128
- log['error']['message'].should == "ErrorJob!"
129
- log['job']['job_class'].should == 'ErrorJob'
130
- ensure
131
- if @worker
132
- @worker.stop
133
- @worker.wait_until_stopped
134
- end
135
- end
136
- end
137
-
138
- it "should receive and respect a notification to stop down when it is working, after its current job completes" do
139
- begin
140
- BlockJob.enqueue :priority => 1
141
- Que::Job.enqueue :priority => 5
142
- DB[:que_jobs].count.should be 2
143
-
144
- @worker = Que::Worker.new
145
-
146
- $q1.pop
147
- @worker.stop
148
- $q2.push nil
149
-
150
- @worker.wait_until_stopped
151
-
152
- DB[:que_jobs].count.should be 1
153
- job = DB[:que_jobs].first
154
- job[:job_class].should == 'Que::Job'
155
- end
156
- end
157
-
158
- it "should receive and respect a notification to stop when it is currently asleep" do
159
- begin
160
- @worker = Que::Worker.new
161
- sleep_until { @worker.sleeping? }
162
-
163
- @worker.stop
164
- @worker.wait_until_stopped
165
- end
166
- end
167
- end
@@ -1,3 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # The benchmarking tasks have been merged together and now live at github.com/chanks/queue-shootout.
@@ -1,14 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'rspec/core/rake_task'
4
-
5
- RSpec::Core::RakeTask.new :default do |spec|
6
- spec.pattern = './spec/**/*_spec.rb'
7
- end
8
-
9
- # Shortcut to skip the adapter specs, and run only with the basic PG
10
- # connection. I use this occasionally to make sure ActiveRecord isn't loaded,
11
- # so any accidental Rails-isms are caught.
12
- RSpec::Core::RakeTask.new :pg do |spec|
13
- spec.pattern = './spec/unit/*_spec.rb'
14
- end
@@ -1,67 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # This task is used to test Que's behavior when its process is shut down.
4
-
5
- # The situation we're trying to avoid occurs when the process dies while a job
6
- # is in the middle of a transaction - ideally, the transaction would be rolled
7
- # back and the job could just be reattempted later, but if we're not careful,
8
- # the transaction could be committed prematurely. For specifics, see here:
9
-
10
- # http://coderrr.wordpress.com/2011/05/03/beware-of-threadkill-or-your-activerecord-transactions-are-in-danger-of-being-partially-committed/
11
-
12
- # So, this task opens a transaction within a job, makes a write, then prompts
13
- # you to kill it with one of a few signals. You can then run it again to make
14
- # sure that the write was rolled back (if it wasn't, Que isn't functioning
15
- # like it should). This task only explicitly tests Sequel, but the behavior
16
- # for ActiveRecord is very similar.
17
-
18
- task :safe_shutdown do
19
- require 'sequel'
20
- require 'que'
21
-
22
- url = ENV['DATABASE_URL'] || 'postgres://postgres:@localhost/que-test'
23
- DB = Sequel.connect(url)
24
-
25
- if DB.table_exists?(:que_jobs)
26
- puts "Uh-oh! Previous shutdown wasn't clean!" if DB[:que_jobs].where(:job_id => 0).count > 0
27
- DB.drop_table :que_jobs
28
- end
29
-
30
- Que.connection = DB
31
- Que.create!
32
-
33
- $queue = Queue.new
34
-
35
- class SafeJob < Que::Job
36
- def run
37
- DB.transaction do
38
- DB[:que_jobs].insert(:job_id => 0, :job_class => 'Que::Job')
39
- $queue.push nil
40
- sleep
41
- end
42
- end
43
- end
44
-
45
- SafeJob.enqueue
46
- Que.mode = :async
47
- $queue.pop
48
-
49
- puts "From a different terminal window, run one of the following:"
50
- %w(SIGINT SIGTERM SIGKILL).each do |signal|
51
- puts "kill -#{signal} #{Process.pid}"
52
- end
53
-
54
- stop = false
55
- trap('INT'){stop = true}
56
-
57
- at_exit do
58
- $stdout.puts "Finishing Que's current jobs before exiting..."
59
- Que.mode = :off
60
- $stdout.puts "Que's jobs finished, exiting..."
61
- end
62
-
63
- loop do
64
- sleep 0.01
65
- break if stop
66
- end
67
- end