que 0.14.3 → 1.0.0.beta

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -0
  3. data/CHANGELOG.md +108 -14
  4. data/LICENSE.txt +1 -1
  5. data/README.md +49 -45
  6. data/bin/command_line_interface.rb +239 -0
  7. data/bin/que +8 -82
  8. data/docs/README.md +2 -0
  9. data/docs/active_job.md +6 -0
  10. data/docs/advanced_setup.md +7 -64
  11. data/docs/command_line_interface.md +45 -0
  12. data/docs/error_handling.md +65 -18
  13. data/docs/inspecting_the_queue.md +30 -80
  14. data/docs/job_helper_methods.md +27 -0
  15. data/docs/logging.md +3 -22
  16. data/docs/managing_workers.md +6 -61
  17. data/docs/middleware.md +15 -0
  18. data/docs/migrating.md +4 -7
  19. data/docs/multiple_queues.md +8 -4
  20. data/docs/shutting_down_safely.md +1 -1
  21. data/docs/using_plain_connections.md +39 -15
  22. data/docs/using_sequel.md +5 -3
  23. data/docs/writing_reliable_jobs.md +15 -24
  24. data/lib/que.rb +98 -182
  25. data/lib/que/active_job/extensions.rb +97 -0
  26. data/lib/que/active_record/connection.rb +51 -0
  27. data/lib/que/active_record/model.rb +48 -0
  28. data/lib/que/connection.rb +179 -0
  29. data/lib/que/connection_pool.rb +78 -0
  30. data/lib/que/job.rb +107 -156
  31. data/lib/que/job_cache.rb +240 -0
  32. data/lib/que/job_methods.rb +168 -0
  33. data/lib/que/listener.rb +176 -0
  34. data/lib/que/locker.rb +466 -0
  35. data/lib/que/metajob.rb +47 -0
  36. data/lib/que/migrations.rb +24 -17
  37. data/lib/que/migrations/4/down.sql +48 -0
  38. data/lib/que/migrations/4/up.sql +265 -0
  39. data/lib/que/poller.rb +267 -0
  40. data/lib/que/rails/railtie.rb +14 -0
  41. data/lib/que/result_queue.rb +35 -0
  42. data/lib/que/sequel/model.rb +51 -0
  43. data/lib/que/utils/assertions.rb +62 -0
  44. data/lib/que/utils/constantization.rb +19 -0
  45. data/lib/que/utils/error_notification.rb +68 -0
  46. data/lib/que/utils/freeze.rb +20 -0
  47. data/lib/que/utils/introspection.rb +50 -0
  48. data/lib/que/utils/json_serialization.rb +21 -0
  49. data/lib/que/utils/logging.rb +78 -0
  50. data/lib/que/utils/middleware.rb +33 -0
  51. data/lib/que/utils/queue_management.rb +18 -0
  52. data/lib/que/utils/transactions.rb +34 -0
  53. data/lib/que/version.rb +1 -1
  54. data/lib/que/worker.rb +128 -167
  55. data/que.gemspec +13 -2
  56. metadata +37 -80
  57. data/.rspec +0 -2
  58. data/.travis.yml +0 -64
  59. data/Gemfile +0 -24
  60. data/docs/customizing_que.md +0 -200
  61. data/lib/generators/que/install_generator.rb +0 -24
  62. data/lib/generators/que/templates/add_que.rb +0 -13
  63. data/lib/que/adapters/active_record.rb +0 -40
  64. data/lib/que/adapters/base.rb +0 -133
  65. data/lib/que/adapters/connection_pool.rb +0 -16
  66. data/lib/que/adapters/pg.rb +0 -21
  67. data/lib/que/adapters/pond.rb +0 -16
  68. data/lib/que/adapters/sequel.rb +0 -20
  69. data/lib/que/railtie.rb +0 -16
  70. data/lib/que/rake_tasks.rb +0 -59
  71. data/lib/que/sql.rb +0 -170
  72. data/spec/adapters/active_record_spec.rb +0 -175
  73. data/spec/adapters/connection_pool_spec.rb +0 -22
  74. data/spec/adapters/pg_spec.rb +0 -41
  75. data/spec/adapters/pond_spec.rb +0 -22
  76. data/spec/adapters/sequel_spec.rb +0 -57
  77. data/spec/gemfiles/Gemfile.current +0 -19
  78. data/spec/gemfiles/Gemfile.old +0 -19
  79. data/spec/gemfiles/Gemfile.older +0 -19
  80. data/spec/gemfiles/Gemfile.oldest +0 -19
  81. data/spec/spec_helper.rb +0 -129
  82. data/spec/support/helpers.rb +0 -25
  83. data/spec/support/jobs.rb +0 -35
  84. data/spec/support/shared_examples/adapter.rb +0 -42
  85. data/spec/support/shared_examples/multi_threaded_adapter.rb +0 -46
  86. data/spec/unit/configuration_spec.rb +0 -31
  87. data/spec/unit/connection_spec.rb +0 -14
  88. data/spec/unit/customization_spec.rb +0 -251
  89. data/spec/unit/enqueue_spec.rb +0 -245
  90. data/spec/unit/helper_spec.rb +0 -12
  91. data/spec/unit/logging_spec.rb +0 -101
  92. data/spec/unit/migrations_spec.rb +0 -84
  93. data/spec/unit/pool_spec.rb +0 -365
  94. data/spec/unit/run_spec.rb +0 -14
  95. data/spec/unit/states_spec.rb +0 -50
  96. data/spec/unit/stats_spec.rb +0 -46
  97. data/spec/unit/transaction_spec.rb +0 -36
  98. data/spec/unit/work_spec.rb +0 -596
  99. data/spec/unit/worker_spec.rb +0 -167
  100. data/tasks/benchmark.rb +0 -3
  101. data/tasks/rspec.rb +0 -14
  102. data/tasks/safe_shutdown.rb +0 -67
@@ -1,22 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
- require 'connection_pool'
5
-
6
- Que.connection = QUE_SPEC_CONNECTION_POOL = ConnectionPool.new &NEW_PG_CONNECTION
7
- QUE_ADAPTERS[:connection_pool] = Que.adapter
8
-
9
- describe "Que using the ConnectionPool adapter" do
10
- before { Que.adapter = QUE_ADAPTERS[:connection_pool] }
11
-
12
- it_behaves_like "a multi-threaded Que adapter"
13
-
14
- it "should be able to tell when it's already in a transaction" do
15
- Que.adapter.should_not be_in_transaction
16
- QUE_SPEC_CONNECTION_POOL.with do |conn|
17
- conn.async_exec "BEGIN"
18
- Que.adapter.should be_in_transaction
19
- conn.async_exec "COMMIT"
20
- end
21
- end
22
- end
@@ -1,41 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
-
5
- describe "Que using a bare PG connection" do
6
- it_behaves_like "a Que adapter"
7
-
8
- it "should synchronize access to that connection" do
9
- lock = Que.adapter.lock
10
- q1, q2 = Queue.new, Queue.new
11
-
12
- thread1 = Thread.new do
13
- Que.adapter.checkout do
14
- q1.push nil
15
- q2.pop
16
- end
17
- end
18
-
19
- q1.pop
20
-
21
- thread2 = Thread.new do
22
- Que.adapter.checkout do
23
- q1.push nil
24
- q2.pop
25
- end
26
- end
27
-
28
- sleep_until { thread2.status == 'sleep' }
29
-
30
- thread1.should be_alive
31
- thread2.should be_alive
32
-
33
- lock.send(:instance_variable_get, :@mon_owner).should == thread1
34
- q2.push nil
35
- q1.pop
36
- lock.send(:instance_variable_get, :@mon_owner).should == thread2
37
- q2.push nil
38
- thread1.join
39
- thread2.join
40
- end
41
- end
@@ -1,22 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
- require 'pond'
5
-
6
- Que.connection = QUE_SPEC_POND = Pond.new &NEW_PG_CONNECTION
7
- QUE_ADAPTERS[:pond] = Que.adapter
8
-
9
- describe "Que using the Pond adapter" do
10
- before { Que.adapter = QUE_ADAPTERS[:pond] }
11
-
12
- it_behaves_like "a multi-threaded Que adapter"
13
-
14
- it "should be able to tell when it's already in a transaction" do
15
- Que.adapter.should_not be_in_transaction
16
- QUE_SPEC_POND.checkout do |conn|
17
- conn.async_exec "BEGIN"
18
- Que.adapter.should be_in_transaction
19
- conn.async_exec "COMMIT"
20
- end
21
- end
22
- end
@@ -1,57 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
-
5
- Que.connection = SEQUEL_ADAPTER_DB = Sequel.connect(QUE_URL)
6
- QUE_ADAPTERS[:sequel] = Que.adapter
7
-
8
- describe "Que using the Sequel adapter" do
9
- before { Que.adapter = QUE_ADAPTERS[:sequel] }
10
-
11
- it_behaves_like "a multi-threaded Que adapter"
12
-
13
- it "should use the same connection that Sequel does" do
14
- begin
15
- class SequelJob < Que::Job
16
- def run
17
- $pid1 = Integer(Que.execute("select pg_backend_pid()").first['pg_backend_pid'])
18
- $pid2 = Integer(SEQUEL_ADAPTER_DB['select pg_backend_pid()'].get)
19
- end
20
- end
21
-
22
- SequelJob.enqueue
23
- Que::Job.work
24
-
25
- $pid1.should == $pid2
26
- ensure
27
- $pid1 = $pid2 = nil
28
- end
29
- end
30
-
31
- it "should wake up a Worker after queueing a job in async mode, waiting for a transaction to commit if necessary" do
32
- Que.mode = :async
33
- Que.worker_count = 4
34
- sleep_until { Que::Worker.workers.all? &:sleeping? }
35
-
36
- # Wakes a worker immediately when not in a transaction.
37
- Que::Job.enqueue
38
- sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
39
-
40
- SEQUEL_ADAPTER_DB.transaction do
41
- Que::Job.enqueue
42
- Que::Worker.workers.each { |worker| worker.should be_sleeping }
43
- end
44
- sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
45
-
46
- # Do nothing when queueing with a specific :run_at.
47
- BlockJob.enqueue :run_at => Time.now
48
- Que::Worker.workers.each { |worker| worker.should be_sleeping }
49
- end
50
-
51
- it "should be able to tell when it's in a Sequel transaction" do
52
- Que.adapter.should_not be_in_transaction
53
- SEQUEL_ADAPTER_DB.transaction do
54
- Que.adapter.should be_in_transaction
55
- end
56
- end
57
- end
@@ -1,19 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem 'que', path: '../..'
4
-
5
- group :development, :test do
6
- gem 'rake', '< 11.0'
7
-
8
- gem 'activerecord', :require => nil
9
- gem 'sequel', :require => nil
10
- gem 'connection_pool', :require => nil
11
- gem 'pond', :require => nil
12
- gem 'pg', :require => nil, :platform => :ruby
13
- gem 'pg_jruby', :require => nil, :platform => :jruby
14
- end
15
-
16
- group :test do
17
- gem 'rspec', '~> 2.14.1'
18
- gem 'pry'
19
- end
@@ -1,19 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem 'que', path: '../..'
4
-
5
- group :development, :test do
6
- gem 'rake', '< 11.0'
7
-
8
- gem 'activerecord', '~> 4.2.0', :require => nil
9
- gem 'sequel', '~> 3.0', :require => nil
10
- gem 'connection_pool', :require => nil
11
- gem 'pond', :require => nil
12
- gem 'pg', :require => nil, :platform => :ruby
13
- gem 'pg_jruby', :require => nil, :platform => :jruby
14
- end
15
-
16
- group :test do
17
- gem 'rspec', '~> 2.14.1'
18
- gem 'pry'
19
- end
@@ -1,19 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem 'que', path: '../..'
4
-
5
- group :development, :test do
6
- gem 'rake', '< 11.0'
7
-
8
- gem 'activerecord', '~> 4.1.0', :require => nil
9
- gem 'sequel', :require => nil
10
- gem 'connection_pool', :require => nil
11
- gem 'pond', :require => nil
12
- gem 'pg', :require => nil, :platform => :ruby
13
- gem 'pg_jruby', :require => nil, :platform => :jruby
14
- end
15
-
16
- group :test do
17
- gem 'rspec', '~> 2.14.1'
18
- gem 'pry'
19
- end
@@ -1,19 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem 'que', path: '../..'
4
-
5
- group :development, :test do
6
- gem 'rake', '< 11.0'
7
-
8
- gem 'activerecord', '~> 4.0.0', :require => nil
9
- gem 'sequel', :require => nil
10
- gem 'connection_pool', :require => nil
11
- gem 'pond', :require => nil
12
- gem 'pg', :require => nil, :platform => :ruby
13
- gem 'pg_jruby', :require => nil, :platform => :jruby
14
- end
15
-
16
- group :test do
17
- gem 'rspec', '~> 2.14.1'
18
- gem 'pry'
19
- end
@@ -1,129 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'que'
4
- require 'uri'
5
- require 'pg'
6
- require 'logger'
7
- require 'json'
8
- require 'pry'
9
-
10
- Dir['./spec/support/**/*.rb'].sort.each &method(:require)
11
-
12
-
13
-
14
- # Handy constants for initializing PG connections:
15
- QUE_URL = ENV['DATABASE_URL'] || 'postgres://postgres:@localhost/que-test'
16
-
17
- NEW_PG_CONNECTION = proc do
18
- uri = URI.parse(QUE_URL)
19
- pg = PG::Connection.open :host => uri.host,
20
- :user => uri.user,
21
- :password => uri.password,
22
- :port => uri.port || 5432,
23
- :dbname => uri.path[1..-1]
24
-
25
- # Avoid annoying NOTICE messages in specs.
26
- pg.async_exec "SET client_min_messages TO 'warning'"
27
- pg
28
- end
29
-
30
-
31
-
32
- # Adapters track which statements have been prepared for their connections,
33
- # and if Que.connection= is called before each spec, we're constantly creating
34
- # new adapters and losing that information, which is bad. So instead, we hang
35
- # onto a few adapters and assign them using Que.adapter= as needed. The plain
36
- # pg adapter is the default.
37
-
38
- # Also, let Que initialize the adapter itself, to make sure that the
39
- # recognition logic works. Similar code can be found in the adapter specs.
40
- Que.connection = NEW_PG_CONNECTION.call
41
- QUE_ADAPTERS = {:pg => Que.adapter}
42
-
43
-
44
-
45
- # We use Sequel to examine the database in specs.
46
- require 'sequel'
47
- DB = Sequel.connect(QUE_URL)
48
-
49
-
50
-
51
- if ENV['CI']
52
- DB.synchronize do |conn|
53
- puts "Ruby #{RUBY_VERSION}"
54
- puts "Sequel #{Sequel::VERSION}"
55
- puts conn.async_exec("SELECT version()").to_a.first['version']
56
- end
57
- end
58
-
59
-
60
-
61
- # Reset the table to the most up-to-date version.
62
- DB.drop_table? :que_jobs
63
- Que::Migrations.migrate!
64
-
65
-
66
-
67
- # Set up a dummy logger.
68
- Que.logger = $logger = Object.new
69
- $logger_mutex = Mutex.new # Protect against rare errors on Rubinius/JRuby.
70
-
71
- def $logger.messages
72
- @messages ||= []
73
- end
74
-
75
- def $logger.method_missing(m, message)
76
- $logger_mutex.synchronize { messages << message }
77
- end
78
-
79
- # Object includes Kernel#warn which is not what we expect, so remove:
80
- def $logger.warn(message)
81
- method_missing(:warn, message)
82
- end
83
-
84
-
85
-
86
- # Helper to display spec descriptions.
87
- description_builder = -> hash do
88
- if g = hash[:example_group]
89
- "#{description_builder.call(g)} #{hash[:description_args].first}"
90
- else
91
- hash[:description_args].first
92
- end
93
- end
94
-
95
- stdout = Logger.new(STDOUT)
96
-
97
- RSpec.configure do |config|
98
- config.around do |spec|
99
- # Figure out which spec is about to run, for logging purposes.
100
- data = example.metadata
101
- desc = description_builder.call(data)
102
- line = "rspec #{data[:file_path]}:#{data[:line_number]}"
103
-
104
- # Optionally log to STDOUT which spec is about to run. This is noisy, but
105
- # helpful in identifying hanging specs.
106
- stdout.info "Running spec: #{desc} @ #{line}" if ENV['LOG_SPEC']
107
-
108
- Que.adapter = QUE_ADAPTERS[:pg]
109
-
110
- Que.worker_count = 0
111
- Que.mode = :async
112
- Que.wake_interval = nil
113
-
114
- $logger.messages.clear
115
-
116
- spec.run
117
-
118
- Que.worker_count = 0
119
- Que.mode = :off
120
- Que.wake_interval = nil
121
-
122
- DB[:que_jobs].delete
123
-
124
- # A bit of lint: make sure that no advisory locks are left open.
125
- unless DB[:pg_locks].where(:locktype => 'advisory').empty?
126
- stdout.info "Advisory lock left open: #{desc} @ #{line}"
127
- end
128
- end
129
- end
@@ -1,25 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Travis seems to freeze the VM the tests run in sometimes, so bump up the limit
4
- # when running in CI.
5
- QUE_SLEEP_UNTIL_TIMEOUT = ENV['CI'] ? 10 : 2
6
-
7
- # Helper for testing threaded code.
8
- def sleep_until(timeout = QUE_SLEEP_UNTIL_TIMEOUT)
9
- deadline = Time.now + timeout
10
- loop do
11
- break if yield
12
- if Time.now > deadline
13
- puts "sleep_until timeout reached!"
14
- raise "sleep_until timeout reached!"
15
- end
16
- sleep 0.01
17
- end
18
- end
19
-
20
- def suppress_warnings
21
- original_verbosity, $VERBOSE = $VERBOSE, nil
22
- yield
23
- ensure
24
- $VERBOSE = original_verbosity
25
- end
@@ -1,35 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Common Job classes for use in specs.
4
-
5
- # Handy for blocking in the middle of processing a job.
6
- class BlockJob < Que::Job
7
- def run
8
- $q1.push nil
9
- $q2.pop
10
- end
11
- end
12
-
13
- RSpec.configure do |config|
14
- config.before { $q1, $q2 = Queue.new, Queue.new }
15
- end
16
-
17
-
18
-
19
- class ErrorJob < Que::Job
20
- def run
21
- raise "ErrorJob!"
22
- end
23
- end
24
-
25
-
26
-
27
- class ArgsJob < Que::Job
28
- def run(*args)
29
- $passed_args = args
30
- end
31
- end
32
-
33
- RSpec.configure do |config|
34
- config.before { $passed_args = nil }
35
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- shared_examples "a Que adapter" do
4
- it "should be able to execute arbitrary SQL and return indifferent hashes" do
5
- result = Que.execute("SELECT 1 AS one")
6
- result.should == [{'one'=>1}]
7
- result.first[:one].should == 1
8
- end
9
-
10
- it "should be able to cast boolean results properly" do
11
- r = Que.execute("SELECT true AS true_value, false AS false_value")
12
- r.should == [{'true_value' => true, 'false_value' => false}]
13
- end
14
-
15
- it "should be able to execute multiple SQL statements in one string" do
16
- Que.execute("SELECT 1 AS one; SELECT 1 AS one")
17
- end
18
-
19
- it "should be able to queue and work a job" do
20
- Que::Job.enqueue
21
- result = Que::Job.work
22
- result[:event].should == :job_worked
23
- result[:job][:job_class].should == 'Que::Job'
24
- end
25
-
26
- it "should yield the same Postgres connection for the duration of the block" do
27
- Que.adapter.checkout do |conn|
28
- conn.should be_a PG::Connection
29
- pid1 = Que.execute "SELECT pg_backend_pid()"
30
- pid2 = Que.execute "SELECT pg_backend_pid()"
31
- pid1.should == pid2
32
- end
33
- end
34
-
35
- it "should allow nested checkouts" do
36
- Que.adapter.checkout do |a|
37
- Que.adapter.checkout do |b|
38
- a.object_id.should == b.object_id
39
- end
40
- end
41
- end
42
- end