que 0.14.3 → 1.0.0.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +5 -5
  2. data/.gitignore +2 -0
  3. data/CHANGELOG.md +108 -14
  4. data/LICENSE.txt +1 -1
  5. data/README.md +49 -45
  6. data/bin/command_line_interface.rb +239 -0
  7. data/bin/que +8 -82
  8. data/docs/README.md +2 -0
  9. data/docs/active_job.md +6 -0
  10. data/docs/advanced_setup.md +7 -64
  11. data/docs/command_line_interface.md +45 -0
  12. data/docs/error_handling.md +65 -18
  13. data/docs/inspecting_the_queue.md +30 -80
  14. data/docs/job_helper_methods.md +27 -0
  15. data/docs/logging.md +3 -22
  16. data/docs/managing_workers.md +6 -61
  17. data/docs/middleware.md +15 -0
  18. data/docs/migrating.md +4 -7
  19. data/docs/multiple_queues.md +8 -4
  20. data/docs/shutting_down_safely.md +1 -1
  21. data/docs/using_plain_connections.md +39 -15
  22. data/docs/using_sequel.md +5 -3
  23. data/docs/writing_reliable_jobs.md +15 -24
  24. data/lib/que.rb +98 -182
  25. data/lib/que/active_job/extensions.rb +97 -0
  26. data/lib/que/active_record/connection.rb +51 -0
  27. data/lib/que/active_record/model.rb +48 -0
  28. data/lib/que/connection.rb +179 -0
  29. data/lib/que/connection_pool.rb +78 -0
  30. data/lib/que/job.rb +107 -156
  31. data/lib/que/job_cache.rb +240 -0
  32. data/lib/que/job_methods.rb +168 -0
  33. data/lib/que/listener.rb +176 -0
  34. data/lib/que/locker.rb +466 -0
  35. data/lib/que/metajob.rb +47 -0
  36. data/lib/que/migrations.rb +24 -17
  37. data/lib/que/migrations/4/down.sql +48 -0
  38. data/lib/que/migrations/4/up.sql +265 -0
  39. data/lib/que/poller.rb +267 -0
  40. data/lib/que/rails/railtie.rb +14 -0
  41. data/lib/que/result_queue.rb +35 -0
  42. data/lib/que/sequel/model.rb +51 -0
  43. data/lib/que/utils/assertions.rb +62 -0
  44. data/lib/que/utils/constantization.rb +19 -0
  45. data/lib/que/utils/error_notification.rb +68 -0
  46. data/lib/que/utils/freeze.rb +20 -0
  47. data/lib/que/utils/introspection.rb +50 -0
  48. data/lib/que/utils/json_serialization.rb +21 -0
  49. data/lib/que/utils/logging.rb +78 -0
  50. data/lib/que/utils/middleware.rb +33 -0
  51. data/lib/que/utils/queue_management.rb +18 -0
  52. data/lib/que/utils/transactions.rb +34 -0
  53. data/lib/que/version.rb +1 -1
  54. data/lib/que/worker.rb +128 -167
  55. data/que.gemspec +13 -2
  56. metadata +37 -80
  57. data/.rspec +0 -2
  58. data/.travis.yml +0 -64
  59. data/Gemfile +0 -24
  60. data/docs/customizing_que.md +0 -200
  61. data/lib/generators/que/install_generator.rb +0 -24
  62. data/lib/generators/que/templates/add_que.rb +0 -13
  63. data/lib/que/adapters/active_record.rb +0 -40
  64. data/lib/que/adapters/base.rb +0 -133
  65. data/lib/que/adapters/connection_pool.rb +0 -16
  66. data/lib/que/adapters/pg.rb +0 -21
  67. data/lib/que/adapters/pond.rb +0 -16
  68. data/lib/que/adapters/sequel.rb +0 -20
  69. data/lib/que/railtie.rb +0 -16
  70. data/lib/que/rake_tasks.rb +0 -59
  71. data/lib/que/sql.rb +0 -170
  72. data/spec/adapters/active_record_spec.rb +0 -175
  73. data/spec/adapters/connection_pool_spec.rb +0 -22
  74. data/spec/adapters/pg_spec.rb +0 -41
  75. data/spec/adapters/pond_spec.rb +0 -22
  76. data/spec/adapters/sequel_spec.rb +0 -57
  77. data/spec/gemfiles/Gemfile.current +0 -19
  78. data/spec/gemfiles/Gemfile.old +0 -19
  79. data/spec/gemfiles/Gemfile.older +0 -19
  80. data/spec/gemfiles/Gemfile.oldest +0 -19
  81. data/spec/spec_helper.rb +0 -129
  82. data/spec/support/helpers.rb +0 -25
  83. data/spec/support/jobs.rb +0 -35
  84. data/spec/support/shared_examples/adapter.rb +0 -42
  85. data/spec/support/shared_examples/multi_threaded_adapter.rb +0 -46
  86. data/spec/unit/configuration_spec.rb +0 -31
  87. data/spec/unit/connection_spec.rb +0 -14
  88. data/spec/unit/customization_spec.rb +0 -251
  89. data/spec/unit/enqueue_spec.rb +0 -245
  90. data/spec/unit/helper_spec.rb +0 -12
  91. data/spec/unit/logging_spec.rb +0 -101
  92. data/spec/unit/migrations_spec.rb +0 -84
  93. data/spec/unit/pool_spec.rb +0 -365
  94. data/spec/unit/run_spec.rb +0 -14
  95. data/spec/unit/states_spec.rb +0 -50
  96. data/spec/unit/stats_spec.rb +0 -46
  97. data/spec/unit/transaction_spec.rb +0 -36
  98. data/spec/unit/work_spec.rb +0 -596
  99. data/spec/unit/worker_spec.rb +0 -167
  100. data/tasks/benchmark.rb +0 -3
  101. data/tasks/rspec.rb +0 -14
  102. data/tasks/safe_shutdown.rb +0 -67
@@ -1,22 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
- require 'connection_pool'
5
-
6
- Que.connection = QUE_SPEC_CONNECTION_POOL = ConnectionPool.new &NEW_PG_CONNECTION
7
- QUE_ADAPTERS[:connection_pool] = Que.adapter
8
-
9
- describe "Que using the ConnectionPool adapter" do
10
- before { Que.adapter = QUE_ADAPTERS[:connection_pool] }
11
-
12
- it_behaves_like "a multi-threaded Que adapter"
13
-
14
- it "should be able to tell when it's already in a transaction" do
15
- Que.adapter.should_not be_in_transaction
16
- QUE_SPEC_CONNECTION_POOL.with do |conn|
17
- conn.async_exec "BEGIN"
18
- Que.adapter.should be_in_transaction
19
- conn.async_exec "COMMIT"
20
- end
21
- end
22
- end
@@ -1,41 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
-
5
- describe "Que using a bare PG connection" do
6
- it_behaves_like "a Que adapter"
7
-
8
- it "should synchronize access to that connection" do
9
- lock = Que.adapter.lock
10
- q1, q2 = Queue.new, Queue.new
11
-
12
- thread1 = Thread.new do
13
- Que.adapter.checkout do
14
- q1.push nil
15
- q2.pop
16
- end
17
- end
18
-
19
- q1.pop
20
-
21
- thread2 = Thread.new do
22
- Que.adapter.checkout do
23
- q1.push nil
24
- q2.pop
25
- end
26
- end
27
-
28
- sleep_until { thread2.status == 'sleep' }
29
-
30
- thread1.should be_alive
31
- thread2.should be_alive
32
-
33
- lock.send(:instance_variable_get, :@mon_owner).should == thread1
34
- q2.push nil
35
- q1.pop
36
- lock.send(:instance_variable_get, :@mon_owner).should == thread2
37
- q2.push nil
38
- thread1.join
39
- thread2.join
40
- end
41
- end
@@ -1,22 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
- require 'pond'
5
-
6
- Que.connection = QUE_SPEC_POND = Pond.new &NEW_PG_CONNECTION
7
- QUE_ADAPTERS[:pond] = Que.adapter
8
-
9
- describe "Que using the Pond adapter" do
10
- before { Que.adapter = QUE_ADAPTERS[:pond] }
11
-
12
- it_behaves_like "a multi-threaded Que adapter"
13
-
14
- it "should be able to tell when it's already in a transaction" do
15
- Que.adapter.should_not be_in_transaction
16
- QUE_SPEC_POND.checkout do |conn|
17
- conn.async_exec "BEGIN"
18
- Que.adapter.should be_in_transaction
19
- conn.async_exec "COMMIT"
20
- end
21
- end
22
- end
@@ -1,57 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
-
5
- Que.connection = SEQUEL_ADAPTER_DB = Sequel.connect(QUE_URL)
6
- QUE_ADAPTERS[:sequel] = Que.adapter
7
-
8
- describe "Que using the Sequel adapter" do
9
- before { Que.adapter = QUE_ADAPTERS[:sequel] }
10
-
11
- it_behaves_like "a multi-threaded Que adapter"
12
-
13
- it "should use the same connection that Sequel does" do
14
- begin
15
- class SequelJob < Que::Job
16
- def run
17
- $pid1 = Integer(Que.execute("select pg_backend_pid()").first['pg_backend_pid'])
18
- $pid2 = Integer(SEQUEL_ADAPTER_DB['select pg_backend_pid()'].get)
19
- end
20
- end
21
-
22
- SequelJob.enqueue
23
- Que::Job.work
24
-
25
- $pid1.should == $pid2
26
- ensure
27
- $pid1 = $pid2 = nil
28
- end
29
- end
30
-
31
- it "should wake up a Worker after queueing a job in async mode, waiting for a transaction to commit if necessary" do
32
- Que.mode = :async
33
- Que.worker_count = 4
34
- sleep_until { Que::Worker.workers.all? &:sleeping? }
35
-
36
- # Wakes a worker immediately when not in a transaction.
37
- Que::Job.enqueue
38
- sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
39
-
40
- SEQUEL_ADAPTER_DB.transaction do
41
- Que::Job.enqueue
42
- Que::Worker.workers.each { |worker| worker.should be_sleeping }
43
- end
44
- sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
45
-
46
- # Do nothing when queueing with a specific :run_at.
47
- BlockJob.enqueue :run_at => Time.now
48
- Que::Worker.workers.each { |worker| worker.should be_sleeping }
49
- end
50
-
51
- it "should be able to tell when it's in a Sequel transaction" do
52
- Que.adapter.should_not be_in_transaction
53
- SEQUEL_ADAPTER_DB.transaction do
54
- Que.adapter.should be_in_transaction
55
- end
56
- end
57
- end
@@ -1,19 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem 'que', path: '../..'
4
-
5
- group :development, :test do
6
- gem 'rake', '< 11.0'
7
-
8
- gem 'activerecord', :require => nil
9
- gem 'sequel', :require => nil
10
- gem 'connection_pool', :require => nil
11
- gem 'pond', :require => nil
12
- gem 'pg', :require => nil, :platform => :ruby
13
- gem 'pg_jruby', :require => nil, :platform => :jruby
14
- end
15
-
16
- group :test do
17
- gem 'rspec', '~> 2.14.1'
18
- gem 'pry'
19
- end
@@ -1,19 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem 'que', path: '../..'
4
-
5
- group :development, :test do
6
- gem 'rake', '< 11.0'
7
-
8
- gem 'activerecord', '~> 4.2.0', :require => nil
9
- gem 'sequel', '~> 3.0', :require => nil
10
- gem 'connection_pool', :require => nil
11
- gem 'pond', :require => nil
12
- gem 'pg', :require => nil, :platform => :ruby
13
- gem 'pg_jruby', :require => nil, :platform => :jruby
14
- end
15
-
16
- group :test do
17
- gem 'rspec', '~> 2.14.1'
18
- gem 'pry'
19
- end
@@ -1,19 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem 'que', path: '../..'
4
-
5
- group :development, :test do
6
- gem 'rake', '< 11.0'
7
-
8
- gem 'activerecord', '~> 4.1.0', :require => nil
9
- gem 'sequel', :require => nil
10
- gem 'connection_pool', :require => nil
11
- gem 'pond', :require => nil
12
- gem 'pg', :require => nil, :platform => :ruby
13
- gem 'pg_jruby', :require => nil, :platform => :jruby
14
- end
15
-
16
- group :test do
17
- gem 'rspec', '~> 2.14.1'
18
- gem 'pry'
19
- end
@@ -1,19 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem 'que', path: '../..'
4
-
5
- group :development, :test do
6
- gem 'rake', '< 11.0'
7
-
8
- gem 'activerecord', '~> 4.0.0', :require => nil
9
- gem 'sequel', :require => nil
10
- gem 'connection_pool', :require => nil
11
- gem 'pond', :require => nil
12
- gem 'pg', :require => nil, :platform => :ruby
13
- gem 'pg_jruby', :require => nil, :platform => :jruby
14
- end
15
-
16
- group :test do
17
- gem 'rspec', '~> 2.14.1'
18
- gem 'pry'
19
- end
@@ -1,129 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'que'
4
- require 'uri'
5
- require 'pg'
6
- require 'logger'
7
- require 'json'
8
- require 'pry'
9
-
10
- Dir['./spec/support/**/*.rb'].sort.each &method(:require)
11
-
12
-
13
-
14
- # Handy constants for initializing PG connections:
15
- QUE_URL = ENV['DATABASE_URL'] || 'postgres://postgres:@localhost/que-test'
16
-
17
- NEW_PG_CONNECTION = proc do
18
- uri = URI.parse(QUE_URL)
19
- pg = PG::Connection.open :host => uri.host,
20
- :user => uri.user,
21
- :password => uri.password,
22
- :port => uri.port || 5432,
23
- :dbname => uri.path[1..-1]
24
-
25
- # Avoid annoying NOTICE messages in specs.
26
- pg.async_exec "SET client_min_messages TO 'warning'"
27
- pg
28
- end
29
-
30
-
31
-
32
- # Adapters track which statements have been prepared for their connections,
33
- # and if Que.connection= is called before each spec, we're constantly creating
34
- # new adapters and losing that information, which is bad. So instead, we hang
35
- # onto a few adapters and assign them using Que.adapter= as needed. The plain
36
- # pg adapter is the default.
37
-
38
- # Also, let Que initialize the adapter itself, to make sure that the
39
- # recognition logic works. Similar code can be found in the adapter specs.
40
- Que.connection = NEW_PG_CONNECTION.call
41
- QUE_ADAPTERS = {:pg => Que.adapter}
42
-
43
-
44
-
45
- # We use Sequel to examine the database in specs.
46
- require 'sequel'
47
- DB = Sequel.connect(QUE_URL)
48
-
49
-
50
-
51
- if ENV['CI']
52
- DB.synchronize do |conn|
53
- puts "Ruby #{RUBY_VERSION}"
54
- puts "Sequel #{Sequel::VERSION}"
55
- puts conn.async_exec("SELECT version()").to_a.first['version']
56
- end
57
- end
58
-
59
-
60
-
61
- # Reset the table to the most up-to-date version.
62
- DB.drop_table? :que_jobs
63
- Que::Migrations.migrate!
64
-
65
-
66
-
67
- # Set up a dummy logger.
68
- Que.logger = $logger = Object.new
69
- $logger_mutex = Mutex.new # Protect against rare errors on Rubinius/JRuby.
70
-
71
- def $logger.messages
72
- @messages ||= []
73
- end
74
-
75
- def $logger.method_missing(m, message)
76
- $logger_mutex.synchronize { messages << message }
77
- end
78
-
79
- # Object includes Kernel#warn which is not what we expect, so remove:
80
- def $logger.warn(message)
81
- method_missing(:warn, message)
82
- end
83
-
84
-
85
-
86
- # Helper to display spec descriptions.
87
- description_builder = -> hash do
88
- if g = hash[:example_group]
89
- "#{description_builder.call(g)} #{hash[:description_args].first}"
90
- else
91
- hash[:description_args].first
92
- end
93
- end
94
-
95
- stdout = Logger.new(STDOUT)
96
-
97
- RSpec.configure do |config|
98
- config.around do |spec|
99
- # Figure out which spec is about to run, for logging purposes.
100
- data = example.metadata
101
- desc = description_builder.call(data)
102
- line = "rspec #{data[:file_path]}:#{data[:line_number]}"
103
-
104
- # Optionally log to STDOUT which spec is about to run. This is noisy, but
105
- # helpful in identifying hanging specs.
106
- stdout.info "Running spec: #{desc} @ #{line}" if ENV['LOG_SPEC']
107
-
108
- Que.adapter = QUE_ADAPTERS[:pg]
109
-
110
- Que.worker_count = 0
111
- Que.mode = :async
112
- Que.wake_interval = nil
113
-
114
- $logger.messages.clear
115
-
116
- spec.run
117
-
118
- Que.worker_count = 0
119
- Que.mode = :off
120
- Que.wake_interval = nil
121
-
122
- DB[:que_jobs].delete
123
-
124
- # A bit of lint: make sure that no advisory locks are left open.
125
- unless DB[:pg_locks].where(:locktype => 'advisory').empty?
126
- stdout.info "Advisory lock left open: #{desc} @ #{line}"
127
- end
128
- end
129
- end
@@ -1,25 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Travis seems to freeze the VM the tests run in sometimes, so bump up the limit
4
- # when running in CI.
5
- QUE_SLEEP_UNTIL_TIMEOUT = ENV['CI'] ? 10 : 2
6
-
7
- # Helper for testing threaded code.
8
- def sleep_until(timeout = QUE_SLEEP_UNTIL_TIMEOUT)
9
- deadline = Time.now + timeout
10
- loop do
11
- break if yield
12
- if Time.now > deadline
13
- puts "sleep_until timeout reached!"
14
- raise "sleep_until timeout reached!"
15
- end
16
- sleep 0.01
17
- end
18
- end
19
-
20
- def suppress_warnings
21
- original_verbosity, $VERBOSE = $VERBOSE, nil
22
- yield
23
- ensure
24
- $VERBOSE = original_verbosity
25
- end
@@ -1,35 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Common Job classes for use in specs.
4
-
5
- # Handy for blocking in the middle of processing a job.
6
- class BlockJob < Que::Job
7
- def run
8
- $q1.push nil
9
- $q2.pop
10
- end
11
- end
12
-
13
- RSpec.configure do |config|
14
- config.before { $q1, $q2 = Queue.new, Queue.new }
15
- end
16
-
17
-
18
-
19
- class ErrorJob < Que::Job
20
- def run
21
- raise "ErrorJob!"
22
- end
23
- end
24
-
25
-
26
-
27
- class ArgsJob < Que::Job
28
- def run(*args)
29
- $passed_args = args
30
- end
31
- end
32
-
33
- RSpec.configure do |config|
34
- config.before { $passed_args = nil }
35
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- shared_examples "a Que adapter" do
4
- it "should be able to execute arbitrary SQL and return indifferent hashes" do
5
- result = Que.execute("SELECT 1 AS one")
6
- result.should == [{'one'=>1}]
7
- result.first[:one].should == 1
8
- end
9
-
10
- it "should be able to cast boolean results properly" do
11
- r = Que.execute("SELECT true AS true_value, false AS false_value")
12
- r.should == [{'true_value' => true, 'false_value' => false}]
13
- end
14
-
15
- it "should be able to execute multiple SQL statements in one string" do
16
- Que.execute("SELECT 1 AS one; SELECT 1 AS one")
17
- end
18
-
19
- it "should be able to queue and work a job" do
20
- Que::Job.enqueue
21
- result = Que::Job.work
22
- result[:event].should == :job_worked
23
- result[:job][:job_class].should == 'Que::Job'
24
- end
25
-
26
- it "should yield the same Postgres connection for the duration of the block" do
27
- Que.adapter.checkout do |conn|
28
- conn.should be_a PG::Connection
29
- pid1 = Que.execute "SELECT pg_backend_pid()"
30
- pid2 = Que.execute "SELECT pg_backend_pid()"
31
- pid1.should == pid2
32
- end
33
- end
34
-
35
- it "should allow nested checkouts" do
36
- Que.adapter.checkout do |a|
37
- Que.adapter.checkout do |b|
38
- a.object_id.should == b.object_id
39
- end
40
- end
41
- end
42
- end