queue_classic_pg2 3.2.0.RC1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +9 -0
  3. data/.travis.yml +19 -0
  4. data/CONTRIBUTING.md +17 -0
  5. data/Gemfile +9 -0
  6. data/LICENSE.txt +20 -0
  7. data/README.md +326 -0
  8. data/Rakefile +14 -0
  9. data/changelog +146 -0
  10. data/lib/generators/queue_classic/install_generator.rb +36 -0
  11. data/lib/generators/queue_classic/templates/add_queue_classic.rb +9 -0
  12. data/lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb +9 -0
  13. data/lib/generators/queue_classic/templates/update_queue_classic_3_0_2.rb +11 -0
  14. data/lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb +9 -0
  15. data/lib/queue_classic/config.rb +85 -0
  16. data/lib/queue_classic/conn_adapter.rb +111 -0
  17. data/lib/queue_classic/queue.rb +119 -0
  18. data/lib/queue_classic/railtie.rb +9 -0
  19. data/lib/queue_classic/setup.rb +58 -0
  20. data/lib/queue_classic/tasks.rb +49 -0
  21. data/lib/queue_classic/version.rb +3 -0
  22. data/lib/queue_classic/worker.rb +166 -0
  23. data/lib/queue_classic.rb +122 -0
  24. data/queue_classic.gemspec +24 -0
  25. data/sql/create_table.sql +25 -0
  26. data/sql/ddl.sql +78 -0
  27. data/sql/downgrade_from_3_0_0.sql +2 -0
  28. data/sql/downgrade_from_3_1_0.sql +1 -0
  29. data/sql/drop_ddl.sql +3 -0
  30. data/sql/update_to_3_0_0.sql +17 -0
  31. data/sql/update_to_3_1_0.sql +9 -0
  32. data/test/benchmark_test.rb +39 -0
  33. data/test/config_test.rb +121 -0
  34. data/test/helper.rb +61 -0
  35. data/test/helper.sql +25 -0
  36. data/test/lib/queue_classic_rails_connection_test.rb +43 -0
  37. data/test/lib/queue_classic_test.rb +42 -0
  38. data/test/queue_test.rb +208 -0
  39. data/test/worker_test.rb +219 -0
  40. metadata +112 -0
@@ -0,0 +1,208 @@
1
+ require_relative 'helper'
2
+
3
+ class QueueTest < QCTest
4
+
5
+ ResetError = Class.new(PG::Error)
6
+
7
+ def test_enqueue
8
+ QC.enqueue("Klass.method")
9
+ end
10
+
11
+ def test_respond_to
12
+ assert_equal(true, QC.respond_to?(:enqueue))
13
+ end
14
+
15
+ def test_lock
16
+ queue = QC::Queue.new("queue_classic_jobs")
17
+ queue.enqueue("Klass.method")
18
+ job = queue.lock
19
+ # See helper.rb for more information about the large initial id number.
20
+ assert_equal((2**34).to_s, job[:id])
21
+ assert_equal("queue_classic_jobs", job[:q_name])
22
+ assert_equal("Klass.method", job[:method])
23
+ assert_equal([], job[:args])
24
+ end
25
+
26
+ def test_lock_when_empty
27
+ assert_nil(QC.lock)
28
+ end
29
+
30
+ def test_lock_with_future_job_with_enqueue_in
31
+ now = Time.now
32
+ QC.enqueue_in(2, "Klass.method")
33
+ assert_nil QC.lock
34
+ sleep 2
35
+ job = QC.lock
36
+ assert_equal("Klass.method", job[:method])
37
+ assert_equal([], job[:args])
38
+ assert_equal((now + 2).to_i, job[:scheduled_at].to_i)
39
+ end
40
+
41
+ def test_lock_with_future_job_with_enqueue_at_with_a_time_object
42
+ future = Time.now + 2
43
+ QC.enqueue_at(future, "Klass.method")
44
+ assert_nil QC.lock
45
+ until Time.now >= future do sleep 0.1 end
46
+ job = QC.lock
47
+ assert_equal("Klass.method", job[:method])
48
+ assert_equal([], job[:args])
49
+ assert_equal(future.to_i, job[:scheduled_at].to_i)
50
+ end
51
+
52
+ def test_lock_with_future_job_with_enqueue_at_with_a_float_timestamp
53
+ offset = (Time.now + 2).to_f
54
+ QC.enqueue_at(offset, "Klass.method")
55
+ assert_nil QC.lock
56
+ sleep 2
57
+ job = QC.lock
58
+ assert_equal("Klass.method", job[:method])
59
+ assert_equal([], job[:args])
60
+ end
61
+
62
+ def test_count
63
+ QC.enqueue("Klass.method")
64
+ assert_equal(1, QC.count)
65
+ end
66
+
67
+ def test_delete
68
+ QC.enqueue("Klass.method")
69
+ assert_equal(1, QC.count)
70
+ QC.delete(QC.lock[:id])
71
+ assert_equal(0, QC.count)
72
+ end
73
+
74
+ def test_delete_all
75
+ QC.enqueue("Klass.method")
76
+ QC.enqueue("Klass.method")
77
+ assert_equal(2, QC.count)
78
+ QC.delete_all
79
+ assert_equal(0, QC.count)
80
+ end
81
+
82
+ def test_delete_all_by_queue_name
83
+ p_queue = QC::Queue.new("priority_queue")
84
+ s_queue = QC::Queue.new("secondary_queue")
85
+ p_queue.enqueue("Klass.method")
86
+ s_queue.enqueue("Klass.method")
87
+ assert_equal(1, p_queue.count)
88
+ assert_equal(1, s_queue.count)
89
+ p_queue.delete_all
90
+ assert_equal(0, p_queue.count)
91
+ assert_equal(1, s_queue.count)
92
+ end
93
+
94
+ def test_queue_instance
95
+ queue = QC::Queue.new("queue_classic_jobs")
96
+ queue.enqueue("Klass.method")
97
+ assert_equal(1, queue.count)
98
+ queue.delete(queue.lock[:id])
99
+ assert_equal(0, queue.count)
100
+ end
101
+
102
+ def test_repair_after_error
103
+ queue = QC::Queue.new("queue_classic_jobs")
104
+ queue.conn_adapter = QC::ConnAdapter.new
105
+ queue.enqueue("Klass.method")
106
+ assert_equal(1, queue.count)
107
+ conn = queue.conn_adapter.connection
108
+ def conn.exec(*args); raise(PG::Error); end
109
+ def conn.reset(*args); raise(ResetError) end
110
+ # We ensure that the reset method is called on the connection.
111
+ assert_raises(PG::Error, ResetError) {queue.enqueue("Klass.other_method")}
112
+ queue.conn_adapter.disconnect
113
+ end
114
+
115
+ def test_custom_default_queue
116
+ queue_class = Class.new do
117
+ attr_accessor :jobs
118
+ def enqueue(method, *args)
119
+ @jobs ||= []
120
+ @jobs << method
121
+ end
122
+ end
123
+
124
+ queue_instance = queue_class.new
125
+ QC.default_queue = queue_instance
126
+
127
+ QC.enqueue("Klass.method1")
128
+ QC.enqueue("Klass.method2")
129
+
130
+ assert_equal ["Klass.method1", "Klass.method2"], queue_instance.jobs
131
+ ensure
132
+ QC.default_queue = nil
133
+ end
134
+
135
+ def test_multi_threaded_server_can_specified_connection
136
+ adapter1 = QC::ConnAdapter.new
137
+ adapter2 = QC::ConnAdapter.new
138
+ q1 = q2 = nil
139
+
140
+ QC.default_conn_adapter = adapter1
141
+
142
+ t1 = Thread.new do
143
+ QC.default_conn_adapter = adapter1
144
+ q1 = QC::Queue.new('queue1').conn_adapter
145
+ end
146
+
147
+ t2 = Thread.new do
148
+ QC.default_conn_adapter = adapter2
149
+ q2 = QC::Queue.new('queue2').conn_adapter
150
+ end
151
+
152
+ t1.join
153
+ t2.join
154
+
155
+ assert_equal adapter1, q1
156
+ assert_equal adapter2, q2
157
+ end
158
+
159
+ def test_multi_threaded_server_each_thread_acquires_unique_connection
160
+ q1 = q2 = nil
161
+
162
+ t1 = Thread.new do
163
+ q1 = QC::Queue.new('queue1').conn_adapter
164
+ end
165
+
166
+ t2 = Thread.new do
167
+ q2 = QC::Queue.new('queue2').conn_adapter
168
+ end
169
+
170
+ t1.join
171
+ t2.join
172
+
173
+ refute_equal q1, q2
174
+ end
175
+
176
+ def test_enqueue_triggers_notify
177
+ adapter = QC.default_conn_adapter
178
+ adapter.execute('LISTEN "' + QC.queue + '"')
179
+ adapter.send(:drain_notify)
180
+
181
+ msgs = adapter.send(:wait_for_notify, 0.25)
182
+ assert_equal(0, msgs.length)
183
+
184
+ QC.enqueue("Klass.method")
185
+ msgs = adapter.send(:wait_for_notify, 0.25)
186
+ assert_equal(1, msgs.length)
187
+ end
188
+
189
+ def test_enqueue_returns_job_id
190
+ enqueued_job = QC.enqueue("Klass.method")
191
+ locked_job = QC.lock
192
+ assert_equal enqueued_job, "id" => locked_job[:id]
193
+ end
194
+
195
+ def test_enqueue_in_returns_job_id
196
+ enqueued_job = QC.enqueue_in(1, "Klass.method")
197
+ sleep 1
198
+ locked_job = QC.lock
199
+ assert_equal enqueued_job, "id" => locked_job[:id]
200
+ end
201
+
202
+ def test_enqueue_at_returns_job_id
203
+ enqueued_job = QC.enqueue_at(Time.now + 1, "Klass.method")
204
+ sleep 1
205
+ locked_job = QC.lock
206
+ assert_equal enqueued_job, "id" => locked_job[:id]
207
+ end
208
+ end
@@ -0,0 +1,219 @@
1
+ require_relative 'helper'
2
+
3
+ module TestObject
4
+ extend self
5
+ def no_args; return nil; end
6
+ def one_arg(a); return a; end
7
+ def two_args(a,b); return [a,b]; end
8
+ def forty_two; OpenStruct.new(number: 42); end
9
+ end
10
+
11
+ # This not only allows me to test what happens
12
+ # when a failure occurs but it also demonstrates
13
+ # how to override the worker to handle failures the way
14
+ # you want.
15
+ class TestWorker < QC::Worker
16
+ attr_accessor :failed_count
17
+
18
+ def initialize(args={})
19
+ super(args.merge(:connection => QC.default_conn_adapter.connection))
20
+ @failed_count = 0
21
+ end
22
+
23
+ def handle_failure(job,e)
24
+ @failed_count += 1
25
+ super
26
+ end
27
+ end
28
+
29
+ class WorkerTest < QCTest
30
+ def test_work
31
+ QC.enqueue("TestObject.no_args")
32
+ worker = TestWorker.new
33
+ assert_equal(1, QC.count)
34
+ worker.work
35
+ assert_equal(0, QC.count)
36
+ assert_equal(0, worker.failed_count)
37
+ end
38
+
39
+ def test_failed_job
40
+ QC.enqueue("TestObject.not_a_method")
41
+ worker = TestWorker.new
42
+ capture_stderr_output { worker.work }
43
+ assert_equal(1, worker.failed_count)
44
+ end
45
+
46
+ def test_failed_job_is_logged
47
+ output = capture_stderr_output do
48
+ QC.enqueue("TestObject.not_a_method")
49
+ TestWorker.new.work
50
+ end
51
+ assert(output.include?("#<NoMethodError: undefined method `not_a_method'"))
52
+ end
53
+
54
+ def test_log_yield
55
+ output = capture_debug_output do
56
+ QC.log_yield(:action => "test") do
57
+ 0 == 1
58
+ end
59
+ end
60
+ expected_output = /lib=queue-classic action=test elapsed=\d*/
61
+ assert_match(expected_output, output, "=== debug output ===\n #{output}")
62
+ end
63
+
64
+ def test_log
65
+ output = capture_debug_output do
66
+ QC.log(:action => "test")
67
+ end
68
+ expected_output = /lib=queue-classic action=test/
69
+ assert_match(expected_output, output, "=== debug output ===\n #{output}")
70
+ end
71
+
72
+ def test_work_with_no_args
73
+ QC.enqueue("TestObject.no_args")
74
+ worker = TestWorker.new
75
+ r = worker.work
76
+ assert_nil(r)
77
+ assert_equal(0, worker.failed_count)
78
+ end
79
+
80
+ def test_work_with_one_arg
81
+ QC.enqueue("TestObject.one_arg", "1")
82
+ worker = TestWorker.new
83
+ r = worker.work
84
+ assert_equal("1", r)
85
+ assert_equal(0, worker.failed_count)
86
+ end
87
+
88
+ def test_work_with_two_args
89
+ QC.enqueue("TestObject.two_args", "1", 2)
90
+ worker = TestWorker.new
91
+ r = worker.work
92
+ assert_equal(["1", 2], r)
93
+ assert_equal(0, worker.failed_count)
94
+ end
95
+
96
+ def test_work_custom_queue
97
+ p_queue = QC::Queue.new("priority_queue")
98
+ p_queue.enqueue("TestObject.two_args", "1", 2)
99
+ worker = TestWorker.new(q_name: "priority_queue")
100
+ r = worker.work
101
+ assert_equal(["1", 2], r)
102
+ assert_equal(0, worker.failed_count)
103
+ end
104
+
105
+ def test_worker_listens_on_chan
106
+ p_queue = QC::Queue.new("priority_queue")
107
+ # Use a new connection because the default connection
108
+ # will be locked by the sleeping worker.
109
+ p_queue.conn_adapter = QC::ConnAdapter.new
110
+ # The wait interval is extreme to demonstrate
111
+ # that the worker is in fact being activated by a NOTIFY.
112
+ worker = TestWorker.new(:q_name => "priority_queue", :wait_interval => 100)
113
+ t = Thread.new do
114
+ r = worker.work
115
+ assert_equal(["1", 2], r)
116
+ assert_equal(0, worker.failed_count)
117
+ end
118
+ sleep(0.5) #Give the thread some time to start the worker.
119
+ p_queue.enqueue("TestObject.two_args", "1", 2)
120
+ p_queue.conn_adapter.disconnect
121
+ t.join
122
+ end
123
+
124
+ def test_worker_uses_one_conn
125
+ skip "This test is broken and needs to be fixed."
126
+
127
+ QC.enqueue("TestObject.no_args")
128
+ worker = TestWorker.new
129
+ worker.work
130
+ assert_equal(
131
+ 1,
132
+ QC.default_conn_adapter.execute("SELECT count(*) from pg_stat_activity WHERE datname = current_database()")["count"].to_i,
133
+ "Multiple connections found -- are there open connections to #{ QC.default_conn_adapter.send(:db_url) } in other terminals?"
134
+ )
135
+ end
136
+
137
+ def test_worker_can_work_multiple_queues
138
+ p_queue = QC::Queue.new("priority_queue")
139
+ p_queue.enqueue("TestObject.two_args", "1", 2)
140
+
141
+ s_queue = QC::Queue.new("secondary_queue")
142
+ s_queue.enqueue("TestObject.two_args", "1", 2)
143
+
144
+ worker = TestWorker.new(:q_names => ["priority_queue", "secondary_queue"])
145
+
146
+ 2.times do
147
+ r = worker.work
148
+ assert_equal(["1", 2], r)
149
+ assert_equal(0, worker.failed_count)
150
+ end
151
+ end
152
+
153
+ def test_worker_works_multiple_queue_left_to_right
154
+ l_queue = QC::Queue.new("left_queue")
155
+ r_queue = QC::Queue.new("right_queue")
156
+
157
+ 3.times { l_queue.enqueue("TestObject.two_args", "1", 2) }
158
+ 3.times { r_queue.enqueue("TestObject.two_args", "1", 2) }
159
+
160
+ worker = TestWorker.new(:q_names => ["left_queue", "right_queue"])
161
+
162
+ worker.work
163
+ assert_equal(2, l_queue.count)
164
+ assert_equal(3, r_queue.count)
165
+
166
+ worker.work
167
+ assert_equal(1, l_queue.count)
168
+ assert_equal(3, r_queue.count)
169
+ end
170
+
171
+ def test_work_with_more_complex_construct
172
+ QC.enqueue("TestObject.forty_two.number")
173
+ worker = TestWorker.new
174
+ r = worker.work
175
+ assert_equal(42, r)
176
+ assert_equal(0, worker.failed_count)
177
+ end
178
+
179
+ def test_init_worker_with_arg
180
+ with_database 'postgres:///invalid' do
181
+ conn = PG.connect(dbname: 'queue_classic_test')
182
+ QC::Worker.new connection: conn
183
+
184
+ conn.close
185
+ end
186
+ end
187
+
188
+ def test_init_worker_with_database_url
189
+ with_database ENV['DATABASE_URL'] || ENV['QC_DATABASE_URL'] do
190
+ worker = QC::Worker.new
191
+ QC.enqueue("TestObject.no_args")
192
+ worker.lock_job
193
+
194
+ QC.default_conn_adapter.disconnect
195
+ end
196
+ end
197
+
198
+ def test_init_worker_without_conn
199
+ with_database nil do
200
+ assert_raises(ArgumentError) { QC::Worker.new }
201
+ end
202
+ end
203
+
204
+ private
205
+
206
+ def with_database(url)
207
+ original_conn_adapter = QC.default_conn_adapter
208
+ original_database_url = ENV['DATABASE_URL']
209
+ original_qc_database_url = ENV['QC_DATABASE_URL']
210
+
211
+ ENV['DATABASE_URL'] = ENV['QC_DATABASE_URL'] = url
212
+ QC.default_conn_adapter = nil
213
+ yield
214
+ ensure
215
+ ENV['DATABASE_URL'] = original_database_url
216
+ ENV['QC_DATABASE_URL'] = original_qc_database_url
217
+ QC.default_conn_adapter = original_conn_adapter
218
+ end
219
+ end
metadata ADDED
@@ -0,0 +1,112 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: queue_classic_pg2
3
+ version: !ruby/object:Gem::Version
4
+ version: 3.2.0.RC1
5
+ platform: ruby
6
+ authors:
7
+ - Ryan Smith (♠ ace hacker)
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-01-18 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: pg
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0.17'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '2.0'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '0.17'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '2.0'
33
+ description: 'Support for Postgres2 : queue_classic is a queueing library for Ruby
34
+ apps. (Rails, Sinatra, Etc...) queue_classic features asynchronous job polling,
35
+ database maintained locks and no ridiculous dependencies. As a matter of fact, queue_classic
36
+ only requires pg.'
37
+ email: r@32k.io
38
+ executables: []
39
+ extensions: []
40
+ extra_rdoc_files: []
41
+ files:
42
+ - ".gitignore"
43
+ - ".travis.yml"
44
+ - CONTRIBUTING.md
45
+ - Gemfile
46
+ - LICENSE.txt
47
+ - README.md
48
+ - Rakefile
49
+ - changelog
50
+ - lib/generators/queue_classic/install_generator.rb
51
+ - lib/generators/queue_classic/templates/add_queue_classic.rb
52
+ - lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb
53
+ - lib/generators/queue_classic/templates/update_queue_classic_3_0_2.rb
54
+ - lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb
55
+ - lib/queue_classic.rb
56
+ - lib/queue_classic/config.rb
57
+ - lib/queue_classic/conn_adapter.rb
58
+ - lib/queue_classic/queue.rb
59
+ - lib/queue_classic/railtie.rb
60
+ - lib/queue_classic/setup.rb
61
+ - lib/queue_classic/tasks.rb
62
+ - lib/queue_classic/version.rb
63
+ - lib/queue_classic/worker.rb
64
+ - queue_classic.gemspec
65
+ - sql/create_table.sql
66
+ - sql/ddl.sql
67
+ - sql/downgrade_from_3_0_0.sql
68
+ - sql/downgrade_from_3_1_0.sql
69
+ - sql/drop_ddl.sql
70
+ - sql/update_to_3_0_0.sql
71
+ - sql/update_to_3_1_0.sql
72
+ - test/benchmark_test.rb
73
+ - test/config_test.rb
74
+ - test/helper.rb
75
+ - test/helper.sql
76
+ - test/lib/queue_classic_rails_connection_test.rb
77
+ - test/lib/queue_classic_test.rb
78
+ - test/queue_test.rb
79
+ - test/worker_test.rb
80
+ homepage: https://github.com/PatrickLef/queue_classic
81
+ licenses:
82
+ - MIT
83
+ metadata: {}
84
+ post_install_message:
85
+ rdoc_options: []
86
+ require_paths:
87
+ - lib
88
+ required_ruby_version: !ruby/object:Gem::Requirement
89
+ requirements:
90
+ - - ">="
91
+ - !ruby/object:Gem::Version
92
+ version: '0'
93
+ required_rubygems_version: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - ">"
96
+ - !ruby/object:Gem::Version
97
+ version: 1.3.1
98
+ requirements: []
99
+ rubyforge_project:
100
+ rubygems_version: 2.7.7
101
+ signing_key:
102
+ specification_version: 4
103
+ summary: Simple, efficient worker queue for Ruby & PostgreSQL.
104
+ test_files:
105
+ - test/benchmark_test.rb
106
+ - test/config_test.rb
107
+ - test/helper.rb
108
+ - test/helper.sql
109
+ - test/lib/queue_classic_rails_connection_test.rb
110
+ - test/lib/queue_classic_test.rb
111
+ - test/queue_test.rb
112
+ - test/worker_test.rb