rabbit_jobs 0.2.0.pre4 → 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -0
- data/bin/rj_scheduler +23 -0
- data/bin/rj_worker +24 -0
- data/examples/client +50 -0
- data/examples/configuration.rb +1 -1
- data/examples/worker +28 -0
- data/lib/rabbit_jobs.rb +89 -5
- data/lib/rabbit_jobs/amqp_helper.rb +81 -29
- data/lib/rabbit_jobs/configuration.rb +28 -11
- data/lib/rabbit_jobs/job.rb +22 -5
- data/lib/rabbit_jobs/publisher.rb +25 -32
- data/lib/rabbit_jobs/scheduler.rb +32 -27
- data/lib/rabbit_jobs/tasks.rb +211 -29
- data/lib/rabbit_jobs/util.rb +1 -1
- data/lib/rabbit_jobs/version.rb +1 -1
- data/lib/rabbit_jobs/worker.rb +39 -36
- data/rabbit_jobs.gemspec +2 -1
- data/spec/integration/publisher_spec.rb +27 -14
- data/spec/integration/scheduler_spec.rb +2 -2
- data/spec/integration/worker_spec.rb +33 -7
- data/spec/unit/configuration_spec.rb +2 -2
- metadata +33 -13
data/lib/rabbit_jobs/util.rb
CHANGED
data/lib/rabbit_jobs/version.rb
CHANGED
data/lib/rabbit_jobs/worker.rb
CHANGED
@@ -27,62 +27,61 @@ module RabbitJobs
|
|
27
27
|
@queues || ['default']
|
28
28
|
end
|
29
29
|
|
30
|
-
# Subscribes to
|
30
|
+
# Subscribes to queue and working on jobs
|
31
31
|
def work(time = 0)
|
32
32
|
return false unless startup
|
33
33
|
|
34
34
|
$0 = self.process_name || "rj_worker (#{queues.join(',')})"
|
35
35
|
|
36
36
|
processed_count = 0
|
37
|
-
RJ.logger.info("Connecting to amqp...")
|
38
37
|
|
39
38
|
begin
|
40
|
-
|
41
|
-
AmqpHelper.prepare_channel
|
42
|
-
AMQP.channel.prefetch(1)
|
43
|
-
|
39
|
+
RJ.run do
|
44
40
|
check_shutdown = Proc.new {
|
45
41
|
if @shutdown
|
46
|
-
RJ.
|
47
|
-
RJ.logger.info "Stopping worker ##{Process.pid}..."
|
48
|
-
|
49
|
-
AMQP.connection.disconnect {
|
42
|
+
RJ.stop {
|
50
43
|
File.delete(self.pidfile) if self.pidfile && File.exists?(self.pidfile)
|
51
|
-
RJ.logger.info "##{Process.pid} stopped."
|
52
|
-
RJ.logger.close
|
53
|
-
|
54
|
-
EM.stop {
|
55
|
-
exit!
|
56
|
-
}
|
57
44
|
}
|
45
|
+
RJ.logger.info "rj_worker[##{Process.pid}] processed jobs: #{processed_count}."
|
46
|
+
RJ.logger.info "rj_worker[##{Process.pid}] stopped."
|
47
|
+
# RJ.logger.close
|
48
|
+
# exit!
|
58
49
|
end
|
59
50
|
}
|
60
51
|
|
61
|
-
|
62
|
-
queue = AMQP.channel.queue(RJ.config.queue_name(routing_key), RJ.config[:queues][routing_key])
|
63
|
-
|
64
|
-
RJ.logger.info "Worker ##{Process.pid} <= #{RJ.config.queue_name(routing_key)}"
|
65
|
-
|
66
|
-
explicit_ack = !!RJ.config[:queues][routing_key][:ack]
|
67
|
-
|
68
|
-
queue.subscribe(ack: explicit_ack) do |metadata, payload|
|
69
|
-
@job = RJ::Job.parse(payload)
|
52
|
+
AmqpHelper.prepare_channel
|
70
53
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
54
|
+
queues.each do |routing_key|
|
55
|
+
AMQP.channel.prefetch(1)
|
56
|
+
AMQP.channel.queue(RJ.config.queue_name(routing_key), RJ.config[:queues][routing_key]) { |queue, declare_ok|
|
57
|
+
RJ.logger.info "rj_worker[##{Process.pid}] subscribed to #{RJ.config.queue_name(routing_key)} (#{declare_ok.to_i + 1})"
|
58
|
+
|
59
|
+
explicit_ack = !!RJ.config[:queues][routing_key][:ack]
|
60
|
+
|
61
|
+
queue.subscribe(ack: explicit_ack) do |metadata, payload|
|
62
|
+
@job = RJ::Job.parse(payload)
|
63
|
+
|
64
|
+
if @job.is_a?(Symbol)
|
65
|
+
# case @job
|
66
|
+
# when :not_found
|
67
|
+
# when :parsing_error
|
68
|
+
# when :error
|
69
|
+
# end
|
70
|
+
metadata.ack if explicit_ack
|
76
71
|
else
|
77
|
-
@job.
|
78
|
-
|
72
|
+
if @job.expired?
|
73
|
+
RJ.logger.warn "rj_worker[##{Process.pid}] Job expired: #{@job.to_ruby_string}"
|
74
|
+
else
|
75
|
+
@job.run_perform
|
76
|
+
processed_count += 1
|
77
|
+
end
|
78
|
+
|
79
|
+
metadata.ack if explicit_ack
|
79
80
|
end
|
80
81
|
|
81
|
-
|
82
|
+
check_shutdown.call
|
82
83
|
end
|
83
|
-
|
84
|
-
check_shutdown.call
|
85
|
-
end
|
84
|
+
}
|
86
85
|
end
|
87
86
|
|
88
87
|
if time > 0
|
@@ -95,6 +94,8 @@ module RabbitJobs
|
|
95
94
|
EM.add_periodic_timer(1) do
|
96
95
|
check_shutdown.call
|
97
96
|
end
|
97
|
+
|
98
|
+
RJ.logger.info "rj_worker[##{Process.pid}] started."
|
98
99
|
end
|
99
100
|
rescue
|
100
101
|
error = $!
|
@@ -125,6 +126,8 @@ module RabbitJobs
|
|
125
126
|
Process.daemon(true)
|
126
127
|
end
|
127
128
|
|
129
|
+
count = RJ._run_after_fork_callbacks
|
130
|
+
|
128
131
|
self.worker_pid ||= Process.pid
|
129
132
|
|
130
133
|
if self.pidfile
|
data/rabbit_jobs.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |gem|
|
|
10
10
|
gem.homepage = ""
|
11
11
|
gem.date = Time.now.strftime('%Y-%m-%d')
|
12
12
|
|
13
|
-
gem.executables =
|
13
|
+
gem.executables = %w(rj_worker rj_scheduler)
|
14
14
|
gem.files = `git ls-files`.split("\n")
|
15
15
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
16
|
gem.name = "rabbit_jobs"
|
@@ -20,4 +20,5 @@ Gem::Specification.new do |gem|
|
|
20
20
|
gem.add_dependency "amqp", "~> 0.9"
|
21
21
|
gem.add_dependency "rake"
|
22
22
|
gem.add_dependency "rufus-scheduler", "~> 2.0"
|
23
|
+
gem.add_dependency "rails", ">= 3.0"
|
23
24
|
end
|
@@ -7,42 +7,55 @@ describe RabbitJobs::Publisher do
|
|
7
7
|
|
8
8
|
before(:each) do
|
9
9
|
RabbitJobs.configure do |c|
|
10
|
-
c.
|
10
|
+
c.server 'amqp://localhost/'
|
11
11
|
c.queue 'rspec_queue'
|
12
12
|
c.queue 'rspec_queue2'
|
13
13
|
c.queue 'rspec_queue3'
|
14
14
|
end
|
15
15
|
|
16
|
-
|
16
|
+
RJ.purge_queue(:rspec_queue, :rspec_queue2, :rspec_queue3)
|
17
17
|
end
|
18
18
|
|
19
19
|
it 'should publish message to queue' do
|
20
|
-
|
21
|
-
|
20
|
+
RJ.publish(TestJob, 'some', 'other', 'params')
|
21
|
+
RJ.purge_queue('rspec_queue').should == 1
|
22
22
|
end
|
23
23
|
|
24
24
|
it 'should accept symbol as queue name' do
|
25
|
-
|
26
|
-
|
25
|
+
RJ.publish_to(:rspec_queue, TestJob)
|
26
|
+
RJ.purge_queue('rspec_queue').should == 1
|
27
27
|
end
|
28
28
|
|
29
29
|
it 'purge_queue should accept many queues' do
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
30
|
+
RJ.publish_to(:rspec_queue, TestJob)
|
31
|
+
RJ.publish_to(:rspec_queue2, TestJob)
|
32
|
+
RJ.publish_to(:rspec_queue3, TestJob)
|
33
|
+
RJ.purge_queue(:rspec_queue, :rspec_queue2, :rspec_queue3).should == 3
|
34
34
|
end
|
35
35
|
|
36
36
|
it 'should publish job with *params' do
|
37
|
-
|
38
|
-
|
37
|
+
RJ.publish_to(:rspec_queue, JobWithArgsArray, 'first value', :some_symbol, 123, 'and string')
|
38
|
+
RJ.purge_queue(:rspec_queue).should == 1
|
39
39
|
end
|
40
40
|
|
41
41
|
it 'should publish 1000 messages in one second' do
|
42
|
+
count = 1000
|
43
|
+
published = 0
|
42
44
|
time = Benchmark.measure {
|
43
|
-
|
45
|
+
RJ.run {
|
46
|
+
count.times {
|
47
|
+
RJ.publish_to(:rspec_queue, TestJob) {
|
48
|
+
published += 1
|
49
|
+
if published == count
|
50
|
+
RJ.purge_queue(:rspec_queue, :rspec_queue2, :rspec_queue3) { |removed|
|
51
|
+
removed.should == 1000
|
52
|
+
RJ.stop
|
53
|
+
}
|
54
|
+
end
|
55
|
+
}
|
56
|
+
}
|
57
|
+
}
|
44
58
|
}
|
45
|
-
RabbitJobs::Publisher.purge_queue(:rspec_queue, :rspec_queue2, :rspec_queue3).should == 1000
|
46
59
|
puts time
|
47
60
|
end
|
48
61
|
end
|
@@ -12,7 +12,7 @@ describe RabbitJobs::Scheduler do
|
|
12
12
|
scheduler.work(10) # work for 1 second
|
13
13
|
|
14
14
|
RJ.config.queue 'default', RJ::Configuration::DEFAULT_QUEUE_PARAMS
|
15
|
-
puts "messages queued: " +
|
16
|
-
|
15
|
+
puts "messages queued: " + RJ.purge_queue('default').to_s
|
16
|
+
RJ.purge_queue('default').should == 0
|
17
17
|
end
|
18
18
|
end
|
@@ -10,15 +10,36 @@ describe RabbitJobs::Worker do
|
|
10
10
|
c.queue 'rspec_durable_queue', auto_delete: false, durable: true, ack: true
|
11
11
|
end
|
12
12
|
|
13
|
-
RJ
|
13
|
+
RJ.run {
|
14
|
+
RJ::Publisher.purge_queue('rspec_durable_queue') {
|
15
|
+
count = 5
|
16
|
+
5.times {
|
17
|
+
RabbitJobs.publish(PrintTimeJob, Time.now) {
|
18
|
+
count -= 1
|
19
|
+
if count <= 0
|
20
|
+
Timecop.freeze(Time.now - 4600) {
|
21
|
+
count = 5
|
22
|
+
5.times {
|
23
|
+
RabbitJobs.publish(JobWithExpire) {
|
24
|
+
count -= 1
|
25
|
+
if count <= 0
|
26
|
+
RabbitJobs.publish(JobWithErrorHook) {
|
27
|
+
RJ.stop
|
28
|
+
}
|
29
|
+
end
|
30
|
+
}
|
31
|
+
}
|
32
|
+
}
|
33
|
+
end
|
34
|
+
}
|
35
|
+
}
|
36
|
+
}
|
37
|
+
}
|
14
38
|
|
15
|
-
5.times { RabbitJobs.publish(PrintTimeJob, Time.now) }
|
16
|
-
Timecop.freeze(Time.now - 4600) { 5.times { RabbitJobs.publish(JobWithExpire) } }
|
17
|
-
1.times { RabbitJobs.publish(JobWithErrorHook) }
|
18
39
|
worker = RabbitJobs::Worker.new
|
19
40
|
|
20
41
|
worker.work(1) # work for 1 second
|
21
|
-
RJ::Publisher.purge_queue('rspec_durable_queue')
|
42
|
+
RJ.run { RJ::Publisher.purge_queue('rspec_durable_queue') { RJ.stop } }
|
22
43
|
end
|
23
44
|
|
24
45
|
it 'should allow to publish jobs from worker' do
|
@@ -27,9 +48,14 @@ describe RabbitJobs::Worker do
|
|
27
48
|
c.queue 'rspec_durable_queue', auto_delete: false, durable: true, ack: true
|
28
49
|
end
|
29
50
|
|
30
|
-
RJ
|
51
|
+
RJ.run {
|
52
|
+
RJ::Publisher.purge_queue('rspec_durable_queue') {
|
53
|
+
RabbitJobs.publish(JobWithPublish) {
|
54
|
+
RJ.stop
|
55
|
+
}
|
56
|
+
}
|
57
|
+
}
|
31
58
|
|
32
|
-
RabbitJobs.publish(JobWithPublish)
|
33
59
|
worker = RabbitJobs::Worker.new
|
34
60
|
|
35
61
|
worker.work(1) # work for 1 second
|
@@ -59,7 +59,7 @@ describe RabbitJobs::Configuration do
|
|
59
59
|
it 'use default config' do
|
60
60
|
RabbitJobs.config.to_hash.should == {
|
61
61
|
error_log: true,
|
62
|
-
url: "amqp://localhost",
|
62
|
+
url: "amqp://localhost/",
|
63
63
|
prefix: "rabbit_jobs",
|
64
64
|
queues: {
|
65
65
|
"default" => {
|
@@ -73,7 +73,7 @@ describe RabbitJobs::Configuration do
|
|
73
73
|
|
74
74
|
it 'returns settings on some methods' do
|
75
75
|
RabbitJobs.config.error_log == true
|
76
|
-
RabbitJobs.config.
|
76
|
+
RabbitJobs.config.servers.should == ['amqp://localhost/']
|
77
77
|
RabbitJobs.config.routing_keys.should == []
|
78
78
|
RabbitJobs.config.prefix.should == 'rabbit_jobs'
|
79
79
|
RabbitJobs.config.queue_name('default').should == 'rabbit_jobs#default'
|
metadata
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rabbit_jobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: '0.3'
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Pavel Lazureykis
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-01-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: amqp
|
16
|
-
requirement: &
|
16
|
+
requirement: &16813080 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0.9'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *16813080
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: rake
|
27
|
-
requirement: &
|
27
|
+
requirement: &16811500 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *16811500
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: rufus-scheduler
|
38
|
-
requirement: &
|
38
|
+
requirement: &16809880 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ~>
|
@@ -43,11 +43,24 @@ dependencies:
|
|
43
43
|
version: '2.0'
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *16809880
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rails
|
49
|
+
requirement: &16808180 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.0'
|
55
|
+
type: :runtime
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *16808180
|
47
58
|
description: Background jobs on RabbitMQ
|
48
59
|
email:
|
49
60
|
- lazureykis@gmail.com
|
50
|
-
executables:
|
61
|
+
executables:
|
62
|
+
- rj_worker
|
63
|
+
- rj_scheduler
|
51
64
|
extensions: []
|
52
65
|
extra_rdoc_files: []
|
53
66
|
files:
|
@@ -57,7 +70,11 @@ files:
|
|
57
70
|
- LICENSE
|
58
71
|
- README.md
|
59
72
|
- Rakefile
|
73
|
+
- bin/rj_scheduler
|
74
|
+
- bin/rj_worker
|
75
|
+
- examples/client
|
60
76
|
- examples/configuration.rb
|
77
|
+
- examples/worker
|
61
78
|
- lib/rabbit_jobs.rb
|
62
79
|
- lib/rabbit_jobs/amqp_helper.rb
|
63
80
|
- lib/rabbit_jobs/configuration.rb
|
@@ -98,13 +115,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
98
115
|
version: '0'
|
99
116
|
segments:
|
100
117
|
- 0
|
101
|
-
hash:
|
118
|
+
hash: -3349151042287683745
|
102
119
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
103
120
|
none: false
|
104
121
|
requirements:
|
105
|
-
- - ! '
|
122
|
+
- - ! '>='
|
106
123
|
- !ruby/object:Gem::Version
|
107
|
-
version:
|
124
|
+
version: '0'
|
125
|
+
segments:
|
126
|
+
- 0
|
127
|
+
hash: -3349151042287683745
|
108
128
|
requirements: []
|
109
129
|
rubyforge_project:
|
110
130
|
rubygems_version: 1.8.11
|