disquo 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.travis.yml +11 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +44 -0
- data/LICENSE +13 -0
- data/README.md +57 -0
- data/Rakefile +8 -0
- data/bin/disquo +21 -0
- data/disquo.gemspec +28 -0
- data/lib/disquo/cli.rb +109 -0
- data/lib/disquo/job.rb +47 -0
- data/lib/disquo/worker.rb +122 -0
- data/lib/disquo.rb +66 -0
- data/spec/disquo/job_spec.rb +40 -0
- data/spec/disquo/worker_spec.rb +45 -0
- data/spec/spec_helper.rb +52 -0
- metadata +132 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: 1f9ffa4b76dc43b762c79ff560a981b71d8e6bfb
|
|
4
|
+
data.tar.gz: 4c320fe08f60dd1d9fc18a190e65ef73d06cff64
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 143053e6aa72ef607a876baa1059457e1db32a75e300448529feb3363ec991919b5c70bae3c147b6ac0b9a3095af862fb4c9dd8962bca0b2702943d650c48984
|
|
7
|
+
data.tar.gz: ef07e3fe2b6c36d4a759d9564475881b43a9bb9319016440662e49b3e3de5aed64d912c5995a56b08abe4ddeb04d62f2cca09bf3ec3a3e9874e528e36297d7c1
|
data/.travis.yml
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
language: ruby
|
|
2
|
+
before_install:
|
|
3
|
+
- wget https://github.com/antirez/disque/archive/master.tar.gz -O disque-master.tar.gz
|
|
4
|
+
- tar xf disque-master.tar.gz && cd disque-master/src/ && make && PREFIX=../ make install && cd -
|
|
5
|
+
before_script:
|
|
6
|
+
- ./disque-master/bin/disque-server --daemonize yes
|
|
7
|
+
script:
|
|
8
|
+
- bundle exec rake
|
|
9
|
+
rvm:
|
|
10
|
+
- 2.4
|
|
11
|
+
- 2.3
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
disquo (0.3.0)
|
|
5
|
+
concurrent-ruby
|
|
6
|
+
connection_pool
|
|
7
|
+
disque
|
|
8
|
+
|
|
9
|
+
GEM
|
|
10
|
+
remote: https://rubygems.org/
|
|
11
|
+
specs:
|
|
12
|
+
concurrent-ruby (1.0.5)
|
|
13
|
+
connection_pool (2.2.1)
|
|
14
|
+
diff-lcs (1.3)
|
|
15
|
+
disque (0.0.6)
|
|
16
|
+
redic (~> 1.5.0)
|
|
17
|
+
hiredis (0.6.1)
|
|
18
|
+
rake (12.3.0)
|
|
19
|
+
redic (1.5.0)
|
|
20
|
+
hiredis
|
|
21
|
+
rspec (3.7.0)
|
|
22
|
+
rspec-core (~> 3.7.0)
|
|
23
|
+
rspec-expectations (~> 3.7.0)
|
|
24
|
+
rspec-mocks (~> 3.7.0)
|
|
25
|
+
rspec-core (3.7.0)
|
|
26
|
+
rspec-support (~> 3.7.0)
|
|
27
|
+
rspec-expectations (3.7.0)
|
|
28
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
29
|
+
rspec-support (~> 3.7.0)
|
|
30
|
+
rspec-mocks (3.7.0)
|
|
31
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
32
|
+
rspec-support (~> 3.7.0)
|
|
33
|
+
rspec-support (3.7.0)
|
|
34
|
+
|
|
35
|
+
PLATFORMS
|
|
36
|
+
ruby
|
|
37
|
+
|
|
38
|
+
DEPENDENCIES
|
|
39
|
+
disquo!
|
|
40
|
+
rake
|
|
41
|
+
rspec
|
|
42
|
+
|
|
43
|
+
BUNDLED WITH
|
|
44
|
+
1.16.0
|
data/LICENSE
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Copyright 2017 Black Square Media Ltd
|
|
2
|
+
|
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
you may not use this file except in compliance with the License.
|
|
5
|
+
You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
See the License for the specific language governing permissions and
|
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Disquo
|
|
2
|
+
|
|
3
|
+
[](https://travis-ci.org/bsm/disquo)
|
|
4
|
+
[](https://opensource.org/licenses/Apache-2.0)
|
|
5
|
+
|
|
6
|
+
Minimalist, threaded high-performance Ruby workers on top of [Disque](https://github.com/antirez/disque).
|
|
7
|
+
|
|
8
|
+
## Installation
|
|
9
|
+
|
|
10
|
+
Add this to your Gemfile:
|
|
11
|
+
|
|
12
|
+
```ruby
|
|
13
|
+
gem 'disquo'
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Then execute:
|
|
17
|
+
|
|
18
|
+
```shell
|
|
19
|
+
$ bundle
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Usage
|
|
23
|
+
|
|
24
|
+
Define a job:
|
|
25
|
+
|
|
26
|
+
```ruby
|
|
27
|
+
require 'disquo'
|
|
28
|
+
|
|
29
|
+
class MyJob
|
|
30
|
+
include Disquo::Job
|
|
31
|
+
|
|
32
|
+
job_options queue: "notdefault", ttl: 3600, async: true
|
|
33
|
+
|
|
34
|
+
def perform(msg)
|
|
35
|
+
$stdout.puts "Hello #{msg}!"
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Enqueue with override
|
|
40
|
+
MyJob.enqueue ["World"], ttl: 7200
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Create a worker config file:
|
|
44
|
+
|
|
45
|
+
```yaml
|
|
46
|
+
queues: ["default", "notdefault"]
|
|
47
|
+
concurrency: <%= ENV['NUM_THREADS'] || 20 %>
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Start a worker:
|
|
51
|
+
|
|
52
|
+
```shell
|
|
53
|
+
$ RACK_ENV=production disquo -C config/disquo.yaml -r config/environment.rb
|
|
54
|
+
I, [#12581] INFO -- : Starting worker - queues: ["default", "notdefault"], concurrency: 20
|
|
55
|
+
I, [#12581] INFO -- : Process {"klass":"MyJob","args":["World"]} - thread: 7807s, job: DI8613f71b34be272dff91e63fa576340076f169bf05a0SQ
|
|
56
|
+
...
|
|
57
|
+
```
|
data/Rakefile
ADDED
data/bin/disquo
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
lib_dir = File.expand_path('../../lib', __FILE__)
|
|
4
|
+
$LOAD_PATH.push(lib_dir) unless $LOAD_PATH.include?(lib_dir)
|
|
5
|
+
|
|
6
|
+
require 'disquo/cli'
|
|
7
|
+
|
|
8
|
+
cli = Disquo::CLI.instance
|
|
9
|
+
begin
|
|
10
|
+
cli.parse!
|
|
11
|
+
cli.run!
|
|
12
|
+
rescue ArgumentError => e
|
|
13
|
+
STDERR.puts " ! #{e.message}\n"
|
|
14
|
+
STDERR.puts
|
|
15
|
+
STDERR.puts cli.parser
|
|
16
|
+
exit 1
|
|
17
|
+
rescue => e
|
|
18
|
+
STDERR.puts e.message
|
|
19
|
+
STDERR.puts e.backtrace.join("\n")
|
|
20
|
+
exit 1
|
|
21
|
+
end
|
data/disquo.gemspec
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
|
2
|
+
|
|
3
|
+
Gem::Specification.new do |s|
|
|
4
|
+
s.name = "disquo"
|
|
5
|
+
s.version = "0.3.0"
|
|
6
|
+
s.platform = Gem::Platform::RUBY
|
|
7
|
+
|
|
8
|
+
s.licenses = ["Apache-2.0"]
|
|
9
|
+
s.summary = "Concurrent background workers on top of Disque"
|
|
10
|
+
s.description = "Concurrent background workers on top of Disque"
|
|
11
|
+
|
|
12
|
+
s.authors = ["Dimitrij Denissenko"]
|
|
13
|
+
s.email = "dimitrij@blacksquaremedia.com"
|
|
14
|
+
s.homepage = "https://github.com/bsm/disquo"
|
|
15
|
+
|
|
16
|
+
s.executables = ['disquo']
|
|
17
|
+
s.files = `git ls-files`.split("\n")
|
|
18
|
+
s.test_files = `git ls-files -- spec/*`.split("\n")
|
|
19
|
+
s.require_paths = ["lib"]
|
|
20
|
+
s.required_ruby_version = ">= 2.3.0"
|
|
21
|
+
|
|
22
|
+
s.add_dependency 'disque'
|
|
23
|
+
s.add_dependency 'connection_pool'
|
|
24
|
+
s.add_dependency 'concurrent-ruby'
|
|
25
|
+
|
|
26
|
+
s.add_development_dependency 'rake'
|
|
27
|
+
s.add_development_dependency 'rspec'
|
|
28
|
+
end
|
data/lib/disquo/cli.rb
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
require 'singleton'
|
|
2
|
+
require 'optparse'
|
|
3
|
+
require 'yaml'
|
|
4
|
+
require 'erb'
|
|
5
|
+
|
|
6
|
+
module Disquo
|
|
7
|
+
class CLI
|
|
8
|
+
include Singleton
|
|
9
|
+
|
|
10
|
+
DEFAULT_OPTIONS = {
|
|
11
|
+
config: nil,
|
|
12
|
+
queues: ["default"],
|
|
13
|
+
concurrency: 10,
|
|
14
|
+
disque_nodes: ["127.0.0.1:7711"],
|
|
15
|
+
disque_opts: {},
|
|
16
|
+
pool_size: nil, # auto
|
|
17
|
+
pool_timeout: 1,
|
|
18
|
+
logfile: nil, # STDOUT
|
|
19
|
+
wait_time: 1,
|
|
20
|
+
wait_count: 100,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
attr_reader :opts
|
|
24
|
+
|
|
25
|
+
def initialize
|
|
26
|
+
@opts = DEFAULT_OPTIONS.dup
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def parse!(argv = ARGV)
|
|
30
|
+
parser.parse!(argv)
|
|
31
|
+
|
|
32
|
+
# Check config file
|
|
33
|
+
if opts[:config] && !File.exist?(opts[:config])
|
|
34
|
+
raise ArgumentError, "Unable to find config file in #{opts[:config]}"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Load config file
|
|
38
|
+
if opts[:config]
|
|
39
|
+
conf = YAML.load(ERB.new(IO.read(opts[:config])).result)
|
|
40
|
+
unless conf.is_a?(Hash)
|
|
41
|
+
raise ArgumentError, "File in #{opts[:config]} does not contain a valid configuration"
|
|
42
|
+
end
|
|
43
|
+
conf.each do |key, value|
|
|
44
|
+
opts[key.to_sym] = value
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Set pool size using concurrency
|
|
49
|
+
opts[:pool_size] ||= opts[:concurrency] + 5
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def run!
|
|
53
|
+
return if @worker
|
|
54
|
+
|
|
55
|
+
require opts[:require] if opts[:require]
|
|
56
|
+
require 'disquo/worker'
|
|
57
|
+
|
|
58
|
+
Disquo.logger = ::Logger.new(opts[:logfile]) if opts[:logfile]
|
|
59
|
+
disque = Disquo.connect \
|
|
60
|
+
nodes: opts[:disque_nodes],
|
|
61
|
+
opts: opts[:disque_opts],
|
|
62
|
+
pool_size: opts[:pool_size],
|
|
63
|
+
pool_timeout: opts[:pool_timeout]
|
|
64
|
+
|
|
65
|
+
Signal.trap("TERM") { shutdown }
|
|
66
|
+
Signal.trap("INT") { shutdown }
|
|
67
|
+
|
|
68
|
+
@worker = Disquo::Worker.new disque,
|
|
69
|
+
queues: opts[:queues],
|
|
70
|
+
concurrency: opts[:concurrency],
|
|
71
|
+
wait_time: opts[:wait_time],
|
|
72
|
+
wait_count: opts[:wait_count]
|
|
73
|
+
@worker.run
|
|
74
|
+
@worker.wait
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def shutdown
|
|
78
|
+
return unless @worker
|
|
79
|
+
|
|
80
|
+
@worker.shutdown
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def parser
|
|
84
|
+
@parser ||= begin
|
|
85
|
+
op = OptionParser.new do |o|
|
|
86
|
+
o.on '-C', '--config FILE', "YAML config file to load" do |v|
|
|
87
|
+
@opts[:config] = v
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
o.on '-r', '--require [PATH|DIR]', "File to require" do |v|
|
|
91
|
+
@opts[:require] = v
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
o.on '-L', '--logfile PATH', "path to writable logfile" do |v|
|
|
95
|
+
@opts[:logfile] = v
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
op.banner = "disquo [options]"
|
|
100
|
+
op.on_tail "-h", "--help", "Show help" do
|
|
101
|
+
$stdout.puts parser
|
|
102
|
+
exit 1
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
data/lib/disquo/job.rb
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
module Disquo::Job
|
|
2
|
+
attr_accessor :job_id, :queue, :disque
|
|
3
|
+
|
|
4
|
+
def self.included(base)
|
|
5
|
+
base.extend(ClassMethods)
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
# Indicate to disque that this job is still in progress
|
|
9
|
+
def working!
|
|
10
|
+
disque.with {|cn| cn.call :working, job_id } if disque && job_id
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
module ClassMethods
|
|
14
|
+
|
|
15
|
+
# Configures the Job
|
|
16
|
+
#
|
|
17
|
+
# @param [Hash] opts the options to enqueue the message with.
|
|
18
|
+
# @option opts [String] :queue a specific queue name. Default: "default"
|
|
19
|
+
# @option opts [Numeric] :timeout number of seconds to wait for server to `:replicate` level (if no `:async` is specified). Default: 10s
|
|
20
|
+
# @option opts [Integer] :replicate number of nodes the job should be replicated to
|
|
21
|
+
# @option opts [Integer] :retry seconds after, if no ACK is received, the job is put into the queue again for delivery. Default: 5m
|
|
22
|
+
# @option opts [Integer] :ttl max job life-time in seconds. Default: 24h
|
|
23
|
+
# @option opts [Integer] :maxlen specifies that if there are already `maxlen` messages queued for the specified queue name, the message is refused.
|
|
24
|
+
# @option opts [Boolean] :async asks the server to let the command return ASAP and replicate the job to other nodes in the background.
|
|
25
|
+
def job_options(opts = {})
|
|
26
|
+
@job_options ||= {}
|
|
27
|
+
@job_options.update(opts) unless opts.nil? || opts.empty?
|
|
28
|
+
@job_options
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Enqueues the job
|
|
32
|
+
# @param [Array] args arguments to pass to #perform
|
|
33
|
+
# @param [Hash] opts the options to enqueue the message with (see Disquo::Job::ClassMethods.job_options)
|
|
34
|
+
# @option opts [Interger] :delay is the number of seconds that should elapse before the job is queued by any server
|
|
35
|
+
def enqueue(args = [], opts = {})
|
|
36
|
+
opts = job_options.merge(opts)
|
|
37
|
+
queue = opts.delete(:queue) || Disquo::DEFAULT_QUEUE
|
|
38
|
+
timeout = ((opts.delete(:timeout) || 10).to_f * 1000).to_i
|
|
39
|
+
payload = Disquo.dump_job(name, args)
|
|
40
|
+
|
|
41
|
+
Disquo.disque.with do |conn|
|
|
42
|
+
conn.push(queue, payload, timeout, opts)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
require 'disquo'
|
|
2
|
+
require 'concurrent'
|
|
3
|
+
require 'concurrent/executor/fixed_thread_pool'
|
|
4
|
+
|
|
5
|
+
class Disquo::Worker
|
|
6
|
+
attr_reader :disque, :queues, :wait_time, :wait_count
|
|
7
|
+
|
|
8
|
+
# Init a new worker instance
|
|
9
|
+
# @param [ConnectionPool] disque client connection pool
|
|
10
|
+
# @param [Hash] options
|
|
11
|
+
# @option [Array<String>] :queues queues to watch. Default: ["default"]
|
|
12
|
+
# @option [Integer] :concurrency the number of concurrent threads. Default: 10
|
|
13
|
+
# @option [Numeric] :wait_time maximum time (in seconds) to block for when retrieving next batch. Default: 1s
|
|
14
|
+
# @option [Integer] :wait_count the minimum number of jobs to wait for when retrieving next batch. Default: 100
|
|
15
|
+
def initialize(disque, queues: [Disquo::DEFAULT_QUEUE], concurrency: 10, wait_time: 1, wait_count: 100)
|
|
16
|
+
@disque = disque
|
|
17
|
+
@queues = Array(queues)
|
|
18
|
+
@threads = Concurrent::FixedThreadPool.new(concurrency)
|
|
19
|
+
@wait_time = wait_time
|
|
20
|
+
@wait_count = wait_count
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Run starts the worker
|
|
24
|
+
def run
|
|
25
|
+
Disquo.logger.info "Starting worker - queues: #{queues.inspect}, concurrency: #{@threads.max_length}"
|
|
26
|
+
|
|
27
|
+
begin
|
|
28
|
+
run_cycle
|
|
29
|
+
rescue => e
|
|
30
|
+
handle_exception(e)
|
|
31
|
+
end until @stopped
|
|
32
|
+
|
|
33
|
+
@threads.shutdown
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Blocks until worker is stopped
|
|
37
|
+
def wait(timeout = nil)
|
|
38
|
+
Disquo.logger.info "Waiting for worker shutdown"
|
|
39
|
+
@threads.wait_for_termination(timeout)
|
|
40
|
+
Disquo.logger.info "Shutdown complete"
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Stops the worker
|
|
44
|
+
def shutdown
|
|
45
|
+
return false if @stopped
|
|
46
|
+
|
|
47
|
+
@stopped = true
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def run_cycle
|
|
53
|
+
jobs = next_batch
|
|
54
|
+
|
|
55
|
+
until @stopped || jobs.empty?
|
|
56
|
+
job = jobs.shift
|
|
57
|
+
perform(*job)
|
|
58
|
+
end
|
|
59
|
+
ensure
|
|
60
|
+
requeue(jobs) unless jobs.nil? || jobs.empty?
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def next_batch
|
|
64
|
+
jobs = disque.with do |cn|
|
|
65
|
+
cn.fetch from: queues, timeout: (wait_time*1000).to_i, count: wait_count
|
|
66
|
+
end
|
|
67
|
+
@is_down = nil
|
|
68
|
+
Array(jobs)
|
|
69
|
+
rescue => e
|
|
70
|
+
if !@is_down
|
|
71
|
+
@is_down = true
|
|
72
|
+
handle_exception(e, message: 'Error retrieving jobs:')
|
|
73
|
+
end
|
|
74
|
+
sleep(1)
|
|
75
|
+
[]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def perform(queue, job_id, payload)
|
|
79
|
+
@threads.post do
|
|
80
|
+
thread_id = Thread.current.object_id.to_s(36)
|
|
81
|
+
Disquo.logger.info { "Process #{payload} - thread: #{thread_id}, job: #{job_id}" }
|
|
82
|
+
|
|
83
|
+
begin
|
|
84
|
+
class_name, args = Disquo.load_job(payload)
|
|
85
|
+
|
|
86
|
+
job = Object.const_get(class_name).new
|
|
87
|
+
job.disque = disque
|
|
88
|
+
job.queue = queue
|
|
89
|
+
job.job_id = job_id
|
|
90
|
+
job.perform(*args)
|
|
91
|
+
rescue => e
|
|
92
|
+
handle_exception e, message: "Error processing #{payload} - thread: #{thread_id}, job: #{job_id}:"
|
|
93
|
+
|
|
94
|
+
disque.with {|cn| cn.call :nack, job_id }
|
|
95
|
+
return
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
begin
|
|
99
|
+
disque.with {|cn| cn.call :ackjob, job_id }
|
|
100
|
+
rescue => e
|
|
101
|
+
handle_exception e, message: "Error ACKing #{payload} - thread: #{thread_id}, job: #{job_id}:"
|
|
102
|
+
return
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def requeue(jobs)
|
|
108
|
+
ids = jobs.map {|_, job_id, _| job_id }
|
|
109
|
+
disque.with {|cn| cn.call :enqueue, *ids }
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def handle_exception(e, opts = {})
|
|
113
|
+
lines = [
|
|
114
|
+
opts[:message],
|
|
115
|
+
"#{e.class.name}: #{e.message}",
|
|
116
|
+
e.backtrace
|
|
117
|
+
].compact.flatten
|
|
118
|
+
|
|
119
|
+
Disquo.logger.error lines.join("\n")
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
end
|
data/lib/disquo.rb
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
require 'disque'
|
|
2
|
+
require 'connection_pool'
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'logger'
|
|
5
|
+
|
|
6
|
+
module Disquo
|
|
7
|
+
DEFAULT_QUEUE = "default".freeze
|
|
8
|
+
|
|
9
|
+
# Configure disque with a block.
|
|
10
|
+
#
|
|
11
|
+
# @example:
|
|
12
|
+
# Disquo.configure do |c|
|
|
13
|
+
# c.disque = c.connect(nodes: ["10.0.0.1:7711", "10.0.0.2:7711", "10.0.0.3:7711"])
|
|
14
|
+
# end
|
|
15
|
+
def self.configure(&block)
|
|
16
|
+
block.call(self)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# @option [Array<String>|String] nodes a list of disque nodes. Default: ["127.0.0.1:7711"]
|
|
20
|
+
# @option [Hash] opts additional disque connection options.
|
|
21
|
+
# @option [Numeric] pool_timeout disque connection pool timeout in seconds. Default: 1.0
|
|
22
|
+
# @option [Integer] pool_size the number of slots in the connection pool. Default: 5
|
|
23
|
+
#
|
|
24
|
+
# @return [ConnectionPool] disque client connection pool
|
|
25
|
+
def self.connect(nodes: ["127.0.0.1:7711"], opts: {}, pool_size: 5, pool_timeout: 1)
|
|
26
|
+
ConnectionPool.new timeout: pool_timeout, size: pool_size do
|
|
27
|
+
Disque.new(nodes, opts)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @return [ConnectionPool] disque client connection pool
|
|
32
|
+
def self.disque
|
|
33
|
+
@disque ||= connect
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @param [ConnectionPool] disque client connection pool
|
|
37
|
+
def self.disque=(pool)
|
|
38
|
+
@disque = pool
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# @return [Logger] returns the logger instance
|
|
42
|
+
def self.logger
|
|
43
|
+
@logger ||= Logger.new(STDOUT)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @param [Logger] log logger instance to use
|
|
47
|
+
def self.logger=(log)
|
|
48
|
+
@logger = log
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# @param [String] payload
|
|
52
|
+
# @return [Array<Class, Array>] job class and argument
|
|
53
|
+
def self.load_job(payload)
|
|
54
|
+
JSON.load(payload).values_at("klass", "args")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# @param [String] class_name class name
|
|
58
|
+
# @param [Array] arguments
|
|
59
|
+
# @return [String] serialised job
|
|
60
|
+
def self.dump_job(class_name, args)
|
|
61
|
+
JSON.dump "klass" => class_name, "args" => Array(args)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
require 'disquo/job'
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
RSpec.describe Disquo::Job do
|
|
4
|
+
|
|
5
|
+
it "should enqueue jobs" do
|
|
6
|
+
jid1 = TestJob.enqueue ["foo", 1]
|
|
7
|
+
expect(qlen).to eq(1)
|
|
8
|
+
expect(jid1).to match(/^D\w+/)
|
|
9
|
+
expect(show(jid1)).to include(
|
|
10
|
+
"id" => jid1,
|
|
11
|
+
"queue" => "__disquo_test__",
|
|
12
|
+
"nacks" => 0,
|
|
13
|
+
"delay" => 0,
|
|
14
|
+
"repl" => 1,
|
|
15
|
+
"state" => "queued",
|
|
16
|
+
"body" => %({"klass":"TestJob","args":["foo",1]}),
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
jid2 = TestJob.enqueue ["bar"], delay: 600
|
|
20
|
+
expect(qlen).to eq(1)
|
|
21
|
+
expect(jid2).to match(/^D\w+/)
|
|
22
|
+
expect(show(jid2)).to include(
|
|
23
|
+
"id" => jid2,
|
|
24
|
+
"queue" => "__disquo_test__",
|
|
25
|
+
"nacks" => 0,
|
|
26
|
+
"delay" => 600,
|
|
27
|
+
"repl" => 1,
|
|
28
|
+
"state" => "active",
|
|
29
|
+
"body" => %({"klass":"TestJob","args":["bar"]}),
|
|
30
|
+
)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
it "should have instance attributes" do
|
|
34
|
+
job1, job2 = TestJob.new, TestJob.new
|
|
35
|
+
job1.job_id = "JOB1"
|
|
36
|
+
expect(job1.job_id).to eq("JOB1")
|
|
37
|
+
expect(job2.job_id).to be_nil
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
RSpec.describe Disquo::Worker do
|
|
4
|
+
|
|
5
|
+
subject do
|
|
6
|
+
described_class.new Disquo.connect, wait_time: 0.1, wait_count: 11, queues: Disquo::TEST::QUEUE
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
it "should run/process/shutdown" do
|
|
10
|
+
runner = Thread.new { subject.run }
|
|
11
|
+
|
|
12
|
+
# seed 200 jobs
|
|
13
|
+
200.times {|n| TestJob.enqueue(n) }
|
|
14
|
+
wait_for { !qlen.zero? }
|
|
15
|
+
|
|
16
|
+
# ensure runner processes them all
|
|
17
|
+
wait_for { qlen.zero? }
|
|
18
|
+
expect(runner).to be_alive
|
|
19
|
+
expect(qlen).to eq(0)
|
|
20
|
+
|
|
21
|
+
# ask runner to quit
|
|
22
|
+
expect(subject.shutdown).to be_truthy
|
|
23
|
+
expect(subject.shutdown).to be_falsey
|
|
24
|
+
|
|
25
|
+
# wait for runner to exit
|
|
26
|
+
subject.wait
|
|
27
|
+
expect(runner).not_to be_alive
|
|
28
|
+
|
|
29
|
+
# check what's been performed
|
|
30
|
+
expect(Disquo::TEST::PERFORMED.size).to eq(200)
|
|
31
|
+
expect(Disquo::TEST::PERFORMED.last).to include(
|
|
32
|
+
klass: "TestJob",
|
|
33
|
+
queue: "__disquo_test__",
|
|
34
|
+
)
|
|
35
|
+
expect(Disquo::TEST::PERFORMED.last[:job_id]).to match(/^D\w+/)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def wait_for
|
|
39
|
+
20.times do
|
|
40
|
+
break if yield
|
|
41
|
+
sleep(0.01)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
end
|
data/spec/spec_helper.rb
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
require 'rspec'
|
|
2
|
+
require 'disquo'
|
|
3
|
+
require 'disquo/worker'
|
|
4
|
+
|
|
5
|
+
module Disquo::TEST
|
|
6
|
+
QUEUE = "__disquo_test__"
|
|
7
|
+
PERFORMED = []
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
class TestJob
|
|
11
|
+
include Disquo::Job
|
|
12
|
+
|
|
13
|
+
job_options queue: Disquo::TEST::QUEUE, async: true
|
|
14
|
+
|
|
15
|
+
def perform(*args)
|
|
16
|
+
Disquo::TEST::PERFORMED.push(klass: self.class.name, args: args, queue: queue, job_id: job_id)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
helpers = Module.new do
|
|
21
|
+
|
|
22
|
+
def qlen
|
|
23
|
+
Disquo.disque.with do |conn|
|
|
24
|
+
conn.call :qlen, Disquo::TEST::QUEUE
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def show(job_id)
|
|
29
|
+
Disquo.disque.with do |conn|
|
|
30
|
+
pairs = conn.call :show, job_id
|
|
31
|
+
Hash[*pairs]
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
RSpec.configure do |c|
|
|
38
|
+
c.include helpers
|
|
39
|
+
|
|
40
|
+
c.before :each do
|
|
41
|
+
Disquo.logger = Logger.new(nil)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
c.after :each do
|
|
45
|
+
Disquo::TEST::PERFORMED.clear
|
|
46
|
+
Disquo.disque.with do |conn|
|
|
47
|
+
_, ids = conn.call :jscan, 0, :count, 10000, :queue, Disquo::TEST::QUEUE
|
|
48
|
+
conn.call :deljob, *ids unless ids.empty?
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: disquo
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.3.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Dimitrij Denissenko
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2017-11-23 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: disque
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '0'
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - ">="
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: connection_pool
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '0'
|
|
34
|
+
type: :runtime
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: concurrent-ruby
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: rake
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - ">="
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '0'
|
|
62
|
+
type: :development
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - ">="
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: '0'
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: rspec
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - ">="
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: '0'
|
|
76
|
+
type: :development
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - ">="
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '0'
|
|
83
|
+
description: Concurrent background workers on top of Disque
|
|
84
|
+
email: dimitrij@blacksquaremedia.com
|
|
85
|
+
executables:
|
|
86
|
+
- disquo
|
|
87
|
+
extensions: []
|
|
88
|
+
extra_rdoc_files: []
|
|
89
|
+
files:
|
|
90
|
+
- ".travis.yml"
|
|
91
|
+
- Gemfile
|
|
92
|
+
- Gemfile.lock
|
|
93
|
+
- LICENSE
|
|
94
|
+
- README.md
|
|
95
|
+
- Rakefile
|
|
96
|
+
- bin/disquo
|
|
97
|
+
- disquo.gemspec
|
|
98
|
+
- lib/disquo.rb
|
|
99
|
+
- lib/disquo/cli.rb
|
|
100
|
+
- lib/disquo/job.rb
|
|
101
|
+
- lib/disquo/worker.rb
|
|
102
|
+
- spec/disquo/job_spec.rb
|
|
103
|
+
- spec/disquo/worker_spec.rb
|
|
104
|
+
- spec/spec_helper.rb
|
|
105
|
+
homepage: https://github.com/bsm/disquo
|
|
106
|
+
licenses:
|
|
107
|
+
- Apache-2.0
|
|
108
|
+
metadata: {}
|
|
109
|
+
post_install_message:
|
|
110
|
+
rdoc_options: []
|
|
111
|
+
require_paths:
|
|
112
|
+
- lib
|
|
113
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - ">="
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: 2.3.0
|
|
118
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
119
|
+
requirements:
|
|
120
|
+
- - ">="
|
|
121
|
+
- !ruby/object:Gem::Version
|
|
122
|
+
version: '0'
|
|
123
|
+
requirements: []
|
|
124
|
+
rubyforge_project:
|
|
125
|
+
rubygems_version: 2.6.11
|
|
126
|
+
signing_key:
|
|
127
|
+
specification_version: 4
|
|
128
|
+
summary: Concurrent background workers on top of Disque
|
|
129
|
+
test_files:
|
|
130
|
+
- spec/disquo/job_spec.rb
|
|
131
|
+
- spec/disquo/worker_spec.rb
|
|
132
|
+
- spec/spec_helper.rb
|