disquo 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.travis.yml +11 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +44 -0
- data/LICENSE +13 -0
- data/README.md +57 -0
- data/Rakefile +8 -0
- data/bin/disquo +21 -0
- data/disquo.gemspec +28 -0
- data/lib/disquo/cli.rb +109 -0
- data/lib/disquo/job.rb +47 -0
- data/lib/disquo/worker.rb +122 -0
- data/lib/disquo.rb +66 -0
- data/spec/disquo/job_spec.rb +40 -0
- data/spec/disquo/worker_spec.rb +45 -0
- data/spec/spec_helper.rb +52 -0
- metadata +132 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 1f9ffa4b76dc43b762c79ff560a981b71d8e6bfb
|
4
|
+
data.tar.gz: 4c320fe08f60dd1d9fc18a190e65ef73d06cff64
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 143053e6aa72ef607a876baa1059457e1db32a75e300448529feb3363ec991919b5c70bae3c147b6ac0b9a3095af862fb4c9dd8962bca0b2702943d650c48984
|
7
|
+
data.tar.gz: ef07e3fe2b6c36d4a759d9564475881b43a9bb9319016440662e49b3e3de5aed64d912c5995a56b08abe4ddeb04d62f2cca09bf3ec3a3e9874e528e36297d7c1
|
data/.travis.yml
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
language: ruby
|
2
|
+
before_install:
|
3
|
+
- wget https://github.com/antirez/disque/archive/master.tar.gz -O disque-master.tar.gz
|
4
|
+
- tar xf disque-master.tar.gz && cd disque-master/src/ && make && PREFIX=../ make install && cd -
|
5
|
+
before_script:
|
6
|
+
- ./disque-master/bin/disque-server --daemonize yes
|
7
|
+
script:
|
8
|
+
- bundle exec rake
|
9
|
+
rvm:
|
10
|
+
- 2.4
|
11
|
+
- 2.3
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
disquo (0.3.0)
|
5
|
+
concurrent-ruby
|
6
|
+
connection_pool
|
7
|
+
disque
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
concurrent-ruby (1.0.5)
|
13
|
+
connection_pool (2.2.1)
|
14
|
+
diff-lcs (1.3)
|
15
|
+
disque (0.0.6)
|
16
|
+
redic (~> 1.5.0)
|
17
|
+
hiredis (0.6.1)
|
18
|
+
rake (12.3.0)
|
19
|
+
redic (1.5.0)
|
20
|
+
hiredis
|
21
|
+
rspec (3.7.0)
|
22
|
+
rspec-core (~> 3.7.0)
|
23
|
+
rspec-expectations (~> 3.7.0)
|
24
|
+
rspec-mocks (~> 3.7.0)
|
25
|
+
rspec-core (3.7.0)
|
26
|
+
rspec-support (~> 3.7.0)
|
27
|
+
rspec-expectations (3.7.0)
|
28
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
29
|
+
rspec-support (~> 3.7.0)
|
30
|
+
rspec-mocks (3.7.0)
|
31
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
32
|
+
rspec-support (~> 3.7.0)
|
33
|
+
rspec-support (3.7.0)
|
34
|
+
|
35
|
+
PLATFORMS
|
36
|
+
ruby
|
37
|
+
|
38
|
+
DEPENDENCIES
|
39
|
+
disquo!
|
40
|
+
rake
|
41
|
+
rspec
|
42
|
+
|
43
|
+
BUNDLED WITH
|
44
|
+
1.16.0
|
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright 2017 Black Square Media Ltd
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
# Disquo
|
2
|
+
|
3
|
+
[![Build Status](https://travis-ci.org/bsm/disquo.png?branch=master)](https://travis-ci.org/bsm/disquo)
|
4
|
+
[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
|
5
|
+
|
6
|
+
Minimalist, threaded high-performance Ruby workers on top of [Disque](https://github.com/antirez/disque).
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Add this to your Gemfile:
|
11
|
+
|
12
|
+
```ruby
|
13
|
+
gem 'disquo'
|
14
|
+
```
|
15
|
+
|
16
|
+
Then execute:
|
17
|
+
|
18
|
+
```shell
|
19
|
+
$ bundle
|
20
|
+
```
|
21
|
+
|
22
|
+
## Usage
|
23
|
+
|
24
|
+
Define a job:
|
25
|
+
|
26
|
+
```ruby
|
27
|
+
require 'disquo'
|
28
|
+
|
29
|
+
class MyJob
|
30
|
+
include Disquo::Job
|
31
|
+
|
32
|
+
job_options queue: "notdefault", ttl: 3600, async: true
|
33
|
+
|
34
|
+
def perform(msg)
|
35
|
+
$stdout.puts "Hello #{msg}!"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# Enqueue with override
|
40
|
+
MyJob.enqueue ["World"], ttl: 7200
|
41
|
+
```
|
42
|
+
|
43
|
+
Create a worker config file:
|
44
|
+
|
45
|
+
```yaml
|
46
|
+
queues: ["default", "notdefault"]
|
47
|
+
concurrency: <%= ENV['NUM_THREADS'] || 20 %>
|
48
|
+
```
|
49
|
+
|
50
|
+
Start a worker:
|
51
|
+
|
52
|
+
```shell
|
53
|
+
$ RACK_ENV=production disquo -C config/disquo.yaml -r config/environment.rb
|
54
|
+
I, [#12581] INFO -- : Starting worker - queues: ["default", "notdefault"], concurrency: 20
|
55
|
+
I, [#12581] INFO -- : Process {"klass":"MyJob","args":["World"]} - thread: 7807s, job: DI8613f71b34be272dff91e63fa576340076f169bf05a0SQ
|
56
|
+
...
|
57
|
+
```
|
data/Rakefile
ADDED
data/bin/disquo
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
lib_dir = File.expand_path('../../lib', __FILE__)
|
4
|
+
$LOAD_PATH.push(lib_dir) unless $LOAD_PATH.include?(lib_dir)
|
5
|
+
|
6
|
+
require 'disquo/cli'
|
7
|
+
|
8
|
+
cli = Disquo::CLI.instance
|
9
|
+
begin
|
10
|
+
cli.parse!
|
11
|
+
cli.run!
|
12
|
+
rescue ArgumentError => e
|
13
|
+
STDERR.puts " ! #{e.message}\n"
|
14
|
+
STDERR.puts
|
15
|
+
STDERR.puts cli.parser
|
16
|
+
exit 1
|
17
|
+
rescue => e
|
18
|
+
STDERR.puts e.message
|
19
|
+
STDERR.puts e.backtrace.join("\n")
|
20
|
+
exit 1
|
21
|
+
end
|
data/disquo.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = "disquo"
|
5
|
+
s.version = "0.3.0"
|
6
|
+
s.platform = Gem::Platform::RUBY
|
7
|
+
|
8
|
+
s.licenses = ["Apache-2.0"]
|
9
|
+
s.summary = "Concurrent background workers on top of Disque"
|
10
|
+
s.description = "Concurrent background workers on top of Disque"
|
11
|
+
|
12
|
+
s.authors = ["Dimitrij Denissenko"]
|
13
|
+
s.email = "dimitrij@blacksquaremedia.com"
|
14
|
+
s.homepage = "https://github.com/bsm/disquo"
|
15
|
+
|
16
|
+
s.executables = ['disquo']
|
17
|
+
s.files = `git ls-files`.split("\n")
|
18
|
+
s.test_files = `git ls-files -- spec/*`.split("\n")
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
s.required_ruby_version = ">= 2.3.0"
|
21
|
+
|
22
|
+
s.add_dependency 'disque'
|
23
|
+
s.add_dependency 'connection_pool'
|
24
|
+
s.add_dependency 'concurrent-ruby'
|
25
|
+
|
26
|
+
s.add_development_dependency 'rake'
|
27
|
+
s.add_development_dependency 'rspec'
|
28
|
+
end
|
data/lib/disquo/cli.rb
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
require 'optparse'
|
3
|
+
require 'yaml'
|
4
|
+
require 'erb'
|
5
|
+
|
6
|
+
module Disquo
|
7
|
+
class CLI
|
8
|
+
include Singleton
|
9
|
+
|
10
|
+
DEFAULT_OPTIONS = {
|
11
|
+
config: nil,
|
12
|
+
queues: ["default"],
|
13
|
+
concurrency: 10,
|
14
|
+
disque_nodes: ["127.0.0.1:7711"],
|
15
|
+
disque_opts: {},
|
16
|
+
pool_size: nil, # auto
|
17
|
+
pool_timeout: 1,
|
18
|
+
logfile: nil, # STDOUT
|
19
|
+
wait_time: 1,
|
20
|
+
wait_count: 100,
|
21
|
+
}
|
22
|
+
|
23
|
+
attr_reader :opts
|
24
|
+
|
25
|
+
def initialize
|
26
|
+
@opts = DEFAULT_OPTIONS.dup
|
27
|
+
end
|
28
|
+
|
29
|
+
def parse!(argv = ARGV)
|
30
|
+
parser.parse!(argv)
|
31
|
+
|
32
|
+
# Check config file
|
33
|
+
if opts[:config] && !File.exist?(opts[:config])
|
34
|
+
raise ArgumentError, "Unable to find config file in #{opts[:config]}"
|
35
|
+
end
|
36
|
+
|
37
|
+
# Load config file
|
38
|
+
if opts[:config]
|
39
|
+
conf = YAML.load(ERB.new(IO.read(opts[:config])).result)
|
40
|
+
unless conf.is_a?(Hash)
|
41
|
+
raise ArgumentError, "File in #{opts[:config]} does not contain a valid configuration"
|
42
|
+
end
|
43
|
+
conf.each do |key, value|
|
44
|
+
opts[key.to_sym] = value
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Set pool size using concurrency
|
49
|
+
opts[:pool_size] ||= opts[:concurrency] + 5
|
50
|
+
end
|
51
|
+
|
52
|
+
def run!
|
53
|
+
return if @worker
|
54
|
+
|
55
|
+
require opts[:require] if opts[:require]
|
56
|
+
require 'disquo/worker'
|
57
|
+
|
58
|
+
Disquo.logger = ::Logger.new(opts[:logfile]) if opts[:logfile]
|
59
|
+
disque = Disquo.connect \
|
60
|
+
nodes: opts[:disque_nodes],
|
61
|
+
opts: opts[:disque_opts],
|
62
|
+
pool_size: opts[:pool_size],
|
63
|
+
pool_timeout: opts[:pool_timeout]
|
64
|
+
|
65
|
+
Signal.trap("TERM") { shutdown }
|
66
|
+
Signal.trap("INT") { shutdown }
|
67
|
+
|
68
|
+
@worker = Disquo::Worker.new disque,
|
69
|
+
queues: opts[:queues],
|
70
|
+
concurrency: opts[:concurrency],
|
71
|
+
wait_time: opts[:wait_time],
|
72
|
+
wait_count: opts[:wait_count]
|
73
|
+
@worker.run
|
74
|
+
@worker.wait
|
75
|
+
end
|
76
|
+
|
77
|
+
def shutdown
|
78
|
+
return unless @worker
|
79
|
+
|
80
|
+
@worker.shutdown
|
81
|
+
end
|
82
|
+
|
83
|
+
def parser
|
84
|
+
@parser ||= begin
|
85
|
+
op = OptionParser.new do |o|
|
86
|
+
o.on '-C', '--config FILE', "YAML config file to load" do |v|
|
87
|
+
@opts[:config] = v
|
88
|
+
end
|
89
|
+
|
90
|
+
o.on '-r', '--require [PATH|DIR]', "File to require" do |v|
|
91
|
+
@opts[:require] = v
|
92
|
+
end
|
93
|
+
|
94
|
+
o.on '-L', '--logfile PATH', "path to writable logfile" do |v|
|
95
|
+
@opts[:logfile] = v
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
op.banner = "disquo [options]"
|
100
|
+
op.on_tail "-h", "--help", "Show help" do
|
101
|
+
$stdout.puts parser
|
102
|
+
exit 1
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
data/lib/disquo/job.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
module Disquo::Job
|
2
|
+
attr_accessor :job_id, :queue, :disque
|
3
|
+
|
4
|
+
def self.included(base)
|
5
|
+
base.extend(ClassMethods)
|
6
|
+
end
|
7
|
+
|
8
|
+
# Indicate to disque that this job is still in progress
|
9
|
+
def working!
|
10
|
+
disque.with {|cn| cn.call :working, job_id } if disque && job_id
|
11
|
+
end
|
12
|
+
|
13
|
+
module ClassMethods
|
14
|
+
|
15
|
+
# Configures the Job
|
16
|
+
#
|
17
|
+
# @param [Hash] opts the options to enqueue the message with.
|
18
|
+
# @option opts [String] :queue a specific queue name. Default: "default"
|
19
|
+
# @option opts [Numeric] :timeout number of seconds to wait for server to `:replicate` level (if no `:async` is specified). Default: 10s
|
20
|
+
# @option opts [Integer] :replicate number of nodes the job should be replicated to
|
21
|
+
# @option opts [Integer] :retry seconds after, if no ACK is received, the job is put into the queue again for delivery. Default: 5m
|
22
|
+
# @option opts [Integer] :ttl max job life-time in seconds. Default: 24h
|
23
|
+
# @option opts [Integer] :maxlen specifies that if there are already `maxlen` messages queued for the specified queue name, the message is refused.
|
24
|
+
# @option opts [Boolean] :async asks the server to let the command return ASAP and replicate the job to other nodes in the background.
|
25
|
+
def job_options(opts = {})
|
26
|
+
@job_options ||= {}
|
27
|
+
@job_options.update(opts) unless opts.nil? || opts.empty?
|
28
|
+
@job_options
|
29
|
+
end
|
30
|
+
|
31
|
+
# Enqueues the job
|
32
|
+
# @param [Array] args arguments to pass to #perform
|
33
|
+
# @param [Hash] opts the options to enqueue the message with (see Disquo::Job::ClassMethods.job_options)
|
34
|
+
# @option opts [Interger] :delay is the number of seconds that should elapse before the job is queued by any server
|
35
|
+
def enqueue(args = [], opts = {})
|
36
|
+
opts = job_options.merge(opts)
|
37
|
+
queue = opts.delete(:queue) || Disquo::DEFAULT_QUEUE
|
38
|
+
timeout = ((opts.delete(:timeout) || 10).to_f * 1000).to_i
|
39
|
+
payload = Disquo.dump_job(name, args)
|
40
|
+
|
41
|
+
Disquo.disque.with do |conn|
|
42
|
+
conn.push(queue, payload, timeout, opts)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
require 'disquo'
|
2
|
+
require 'concurrent'
|
3
|
+
require 'concurrent/executor/fixed_thread_pool'
|
4
|
+
|
5
|
+
class Disquo::Worker
|
6
|
+
attr_reader :disque, :queues, :wait_time, :wait_count
|
7
|
+
|
8
|
+
# Init a new worker instance
|
9
|
+
# @param [ConnectionPool] disque client connection pool
|
10
|
+
# @param [Hash] options
|
11
|
+
# @option [Array<String>] :queues queues to watch. Default: ["default"]
|
12
|
+
# @option [Integer] :concurrency the number of concurrent threads. Default: 10
|
13
|
+
# @option [Numeric] :wait_time maximum time (in seconds) to block for when retrieving next batch. Default: 1s
|
14
|
+
# @option [Integer] :wait_count the minimum number of jobs to wait for when retrieving next batch. Default: 100
|
15
|
+
def initialize(disque, queues: [Disquo::DEFAULT_QUEUE], concurrency: 10, wait_time: 1, wait_count: 100)
|
16
|
+
@disque = disque
|
17
|
+
@queues = Array(queues)
|
18
|
+
@threads = Concurrent::FixedThreadPool.new(concurrency)
|
19
|
+
@wait_time = wait_time
|
20
|
+
@wait_count = wait_count
|
21
|
+
end
|
22
|
+
|
23
|
+
# Run starts the worker
|
24
|
+
def run
|
25
|
+
Disquo.logger.info "Starting worker - queues: #{queues.inspect}, concurrency: #{@threads.max_length}"
|
26
|
+
|
27
|
+
begin
|
28
|
+
run_cycle
|
29
|
+
rescue => e
|
30
|
+
handle_exception(e)
|
31
|
+
end until @stopped
|
32
|
+
|
33
|
+
@threads.shutdown
|
34
|
+
end
|
35
|
+
|
36
|
+
# Blocks until worker is stopped
|
37
|
+
def wait(timeout = nil)
|
38
|
+
Disquo.logger.info "Waiting for worker shutdown"
|
39
|
+
@threads.wait_for_termination(timeout)
|
40
|
+
Disquo.logger.info "Shutdown complete"
|
41
|
+
end
|
42
|
+
|
43
|
+
# Stops the worker
|
44
|
+
def shutdown
|
45
|
+
return false if @stopped
|
46
|
+
|
47
|
+
@stopped = true
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def run_cycle
|
53
|
+
jobs = next_batch
|
54
|
+
|
55
|
+
until @stopped || jobs.empty?
|
56
|
+
job = jobs.shift
|
57
|
+
perform(*job)
|
58
|
+
end
|
59
|
+
ensure
|
60
|
+
requeue(jobs) unless jobs.nil? || jobs.empty?
|
61
|
+
end
|
62
|
+
|
63
|
+
def next_batch
|
64
|
+
jobs = disque.with do |cn|
|
65
|
+
cn.fetch from: queues, timeout: (wait_time*1000).to_i, count: wait_count
|
66
|
+
end
|
67
|
+
@is_down = nil
|
68
|
+
Array(jobs)
|
69
|
+
rescue => e
|
70
|
+
if !@is_down
|
71
|
+
@is_down = true
|
72
|
+
handle_exception(e, message: 'Error retrieving jobs:')
|
73
|
+
end
|
74
|
+
sleep(1)
|
75
|
+
[]
|
76
|
+
end
|
77
|
+
|
78
|
+
def perform(queue, job_id, payload)
|
79
|
+
@threads.post do
|
80
|
+
thread_id = Thread.current.object_id.to_s(36)
|
81
|
+
Disquo.logger.info { "Process #{payload} - thread: #{thread_id}, job: #{job_id}" }
|
82
|
+
|
83
|
+
begin
|
84
|
+
class_name, args = Disquo.load_job(payload)
|
85
|
+
|
86
|
+
job = Object.const_get(class_name).new
|
87
|
+
job.disque = disque
|
88
|
+
job.queue = queue
|
89
|
+
job.job_id = job_id
|
90
|
+
job.perform(*args)
|
91
|
+
rescue => e
|
92
|
+
handle_exception e, message: "Error processing #{payload} - thread: #{thread_id}, job: #{job_id}:"
|
93
|
+
|
94
|
+
disque.with {|cn| cn.call :nack, job_id }
|
95
|
+
return
|
96
|
+
end
|
97
|
+
|
98
|
+
begin
|
99
|
+
disque.with {|cn| cn.call :ackjob, job_id }
|
100
|
+
rescue => e
|
101
|
+
handle_exception e, message: "Error ACKing #{payload} - thread: #{thread_id}, job: #{job_id}:"
|
102
|
+
return
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def requeue(jobs)
|
108
|
+
ids = jobs.map {|_, job_id, _| job_id }
|
109
|
+
disque.with {|cn| cn.call :enqueue, *ids }
|
110
|
+
end
|
111
|
+
|
112
|
+
def handle_exception(e, opts = {})
|
113
|
+
lines = [
|
114
|
+
opts[:message],
|
115
|
+
"#{e.class.name}: #{e.message}",
|
116
|
+
e.backtrace
|
117
|
+
].compact.flatten
|
118
|
+
|
119
|
+
Disquo.logger.error lines.join("\n")
|
120
|
+
end
|
121
|
+
|
122
|
+
end
|
data/lib/disquo.rb
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'disque'
|
2
|
+
require 'connection_pool'
|
3
|
+
require 'json'
|
4
|
+
require 'logger'
|
5
|
+
|
6
|
+
module Disquo
|
7
|
+
DEFAULT_QUEUE = "default".freeze
|
8
|
+
|
9
|
+
# Configure disque with a block.
|
10
|
+
#
|
11
|
+
# @example:
|
12
|
+
# Disquo.configure do |c|
|
13
|
+
# c.disque = c.connect(nodes: ["10.0.0.1:7711", "10.0.0.2:7711", "10.0.0.3:7711"])
|
14
|
+
# end
|
15
|
+
def self.configure(&block)
|
16
|
+
block.call(self)
|
17
|
+
end
|
18
|
+
|
19
|
+
# @option [Array<String>|String] nodes a list of disque nodes. Default: ["127.0.0.1:7711"]
|
20
|
+
# @option [Hash] opts additional disque connection options.
|
21
|
+
# @option [Numeric] pool_timeout disque connection pool timeout in seconds. Default: 1.0
|
22
|
+
# @option [Integer] pool_size the number of slots in the connection pool. Default: 5
|
23
|
+
#
|
24
|
+
# @return [ConnectionPool] disque client connection pool
|
25
|
+
def self.connect(nodes: ["127.0.0.1:7711"], opts: {}, pool_size: 5, pool_timeout: 1)
|
26
|
+
ConnectionPool.new timeout: pool_timeout, size: pool_size do
|
27
|
+
Disque.new(nodes, opts)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
# @return [ConnectionPool] disque client connection pool
|
32
|
+
def self.disque
|
33
|
+
@disque ||= connect
|
34
|
+
end
|
35
|
+
|
36
|
+
# @param [ConnectionPool] disque client connection pool
|
37
|
+
def self.disque=(pool)
|
38
|
+
@disque = pool
|
39
|
+
end
|
40
|
+
|
41
|
+
# @return [Logger] returns the logger instance
|
42
|
+
def self.logger
|
43
|
+
@logger ||= Logger.new(STDOUT)
|
44
|
+
end
|
45
|
+
|
46
|
+
# @param [Logger] log logger instance to use
|
47
|
+
def self.logger=(log)
|
48
|
+
@logger = log
|
49
|
+
end
|
50
|
+
|
51
|
+
# @param [String] payload
|
52
|
+
# @return [Array<Class, Array>] job class and argument
|
53
|
+
def self.load_job(payload)
|
54
|
+
JSON.load(payload).values_at("klass", "args")
|
55
|
+
end
|
56
|
+
|
57
|
+
# @param [String] class_name class name
|
58
|
+
# @param [Array] arguments
|
59
|
+
# @return [String] serialised job
|
60
|
+
def self.dump_job(class_name, args)
|
61
|
+
JSON.dump "klass" => class_name, "args" => Array(args)
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
require 'disquo/job'
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Disquo::Job do
|
4
|
+
|
5
|
+
it "should enqueue jobs" do
|
6
|
+
jid1 = TestJob.enqueue ["foo", 1]
|
7
|
+
expect(qlen).to eq(1)
|
8
|
+
expect(jid1).to match(/^D\w+/)
|
9
|
+
expect(show(jid1)).to include(
|
10
|
+
"id" => jid1,
|
11
|
+
"queue" => "__disquo_test__",
|
12
|
+
"nacks" => 0,
|
13
|
+
"delay" => 0,
|
14
|
+
"repl" => 1,
|
15
|
+
"state" => "queued",
|
16
|
+
"body" => %({"klass":"TestJob","args":["foo",1]}),
|
17
|
+
)
|
18
|
+
|
19
|
+
jid2 = TestJob.enqueue ["bar"], delay: 600
|
20
|
+
expect(qlen).to eq(1)
|
21
|
+
expect(jid2).to match(/^D\w+/)
|
22
|
+
expect(show(jid2)).to include(
|
23
|
+
"id" => jid2,
|
24
|
+
"queue" => "__disquo_test__",
|
25
|
+
"nacks" => 0,
|
26
|
+
"delay" => 600,
|
27
|
+
"repl" => 1,
|
28
|
+
"state" => "active",
|
29
|
+
"body" => %({"klass":"TestJob","args":["bar"]}),
|
30
|
+
)
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should have instance attributes" do
|
34
|
+
job1, job2 = TestJob.new, TestJob.new
|
35
|
+
job1.job_id = "JOB1"
|
36
|
+
expect(job1.job_id).to eq("JOB1")
|
37
|
+
expect(job2.job_id).to be_nil
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Disquo::Worker do
|
4
|
+
|
5
|
+
subject do
|
6
|
+
described_class.new Disquo.connect, wait_time: 0.1, wait_count: 11, queues: Disquo::TEST::QUEUE
|
7
|
+
end
|
8
|
+
|
9
|
+
it "should run/process/shutdown" do
|
10
|
+
runner = Thread.new { subject.run }
|
11
|
+
|
12
|
+
# seed 200 jobs
|
13
|
+
200.times {|n| TestJob.enqueue(n) }
|
14
|
+
wait_for { !qlen.zero? }
|
15
|
+
|
16
|
+
# ensure runner processes them all
|
17
|
+
wait_for { qlen.zero? }
|
18
|
+
expect(runner).to be_alive
|
19
|
+
expect(qlen).to eq(0)
|
20
|
+
|
21
|
+
# ask runner to quit
|
22
|
+
expect(subject.shutdown).to be_truthy
|
23
|
+
expect(subject.shutdown).to be_falsey
|
24
|
+
|
25
|
+
# wait for runner to exit
|
26
|
+
subject.wait
|
27
|
+
expect(runner).not_to be_alive
|
28
|
+
|
29
|
+
# check what's been performed
|
30
|
+
expect(Disquo::TEST::PERFORMED.size).to eq(200)
|
31
|
+
expect(Disquo::TEST::PERFORMED.last).to include(
|
32
|
+
klass: "TestJob",
|
33
|
+
queue: "__disquo_test__",
|
34
|
+
)
|
35
|
+
expect(Disquo::TEST::PERFORMED.last[:job_id]).to match(/^D\w+/)
|
36
|
+
end
|
37
|
+
|
38
|
+
def wait_for
|
39
|
+
20.times do
|
40
|
+
break if yield
|
41
|
+
sleep(0.01)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require 'disquo'
|
3
|
+
require 'disquo/worker'
|
4
|
+
|
5
|
+
module Disquo::TEST
|
6
|
+
QUEUE = "__disquo_test__"
|
7
|
+
PERFORMED = []
|
8
|
+
end
|
9
|
+
|
10
|
+
class TestJob
|
11
|
+
include Disquo::Job
|
12
|
+
|
13
|
+
job_options queue: Disquo::TEST::QUEUE, async: true
|
14
|
+
|
15
|
+
def perform(*args)
|
16
|
+
Disquo::TEST::PERFORMED.push(klass: self.class.name, args: args, queue: queue, job_id: job_id)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
helpers = Module.new do
|
21
|
+
|
22
|
+
def qlen
|
23
|
+
Disquo.disque.with do |conn|
|
24
|
+
conn.call :qlen, Disquo::TEST::QUEUE
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def show(job_id)
|
29
|
+
Disquo.disque.with do |conn|
|
30
|
+
pairs = conn.call :show, job_id
|
31
|
+
Hash[*pairs]
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
RSpec.configure do |c|
|
38
|
+
c.include helpers
|
39
|
+
|
40
|
+
c.before :each do
|
41
|
+
Disquo.logger = Logger.new(nil)
|
42
|
+
end
|
43
|
+
|
44
|
+
c.after :each do
|
45
|
+
Disquo::TEST::PERFORMED.clear
|
46
|
+
Disquo.disque.with do |conn|
|
47
|
+
_, ids = conn.call :jscan, 0, :count, 10000, :queue, Disquo::TEST::QUEUE
|
48
|
+
conn.call :deljob, *ids unless ids.empty?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
metadata
ADDED
@@ -0,0 +1,132 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: disquo
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Dimitrij Denissenko
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-11-23 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: disque
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: connection_pool
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: concurrent-ruby
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rspec
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description: Concurrent background workers on top of Disque
|
84
|
+
email: dimitrij@blacksquaremedia.com
|
85
|
+
executables:
|
86
|
+
- disquo
|
87
|
+
extensions: []
|
88
|
+
extra_rdoc_files: []
|
89
|
+
files:
|
90
|
+
- ".travis.yml"
|
91
|
+
- Gemfile
|
92
|
+
- Gemfile.lock
|
93
|
+
- LICENSE
|
94
|
+
- README.md
|
95
|
+
- Rakefile
|
96
|
+
- bin/disquo
|
97
|
+
- disquo.gemspec
|
98
|
+
- lib/disquo.rb
|
99
|
+
- lib/disquo/cli.rb
|
100
|
+
- lib/disquo/job.rb
|
101
|
+
- lib/disquo/worker.rb
|
102
|
+
- spec/disquo/job_spec.rb
|
103
|
+
- spec/disquo/worker_spec.rb
|
104
|
+
- spec/spec_helper.rb
|
105
|
+
homepage: https://github.com/bsm/disquo
|
106
|
+
licenses:
|
107
|
+
- Apache-2.0
|
108
|
+
metadata: {}
|
109
|
+
post_install_message:
|
110
|
+
rdoc_options: []
|
111
|
+
require_paths:
|
112
|
+
- lib
|
113
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 2.3.0
|
118
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
requirements: []
|
124
|
+
rubyforge_project:
|
125
|
+
rubygems_version: 2.6.11
|
126
|
+
signing_key:
|
127
|
+
specification_version: 4
|
128
|
+
summary: Concurrent background workers on top of Disque
|
129
|
+
test_files:
|
130
|
+
- spec/disquo/job_spec.rb
|
131
|
+
- spec/disquo/worker_spec.rb
|
132
|
+
- spec/spec_helper.rb
|