tennis-jobs 0.3.1 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.markdown +38 -169
- data/examples/async_sum.rb +36 -0
- data/lib/tennis.rb +6 -11
- data/lib/tennis/action.rb +30 -0
- data/lib/tennis/actor.rb +49 -0
- data/lib/tennis/backend/abstract.rb +38 -0
- data/lib/tennis/backend/memory.rb +57 -0
- data/lib/tennis/{serializer/generic.rb → backend/serializer.rb} +30 -13
- data/lib/tennis/backend/task.rb +36 -0
- data/lib/tennis/cli.rb +21 -61
- data/lib/tennis/configuration.rb +9 -10
- data/lib/tennis/exceptions.rb +3 -0
- data/lib/tennis/fetcher.rb +36 -0
- data/lib/tennis/job.rb +30 -0
- data/lib/tennis/launcher.rb +38 -0
- data/lib/tennis/worker.rb +46 -0
- data/lib/tennis/worker_pool.rb +114 -0
- data/tennis-jobs.gemspec +4 -4
- metadata +19 -16
- data/examples/deferable.rb +0 -46
- data/examples/example.rb +0 -52
- data/examples/serializer.rb +0 -37
- data/lib/tennis/worker/deferable.rb +0 -55
- data/lib/tennis/worker/deferable/action.rb +0 -25
- data/lib/tennis/worker/generic.rb +0 -58
- data/lib/tennis/worker/generic/before_hook.rb +0 -17
- data/lib/tennis/worker/generic/options.rb +0 -19
- data/lib/tennis/worker/generic/serialize.rb +0 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 524542c022f441416696197818564f5c424a02fa
|
4
|
+
data.tar.gz: 7c41960d2ca72c6f4a0dee4e3d4af48bdca0c0ee
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1fbd7b880e73ce697cd5b6b001d127e8d2501a60596ea68966ad7739e003d7640c41e54e20a7c277b9059dc66de991b1b24370a3f739be89bef20c932a72abcd
|
7
|
+
data.tar.gz: d6b25ced4f0566bcbb9f44d781e4b6a922dbe5f5606dc6ee104bc6e8a293f6ade4f6044b4ec307c2e0e86fdc1bf2aa55a760cbdd6edc40896de86c709099691a
|
data/README.markdown
CHANGED
@@ -1,207 +1,76 @@
|
|
1
|
-
This
|
2
|
-
|
1
|
+
This is a celluloid based background jobs library heavily inspired by [Sidekiq][sidekiq].
|
2
|
+
The difference is that it allows you to change the backend, Redis, to other ones like RabitMQ.
|
3
|
+
You can start with something _à la_ [sucker_punch][sucker_punch]: background jobs threaded within the application process
|
4
|
+
and switch to separated processes later using another backend.
|
3
5
|
|
4
6
|
<a target="_blank" href="https://travis-ci.org/nicoolas25/tennis"><img src="https://travis-ci.org/nicoolas25/tennis.svg?branch=master" /></a>
|
5
7
|
<a target="_blank" href="https://codeclimate.com/github/nicoolas25/tennis"><img src="https://codeclimate.com/github/nicoolas25/tennis/badges/gpa.svg" /></a>
|
6
8
|
<a target="_blank" href="https://codeclimate.com/github/nicoolas25/tennis/coverage"><img src="https://codeclimate.com/github/nicoolas25/tennis/badges/coverage.svg" /></a>
|
7
9
|
<a target="_blank" href="https://rubygems.org/gems/tennis-jobs"><img src="https://badge.fury.io/rb/tennis-jobs.svg" /></a>
|
8
10
|
|
9
|
-
## Features
|
10
|
-
|
11
|
-
- Hooks: `.before(symbol, &block)`
|
12
|
-
- Serializers: `.serialize(loader:)`
|
13
|
-
- Helpers for defering method calls: `object.defer.method(*arguments)`
|
14
|
-
|
15
|
-
**Extra**
|
16
|
-
|
17
|
-
- A `Tennis::Serializer::Generic` handling classes and ActiveRecord objects
|
18
|
-
|
19
11
|
## Configuration
|
20
12
|
|
21
|
-
|
22
|
-
do asynchronously. Tennis uses YAML configuration file in order to launch thoses
|
23
|
-
processes.
|
24
|
-
|
25
|
-
``` yaml
|
26
|
-
# tennis.conf.yml
|
27
|
-
group1:
|
28
|
-
exchange: "default"
|
29
|
-
workers: 1
|
30
|
-
classes:
|
31
|
-
- "Scope::MyClass"
|
32
|
-
- "Scope::Model"
|
33
|
-
group2:
|
34
|
-
exchange: "important"
|
35
|
-
workers: 10
|
36
|
-
classes:
|
37
|
-
- "Only::ImportantWorker"
|
38
|
-
```
|
39
|
-
|
40
|
-
Here we see two groups of worker. Each group can be launch with the `tennis`
|
41
|
-
command:
|
42
|
-
|
43
|
-
$ bundle exec tennis group1
|
44
|
-
|
45
|
-
The `workers` options is directly given to sneakers, it will determine the
|
46
|
-
number of subprocesses that will handle the messages, the level of parallelism.
|
47
|
-
|
48
|
-
The classes are the classes that will receive your `send_work` or `defer` calls
|
49
|
-
but we'll see that later...
|
50
|
-
|
51
|
-
Also it is possible to add options directly in your workers:
|
13
|
+
Install the gem in your Gemfile:
|
52
14
|
|
53
15
|
``` ruby
|
54
|
-
|
55
|
-
|
56
|
-
AfterFork = -> { ActiveRecord::Base.establish_connection }
|
57
|
-
end
|
58
|
-
|
59
|
-
class MyClass
|
60
|
-
include Tennis::Worker::Generic
|
61
|
-
|
62
|
-
set_option :before_fork, WorkerHelpers::BeforeFork
|
63
|
-
set_option :after_fork, WorkerHelpers::AfterFork
|
64
|
-
set_option :handler, Sneakers::Handlers::MaxretryWithoutErrors
|
65
|
-
|
66
|
-
work do |message|
|
67
|
-
MyActiveRecordModel.create(message: message)
|
68
|
-
end
|
69
|
-
end
|
16
|
+
gem "tennis-jobs"
|
17
|
+
gem "tennis-jobs-redis" # Not available at the moment
|
70
18
|
```
|
71
19
|
|
72
|
-
|
73
|
-
having different options for the workers of a same group isn't possible. By
|
74
|
-
using this constant, I can have another worker with those same options and put
|
75
|
-
it in the same group.
|
76
|
-
|
77
|
-
## Examples
|
78
|
-
|
79
|
-
Those examples are what we wish to achieve.
|
80
|
-
|
81
|
-
The name of the queue is the name of the class by default and can be reset
|
82
|
-
using `set_option :queue_name, "your-queue-name"`.
|
83
|
-
|
84
|
-
### Hooks
|
20
|
+
Configure Tennis in your `config/application.rb` (or any other file):
|
85
21
|
|
86
22
|
``` ruby
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
before do
|
91
|
-
puts "Before processing"
|
92
|
-
end
|
23
|
+
Tennis.configure do |config|
|
24
|
+
config.backend Tennis::Backend::Redis.new(redis_url)
|
93
25
|
|
94
|
-
|
95
|
-
|
96
|
-
ack!
|
97
|
-
end
|
26
|
+
# require "logger"
|
27
|
+
# config.logger = Logger.new(STDOUT)
|
98
28
|
end
|
99
|
-
|
100
|
-
MyClass.send_work("my class")
|
101
|
-
# => Before processing
|
102
|
-
# => Working with my class
|
103
29
|
```
|
104
30
|
|
105
|
-
|
106
|
-
|
107
|
-
You can provide a Proc for the loader and/or the dumper keywords.
|
108
|
-
The dumper will be used when calling `MyClass.send_work(message)` receiving
|
109
|
-
the `message` as argument. It should return a string. The loader will be
|
110
|
-
used when the message is poped from the RabbitMQ queue.
|
31
|
+
Start tennis from the command line:
|
111
32
|
|
112
|
-
``` ruby
|
113
|
-
class MyClass
|
114
|
-
include Tennis::Worker::Generic
|
115
|
-
|
116
|
-
serialize loader: ->(message){ JSON.parse(message) },
|
117
|
-
dumper: ->(message){ JSON.generate(message) }
|
118
|
-
|
119
|
-
work do |message|
|
120
|
-
one, two = message
|
121
|
-
puts "Message is serialized and deserialized correctly"
|
122
|
-
puts "one: #{one}, two: #{two}"
|
123
|
-
ack!
|
124
|
-
end
|
125
|
-
end
|
126
|
-
|
127
|
-
MyClass.send_work([1, "foo"])
|
128
|
-
# => Message is serialized and deserialized correctly
|
129
|
-
# => one: 1, two: foo
|
130
33
|
```
|
34
|
+
bundle exec tennis --concurrency 4 --require ./config/application.rb --jobs "MyJob,MyOtherJob"
|
131
35
|
|
132
|
-
|
133
|
-
|
134
|
-
include Tennis::Worker::Generic
|
135
|
-
|
136
|
-
serialize Tennis::Serializer::Generic.new
|
137
|
-
|
138
|
-
work do |message|
|
139
|
-
klass, active_record_object = message
|
140
|
-
puts "Classes can be passed: #{klass.name} - #{klass.class}"
|
141
|
-
puts "Active record object can be passed too: #{active_record_object}"
|
142
|
-
ack!
|
143
|
-
end
|
144
|
-
end
|
145
|
-
|
146
|
-
MyClass.send_work([String, User.find(1)])
|
147
|
-
# => Classes can be passed: String - Class
|
148
|
-
# => Active record object can be passed too: <User#1 ...>
|
36
|
+
# There is also a shorter equivalent:
|
37
|
+
# bundle exex tennis -c 4 -r ./config/application.rb -j "MyJob,MyOtherJob"
|
149
38
|
```
|
150
39
|
|
151
|
-
|
152
|
-
|
153
|
-
Any class method can be defered:
|
40
|
+
## Usage
|
154
41
|
|
155
42
|
``` ruby
|
156
|
-
|
157
|
-
include Tennis::Worker::Deferable
|
43
|
+
MINUTES = 60
|
158
44
|
|
159
|
-
|
160
|
-
|
45
|
+
class MyJob
|
46
|
+
include Tennis::Job
|
47
|
+
|
48
|
+
def my_method(*args)
|
49
|
+
puts "=> #{args}.sum = args.inject(0, &:+)"
|
161
50
|
end
|
162
51
|
end
|
163
52
|
|
164
|
-
|
165
|
-
|
166
|
-
```
|
167
|
-
|
168
|
-
An ActiveRecord::Base instance can be the receiver if it has an `id`:
|
53
|
+
my_job_instance = MyJob.new
|
54
|
+
my_job_instance.async.my_method(1, 2, 3)
|
169
55
|
|
170
|
-
|
171
|
-
|
172
|
-
include Tennis::Worker::Deferable
|
56
|
+
# Will print in your `tennis` process:
|
57
|
+
# => [1, 2, 3].sum = 6
|
173
58
|
|
174
|
-
|
175
|
-
puts "Running my method on #{self}"
|
176
|
-
end
|
177
|
-
end
|
59
|
+
my_job_instance.async_in(2 * MINUTES).my_method(4, 5, 6)
|
178
60
|
|
179
|
-
|
180
|
-
|
181
|
-
# => Running my method on <MyModel#1 ...>
|
61
|
+
# Will print, in approximatively two minutes, in your `tennis` process:
|
62
|
+
# => [4, 5, 6].sum = 15
|
182
63
|
```
|
183
64
|
|
184
|
-
|
185
|
-
|
186
|
-
Sneakers' `work` method.
|
65
|
+
The `my_method`'s arguments can be quite complex depending on your backend support.
|
66
|
+
The same goes for the `MyJob`'s instance.
|
187
67
|
|
188
|
-
|
189
|
-
class MyModel < ActiveRecord::Base
|
190
|
-
include Tennis::Worker::Deferable
|
68
|
+
With the `Tennis::Backend::Memory` backend, you can use anything and it will be kept as it is.
|
191
69
|
|
192
|
-
|
193
|
-
puts "I just saved the day handling a #{exception}"
|
194
|
-
reject!
|
195
|
-
end
|
70
|
+
## Testing
|
196
71
|
|
197
|
-
|
198
|
-
puts "Running my method on #{self}"
|
199
|
-
raise StandardError
|
200
|
-
end
|
201
|
-
end
|
72
|
+
This section is waiting to beeing written.
|
202
73
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
# => I just saved the day handling a StandardError
|
207
|
-
```
|
74
|
+
|
75
|
+
[sidekiq]: https://github.com/mperham/sidekiq
|
76
|
+
[sucker_punch]: https://github.com/brandonhilkert/sucker_punch
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require "tennis"
|
2
|
+
require "tennis/backend/memory"
|
3
|
+
|
4
|
+
logger = Logger.new(STDOUT)
|
5
|
+
logger.level = Logger::DEBUG
|
6
|
+
|
7
|
+
Tennis.configure do |config|
|
8
|
+
config.backend = Tennis::Backend::Memory.new(logger: logger)
|
9
|
+
config.logger = logger
|
10
|
+
end
|
11
|
+
|
12
|
+
class Job
|
13
|
+
include Tennis::Job
|
14
|
+
|
15
|
+
def sum(*numbers)
|
16
|
+
sleep 0.4
|
17
|
+
total = numbers.inject(&:+)
|
18
|
+
puts "Sum #{numbers} => #{total}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
require "tennis/launcher"
|
23
|
+
|
24
|
+
# Instanciate a job and add the sum to the job to do.
|
25
|
+
numbers = (1..9).to_a
|
26
|
+
10.times do
|
27
|
+
Job.new.async.sum(*numbers.sample(3))
|
28
|
+
end
|
29
|
+
|
30
|
+
# Start Tennis.
|
31
|
+
launcher = Tennis::Launcher.new(concurrency: 2, job_classes: [Job])
|
32
|
+
launcher.async.start
|
33
|
+
|
34
|
+
# Wait 1 seconds and stop Tennis
|
35
|
+
sleep 1
|
36
|
+
launcher.async.stop
|
data/lib/tennis.rb
CHANGED
@@ -1,16 +1,7 @@
|
|
1
|
-
require "
|
1
|
+
require "tennis/configuration"
|
2
2
|
|
3
3
|
module Tennis
|
4
|
-
|
5
|
-
autoload :Deferable, "tennis/worker/deferable"
|
6
|
-
autoload :Generic, "tennis/worker/generic"
|
7
|
-
end
|
8
|
-
|
9
|
-
module Serializer
|
10
|
-
autoload :Generic, "tennis/serializer/generic"
|
11
|
-
end
|
12
|
-
|
13
|
-
autoload :Configuration, "tennis/configuration"
|
4
|
+
autoload :Job, "tennis/job"
|
14
5
|
|
15
6
|
def self.configure
|
16
7
|
@config = Configuration.new
|
@@ -22,4 +13,8 @@ module Tennis
|
|
22
13
|
@config or fail "You must run Tennis.configure before accessing the configuration"
|
23
14
|
end
|
24
15
|
|
16
|
+
def self.logger
|
17
|
+
@config.logger
|
18
|
+
end
|
19
|
+
|
25
20
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Tennis
|
2
|
+
class Action
|
3
|
+
|
4
|
+
attr_reader :_receiver
|
5
|
+
|
6
|
+
def initialize(receiver, delay: nil)
|
7
|
+
@_receiver = receiver
|
8
|
+
@delay = delay
|
9
|
+
_create_methods!
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def _create_methods!
|
15
|
+
_methods.each do |method|
|
16
|
+
self.define_singleton_method(method) do |*arguments|
|
17
|
+
_store(job: @_receiver, method: method, args: arguments)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def _methods
|
23
|
+
@_receiver.class.instance_methods(false).map(&:to_s)
|
24
|
+
end
|
25
|
+
|
26
|
+
def _store(**kwargs)
|
27
|
+
Tennis.config.backend.enqueue(**kwargs.merge(delay: @delay))
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/tennis/actor.rb
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
require "celluloid" unless $TESTING
|
2
|
+
|
3
|
+
module Tennis
|
4
|
+
module Actor
|
5
|
+
|
6
|
+
module ClassMethods
|
7
|
+
def new_link(*args)
|
8
|
+
new(*args)
|
9
|
+
end
|
10
|
+
|
11
|
+
def trap_exit(*args)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
module InstanceMethods
|
16
|
+
def current_actor
|
17
|
+
self
|
18
|
+
end
|
19
|
+
|
20
|
+
def after(interval)
|
21
|
+
yield
|
22
|
+
end
|
23
|
+
|
24
|
+
def alive?
|
25
|
+
@dead = false unless defined?(@dead)
|
26
|
+
!@dead
|
27
|
+
end
|
28
|
+
|
29
|
+
def terminate
|
30
|
+
@dead = true
|
31
|
+
end
|
32
|
+
|
33
|
+
def async
|
34
|
+
self
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
# :nocov:
|
39
|
+
def self.included(klass)
|
40
|
+
if $TESTING
|
41
|
+
klass.include InstanceMethods
|
42
|
+
klass.extend ClassMethods
|
43
|
+
else
|
44
|
+
klass.include Celluloid
|
45
|
+
end
|
46
|
+
end
|
47
|
+
# :nocov:
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Tennis
|
2
|
+
module Backend
|
3
|
+
class Abstract
|
4
|
+
|
5
|
+
attr_reader :logger
|
6
|
+
|
7
|
+
def initialize(logger:)
|
8
|
+
@logger = logger
|
9
|
+
end
|
10
|
+
|
11
|
+
# :nocov:
|
12
|
+
|
13
|
+
# Creates and enqueues a Task.
|
14
|
+
def enqueue(job:, method:, args:, delay: nil)
|
15
|
+
raise NotImplementedError
|
16
|
+
end
|
17
|
+
|
18
|
+
# Returns a Task that have previously been queued. The Task should
|
19
|
+
# contain a Job that is an instance of one of the given classes.
|
20
|
+
def receive(job_classes:, timeout: 1.0)
|
21
|
+
raise NotImplementedError
|
22
|
+
end
|
23
|
+
|
24
|
+
# Acknowledge that a Task has been done.
|
25
|
+
def ack(task)
|
26
|
+
raise NotImplementedError
|
27
|
+
end
|
28
|
+
|
29
|
+
# Requeue a Task that haven't been acked.
|
30
|
+
def requeue(task)
|
31
|
+
raise NotImplementedError
|
32
|
+
end
|
33
|
+
|
34
|
+
# :nocov:
|
35
|
+
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require "thread"
|
2
|
+
|
3
|
+
require "tennis/backend/abstract"
|
4
|
+
require "tennis/backend/task"
|
5
|
+
|
6
|
+
module Tennis
|
7
|
+
module Backend
|
8
|
+
class Memory < Abstract
|
9
|
+
|
10
|
+
attr_reader :queue, :acked_tasks
|
11
|
+
|
12
|
+
def initialize(**kwargs)
|
13
|
+
super
|
14
|
+
@mutex = Mutex.new
|
15
|
+
@task_id = 0
|
16
|
+
@queue = []
|
17
|
+
@acked_tasks = []
|
18
|
+
@acked_history_size = kwargs.fetch(:acked_history_size, 10)
|
19
|
+
end
|
20
|
+
|
21
|
+
def enqueue(job:, method:, args:, delay: nil)
|
22
|
+
@mutex.synchronize do
|
23
|
+
@task_id += 1
|
24
|
+
queue << Task.new(self, @task_id, job, method, args)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def receive(job_classes:, timeout: 1.0)
|
29
|
+
@mutex.synchronize do
|
30
|
+
task = queue.find { |task| job_classes.include?(task.job.class) }
|
31
|
+
|
32
|
+
if task.nil?
|
33
|
+
sleep(timeout)
|
34
|
+
nil
|
35
|
+
else
|
36
|
+
queue.delete(task)
|
37
|
+
task
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def ack(task)
|
43
|
+
@mutex.synchronize do
|
44
|
+
acked_tasks.unshift task
|
45
|
+
acked_tasks.pop if acked_tasks.size > @acked_history_size
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def requeue(task)
|
50
|
+
@mutex.synchronize do
|
51
|
+
queue << task
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|