disc 0.0.6 → 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gems +3 -1
- data/{LICENSE.md → LICENSE} +0 -0
- data/Makefile +39 -5
- data/README.md +60 -16
- data/bin/disc +21 -8
- data/disc.gemspec +13 -9
- data/examples/disc_init.rb +1 -0
- data/examples/fail_job.rb +10 -0
- data/lib/disc.rb +31 -11
- data/lib/disc/version.rb +3 -0
- metadata +21 -6
- data/disc-wars.jpg +0 -0
- data/disc_init_example.rb +0 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7d8bbbb6f541c89e165acb164db27941d0c13481
|
|
4
|
+
data.tar.gz: b74592344a5a936f8e2aed6039772a39af8f1dbc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1452ef79a333b0562f9cce994b28770c6d66aadd41d14577edebeeded2678735e22e0486b75b31462250c5b40b93565f2f4ca0310b145263c74c2e64a39f4968
|
|
7
|
+
data.tar.gz: 4679185b060daf9f1778ce2f0976b37787e7febed760c5838a065b25c6c440cd968a9c8d713ed01f32365cb4fffbadc67ef8de666dcfdc702235a40ad1d3ebad
|
data/.gems
CHANGED
data/{LICENSE.md → LICENSE}
RENAMED
|
File without changes
|
data/Makefile
CHANGED
|
@@ -1,8 +1,42 @@
|
|
|
1
|
+
ifndef GS_NAME
|
|
2
|
+
$(error GS_NAME not set. Have you `gs in` yet?)
|
|
3
|
+
endif
|
|
1
4
|
|
|
2
|
-
|
|
3
|
-
|
|
5
|
+
PACKAGES := disc
|
|
6
|
+
VERSION_FILE := lib/disc/version.rb
|
|
4
7
|
|
|
5
|
-
|
|
6
|
-
|
|
8
|
+
DEPS := ${GEM_HOME}/installed
|
|
9
|
+
VERSION := $(shell sed -ne '/.*VERSION *= *"\(.*\)".*/s//\1/p' <$(VERSION_FILE))
|
|
10
|
+
GEMS := $(addprefix pkg/, $(addsuffix -$(VERSION).gem, $(PACKAGES)))
|
|
7
11
|
|
|
8
|
-
|
|
12
|
+
export RUBYLIB := lib:test:$(RUBYLIB)
|
|
13
|
+
|
|
14
|
+
all: test $(GEMS)
|
|
15
|
+
|
|
16
|
+
console: $(DEPS)
|
|
17
|
+
irb -r disc
|
|
18
|
+
|
|
19
|
+
test: $(DEPS)
|
|
20
|
+
cutest ./test/**/*_test.rb
|
|
21
|
+
|
|
22
|
+
clean:
|
|
23
|
+
rm pkg/*.gem
|
|
24
|
+
|
|
25
|
+
release: $(GEMS)
|
|
26
|
+
git tag v$(VERSION)
|
|
27
|
+
git push --tags
|
|
28
|
+
for gem in $^; do gem push $$gem; done
|
|
29
|
+
|
|
30
|
+
pkg/%-$(VERSION).gem: %.gemspec $(VERSION_FILE) | pkg
|
|
31
|
+
gem build $<
|
|
32
|
+
mv $(@F) pkg/
|
|
33
|
+
|
|
34
|
+
$(DEPS): $(GEM_HOME) .gems
|
|
35
|
+
which dep &>/dev/null || gem install dep
|
|
36
|
+
dep install
|
|
37
|
+
touch $(GEM_HOME)/installed
|
|
38
|
+
|
|
39
|
+
pkg:
|
|
40
|
+
mkdir -p $@
|
|
41
|
+
|
|
42
|
+
.PHONY: all test release clean
|
data/README.md
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Disc fills the gap between your Ruby service objects and [antirez](http://antirez.com/)'s wonderful [Disque](https://github.com/antirez/disque) backend.
|
|
4
4
|
|
|
5
|
-
<a href=https://www.flickr.com/photos/noodlefish/5321412234
|
|
6
|
-
![Disc Wars!](
|
|
5
|
+
<a href=https://www.flickr.com/photos/noodlefish/5321412234" target="blank_">
|
|
6
|
+
|
|
7
7
|
</a>
|
|
8
8
|
|
|
9
9
|
## Usage
|
|
@@ -46,19 +46,13 @@ Disc fills the gap between your Ruby service objects and [antirez](http://antire
|
|
|
46
46
|
```ruby
|
|
47
47
|
# disc_init.rb
|
|
48
48
|
require 'ohm'
|
|
49
|
-
Dir
|
|
50
|
-
```
|
|
51
|
-
|
|
52
|
-
6. Set your require file
|
|
53
|
-
|
|
54
|
-
```bash
|
|
55
|
-
$ export DISC_REQUIRE='./disc_init.rb'
|
|
49
|
+
Dir['./jobs/**/*.rb'].each { |job| require job }
|
|
56
50
|
```
|
|
57
51
|
|
|
58
52
|
7. Run as many Disc Worker processes as you wish.
|
|
59
53
|
|
|
60
54
|
```bash
|
|
61
|
-
$ QUEUES=urgent,default disc
|
|
55
|
+
$ QUEUES=urgent,default disc -r ./disc_init.rb
|
|
62
56
|
```
|
|
63
57
|
|
|
64
58
|
## Settings
|
|
@@ -67,7 +61,6 @@ Disc takes its configuration from environment variables.
|
|
|
67
61
|
|
|
68
62
|
| ENV Variable | Default Value | Description
|
|
69
63
|
|:----------------:|:-----------------|:------------|
|
|
70
|
-
| DISC_REQUIRE | null | Ruby file that will be required by the worker processes, it should load all Disc::Job classes on your application and whatever else is needed to run them.
|
|
71
64
|
| QUEUES | 'default' | The list of queues that `Disc::Worker` will listen to, it can be a single queue name or a list of comma-separated queues |
|
|
72
65
|
| DISC_CONCURRENCY | '25' | Amount of threads to spawn when Celluloid is available. |
|
|
73
66
|
| DISQUE_NODES | 'localhost:7711' | This is the list of Disque servers to connect to, it can be a single node or a list of comma-separated nodes |
|
|
@@ -75,16 +68,67 @@ Disc takes its configuration from environment variables.
|
|
|
75
68
|
| DISQUE_TIMEOUT | '100' | Time in milliseconds that the client will wait for the Disque server to acknowledge and replicate a job |
|
|
76
69
|
| DISQUE_CYCLE | '1000' | The client keeps track of which nodes are providing more jobs, after the amount of operations specified in cycle it tries to connect to the preferred node. |
|
|
77
70
|
|
|
71
|
+
## Error handling
|
|
72
|
+
|
|
73
|
+
When a job raises an exception, `Disc.on_error` is invoked with the error and
|
|
74
|
+
the job data. By default, this method prints the error to standard error, but
|
|
75
|
+
you can override it to report the error to your favorite error aggregator.
|
|
76
|
+
|
|
77
|
+
``` ruby
|
|
78
|
+
# On disc_init.rb
|
|
79
|
+
def Disc.on_error(exception, job)
|
|
80
|
+
# ... report the error
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
Dir["./jobs/**/*.rb"].each { |job| require job }
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Lifecycle Callbacks
|
|
78
87
|
|
|
88
|
+
You can optionally define two methods on classes that include `Disc::Job` that
|
|
89
|
+
will act as callbacks:
|
|
79
90
|
|
|
91
|
+
``` ruby
|
|
92
|
+
def disc_start(job)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def disc_done(exception_or_nil)
|
|
96
|
+
end
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Before the `perform` method is invoked, Disc will call `disc_start` and pass the
|
|
100
|
+
job data as a Hash. After the job is finished, it will call `disc_done`.
|
|
101
|
+
|
|
102
|
+
You could use these callbacks for things like aggregating metrics about the jobs
|
|
103
|
+
(timing, number of times a certain job is run, etc), or for advanced logging,
|
|
104
|
+
for example.
|
|
105
|
+
|
|
106
|
+
## Job Definition
|
|
107
|
+
|
|
108
|
+
Both the error handler function and the `disc_start` callback get the data of
|
|
109
|
+
the current job as a Hash, that has the following schema.
|
|
110
|
+
|
|
111
|
+
| `'class'` | (String) The Job class. |
|
|
112
|
+
| `'arguments'` | (Array) The arguments passed to perform. |
|
|
113
|
+
| `'queue'` | (String) The queue from which this job was picked up. |
|
|
114
|
+
| `'id'` | (String) Disque's job ID. |
|
|
80
115
|
|
|
81
116
|
## PowerUps
|
|
82
117
|
|
|
83
|
-
Disc workers can run just fine on their own, but if you're using
|
|
118
|
+
Disc workers can run just fine on their own, but if you're using
|
|
119
|
+
[Celluloid](https://github.com/celluloid/celluloid) you migth want Disc to take
|
|
120
|
+
advantage of it and spawn multiple worker threads per process, doing this is
|
|
121
|
+
trivial! Just require Celluloid before your init file:
|
|
84
122
|
|
|
85
|
-
```
|
|
86
|
-
|
|
87
|
-
require 'celluloid'
|
|
123
|
+
```bash
|
|
124
|
+
$ QUEUES=urgent,default disc -r celluloid/current -r ./disc_init.rb
|
|
88
125
|
```
|
|
89
126
|
|
|
90
|
-
Whenever Disc detects that Celluloid is available it will use it to spawn a
|
|
127
|
+
Whenever Disc detects that Celluloid is available it will use it to spawn a
|
|
128
|
+
number of threads equal to the `DISC_CONCURRENCY` environment variable, or 25 by
|
|
129
|
+
default.
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
The code is released under an MIT license. See the [LICENSE](./LICENSE) file for
|
|
134
|
+
more information.
|
data/bin/disc
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
|
|
3
3
|
require 'disc'
|
|
4
|
-
require
|
|
4
|
+
require 'clap'
|
|
5
|
+
|
|
6
|
+
Clap.run ARGV,
|
|
7
|
+
"-r" => lambda { |file| require file }
|
|
5
8
|
|
|
6
9
|
if defined?(Celluloid)
|
|
7
10
|
concurrency = ENV.fetch('DISC_CONCURRENCY', '25').to_i
|
|
@@ -10,14 +13,24 @@ if defined?(Celluloid)
|
|
|
10
13
|
#{ concurrency }."
|
|
11
14
|
)
|
|
12
15
|
|
|
13
|
-
|
|
14
|
-
pool Disc::Worker,
|
|
15
|
-
size: ENV.fetch('DISC_CONCURRENCY', '25').to_i,
|
|
16
|
-
as: :worker_pool,
|
|
17
|
-
args: [{ run: true }]
|
|
18
|
-
end
|
|
16
|
+
Disc::Worker.send(:include, Celluloid)
|
|
19
17
|
|
|
20
|
-
|
|
18
|
+
if defined?(Celluloid::SupervisionGroup)
|
|
19
|
+
# Deprecated as of Celluloid 0.17, but still supported via "backported mode"
|
|
20
|
+
class Disc::WorkerGroup < Celluloid::SupervisionGroup
|
|
21
|
+
pool Disc::Worker,
|
|
22
|
+
size: ENV.fetch('DISC_CONCURRENCY', '25').to_i,
|
|
23
|
+
as: :worker_pool,
|
|
24
|
+
args: [{ run: true }]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
Disc::WorkerGroup.run
|
|
28
|
+
else
|
|
29
|
+
Disc::Worker.pool(
|
|
30
|
+
size: concurrency,
|
|
31
|
+
args: [{ run: true }]
|
|
32
|
+
)
|
|
33
|
+
end
|
|
21
34
|
else
|
|
22
35
|
STDOUT.puts(
|
|
23
36
|
"[Notice] Disc running in non-threaded mode, consider requiring celluloid\
|
data/disc.gemspec
CHANGED
|
@@ -1,15 +1,19 @@
|
|
|
1
|
+
require_relative "lib/disc/version"
|
|
2
|
+
|
|
1
3
|
Gem::Specification.new do |s|
|
|
2
|
-
s.name
|
|
3
|
-
s.version
|
|
4
|
-
s.summary
|
|
5
|
-
s.description
|
|
6
|
-
s.authors
|
|
7
|
-
s.email
|
|
8
|
-
s.homepage
|
|
9
|
-
s.license
|
|
10
|
-
s.files
|
|
4
|
+
s.name = 'disc'
|
|
5
|
+
s.version = Disc::VERSION
|
|
6
|
+
s.summary = 'A simple and powerful Disque job implementation'
|
|
7
|
+
s.description = 'Easily define and run background jobs using Disque'
|
|
8
|
+
s.authors = ['pote']
|
|
9
|
+
s.email = ['pote@tardis.com.uy']
|
|
10
|
+
s.homepage = 'https://github.com/pote/disc'
|
|
11
|
+
s.license = 'MIT'
|
|
12
|
+
s.files = `git ls-files`.split("\n")
|
|
11
13
|
|
|
12
14
|
s.executables.push('disc')
|
|
15
|
+
|
|
13
16
|
s.add_dependency('disque', '~> 0.0.6')
|
|
14
17
|
s.add_dependency('msgpack', '~> 0.6.1')
|
|
18
|
+
s.add_dependency('clap', '~> 1.0')
|
|
15
19
|
end
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Dir.glob("./**/*.rb") { |f| require f }
|
data/lib/disc.rb
CHANGED
|
@@ -2,6 +2,8 @@ require 'date'
|
|
|
2
2
|
require 'disque'
|
|
3
3
|
require 'msgpack'
|
|
4
4
|
|
|
5
|
+
require_relative 'disc/version'
|
|
6
|
+
|
|
5
7
|
class Disc
|
|
6
8
|
attr_reader :disque,
|
|
7
9
|
:disque_timeout
|
|
@@ -26,9 +28,11 @@ class Disc
|
|
|
26
28
|
@disque_timeout = timeout
|
|
27
29
|
end
|
|
28
30
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
+
def self.on_error(exception, job)
|
|
32
|
+
STDERR.puts exception
|
|
33
|
+
end
|
|
31
34
|
|
|
35
|
+
class Worker
|
|
32
36
|
attr_reader :disque,
|
|
33
37
|
:queues,
|
|
34
38
|
:timeout,
|
|
@@ -59,18 +63,28 @@ class Disc
|
|
|
59
63
|
self
|
|
60
64
|
end
|
|
61
65
|
|
|
62
|
-
|
|
63
66
|
def run
|
|
64
67
|
STDOUT.puts("Disc::Worker listening in #{queues}")
|
|
65
68
|
loop do
|
|
66
|
-
disque.fetch(
|
|
67
|
-
|
|
68
|
-
timeout: timeout,
|
|
69
|
-
count: count
|
|
70
|
-
) do |serialized_job, _|
|
|
69
|
+
jobs = disque.fetch(from: queues, timeout: timeout, count: count)
|
|
70
|
+
Array(jobs).each do |queue, msgid, serialized_job|
|
|
71
71
|
job = MessagePack.unpack(serialized_job)
|
|
72
|
-
|
|
73
|
-
|
|
72
|
+
job.update('id' => msgid, 'queue' => queue)
|
|
73
|
+
|
|
74
|
+
instance = Object.const_get(job['class']).new
|
|
75
|
+
begin
|
|
76
|
+
instance.disc_start(job)
|
|
77
|
+
instance.perform(*job['arguments'])
|
|
78
|
+
disque.call('ACKJOB', msgid)
|
|
79
|
+
rescue => err
|
|
80
|
+
Disc.on_error(err, job)
|
|
81
|
+
ensure
|
|
82
|
+
begin
|
|
83
|
+
instance.disc_done(err)
|
|
84
|
+
rescue => boom
|
|
85
|
+
Disc.on_error(boom, job)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
74
88
|
end
|
|
75
89
|
end
|
|
76
90
|
end
|
|
@@ -85,6 +99,12 @@ class Disc
|
|
|
85
99
|
base.extend(ClassMethods)
|
|
86
100
|
end
|
|
87
101
|
|
|
102
|
+
def disc_start(job)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def disc_done(error = nil)
|
|
106
|
+
end
|
|
107
|
+
|
|
88
108
|
module ClassMethods
|
|
89
109
|
def disque
|
|
90
110
|
defined?(@disque) ? @disque : Disc.disque
|
|
@@ -118,7 +138,7 @@ class Disc
|
|
|
118
138
|
queue,
|
|
119
139
|
args.to_msgpack,
|
|
120
140
|
Disc.disque_timeout,
|
|
121
|
-
delay:
|
|
141
|
+
delay: datetime && (datetime.to_time.to_i - DateTime.now.to_time.to_i)
|
|
122
142
|
)
|
|
123
143
|
end
|
|
124
144
|
|
data/lib/disc/version.rb
ADDED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: disc
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- pote
|
|
@@ -38,7 +38,21 @@ dependencies:
|
|
|
38
38
|
- - "~>"
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: 0.6.1
|
|
41
|
-
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: clap
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '1.0'
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '1.0'
|
|
55
|
+
description: Easily define and run background jobs using Disque
|
|
42
56
|
email:
|
|
43
57
|
- pote@tardis.com.uy
|
|
44
58
|
executables:
|
|
@@ -50,18 +64,19 @@ files:
|
|
|
50
64
|
- ".gems"
|
|
51
65
|
- ".gitignore"
|
|
52
66
|
- CONTRIBUTING.md
|
|
53
|
-
- LICENSE
|
|
67
|
+
- LICENSE
|
|
54
68
|
- Makefile
|
|
55
69
|
- README.md
|
|
56
70
|
- bin/disc
|
|
57
|
-
- disc-wars.jpg
|
|
58
71
|
- disc.gemspec
|
|
59
|
-
-
|
|
72
|
+
- examples/disc_init.rb
|
|
73
|
+
- examples/fail_job.rb
|
|
60
74
|
- examples/failer.rb
|
|
61
75
|
- examples/greeter.rb
|
|
62
76
|
- lib/disc.rb
|
|
77
|
+
- lib/disc/version.rb
|
|
63
78
|
- test/disc_test.rb
|
|
64
|
-
homepage: https://github.com/pote/
|
|
79
|
+
homepage: https://github.com/pote/disc
|
|
65
80
|
licenses:
|
|
66
81
|
- MIT
|
|
67
82
|
metadata: {}
|
data/disc-wars.jpg
DELETED
|
Binary file
|
data/disc_init_example.rb
DELETED