disc 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gems +3 -1
- data/{LICENSE.md → LICENSE} +0 -0
- data/Makefile +39 -5
- data/README.md +60 -16
- data/bin/disc +21 -8
- data/disc.gemspec +13 -9
- data/examples/disc_init.rb +1 -0
- data/examples/fail_job.rb +10 -0
- data/lib/disc.rb +31 -11
- data/lib/disc/version.rb +3 -0
- metadata +21 -6
- data/disc-wars.jpg +0 -0
- data/disc_init_example.rb +0 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7d8bbbb6f541c89e165acb164db27941d0c13481
|
|
4
|
+
data.tar.gz: b74592344a5a936f8e2aed6039772a39af8f1dbc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1452ef79a333b0562f9cce994b28770c6d66aadd41d14577edebeeded2678735e22e0486b75b31462250c5b40b93565f2f4ca0310b145263c74c2e64a39f4968
|
|
7
|
+
data.tar.gz: 4679185b060daf9f1778ce2f0976b37787e7febed760c5838a065b25c6c440cd968a9c8d713ed01f32365cb4fffbadc67ef8de666dcfdc702235a40ad1d3ebad
|
data/.gems
CHANGED
data/{LICENSE.md → LICENSE}
RENAMED
|
File without changes
|
data/Makefile
CHANGED
|
@@ -1,8 +1,42 @@
|
|
|
1
|
+
ifndef GS_NAME
|
|
2
|
+
$(error GS_NAME not set. Have you `gs in` yet?)
|
|
3
|
+
endif
|
|
1
4
|
|
|
2
|
-
|
|
3
|
-
|
|
5
|
+
PACKAGES := disc
|
|
6
|
+
VERSION_FILE := lib/disc/version.rb
|
|
4
7
|
|
|
5
|
-
|
|
6
|
-
|
|
8
|
+
DEPS := ${GEM_HOME}/installed
|
|
9
|
+
VERSION := $(shell sed -ne '/.*VERSION *= *"\(.*\)".*/s//\1/p' <$(VERSION_FILE))
|
|
10
|
+
GEMS := $(addprefix pkg/, $(addsuffix -$(VERSION).gem, $(PACKAGES)))
|
|
7
11
|
|
|
8
|
-
|
|
12
|
+
export RUBYLIB := lib:test:$(RUBYLIB)
|
|
13
|
+
|
|
14
|
+
all: test $(GEMS)
|
|
15
|
+
|
|
16
|
+
console: $(DEPS)
|
|
17
|
+
irb -r disc
|
|
18
|
+
|
|
19
|
+
test: $(DEPS)
|
|
20
|
+
cutest ./test/**/*_test.rb
|
|
21
|
+
|
|
22
|
+
clean:
|
|
23
|
+
rm pkg/*.gem
|
|
24
|
+
|
|
25
|
+
release: $(GEMS)
|
|
26
|
+
git tag v$(VERSION)
|
|
27
|
+
git push --tags
|
|
28
|
+
for gem in $^; do gem push $$gem; done
|
|
29
|
+
|
|
30
|
+
pkg/%-$(VERSION).gem: %.gemspec $(VERSION_FILE) | pkg
|
|
31
|
+
gem build $<
|
|
32
|
+
mv $(@F) pkg/
|
|
33
|
+
|
|
34
|
+
$(DEPS): $(GEM_HOME) .gems
|
|
35
|
+
which dep &>/dev/null || gem install dep
|
|
36
|
+
dep install
|
|
37
|
+
touch $(GEM_HOME)/installed
|
|
38
|
+
|
|
39
|
+
pkg:
|
|
40
|
+
mkdir -p $@
|
|
41
|
+
|
|
42
|
+
.PHONY: all test release clean
|
data/README.md
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Disc fills the gap between your Ruby service objects and [antirez](http://antirez.com/)'s wonderful [Disque](https://github.com/antirez/disque) backend.
|
|
4
4
|
|
|
5
|
-
<a href=https://www.flickr.com/photos/noodlefish/5321412234
|
|
6
|
-
![Disc Wars!](
|
|
5
|
+
<a href=https://www.flickr.com/photos/noodlefish/5321412234" target="blank_">
|
|
6
|
+
|
|
7
7
|
</a>
|
|
8
8
|
|
|
9
9
|
## Usage
|
|
@@ -46,19 +46,13 @@ Disc fills the gap between your Ruby service objects and [antirez](http://antire
|
|
|
46
46
|
```ruby
|
|
47
47
|
# disc_init.rb
|
|
48
48
|
require 'ohm'
|
|
49
|
-
Dir
|
|
50
|
-
```
|
|
51
|
-
|
|
52
|
-
6. Set your require file
|
|
53
|
-
|
|
54
|
-
```bash
|
|
55
|
-
$ export DISC_REQUIRE='./disc_init.rb'
|
|
49
|
+
Dir['./jobs/**/*.rb'].each { |job| require job }
|
|
56
50
|
```
|
|
57
51
|
|
|
58
52
|
7. Run as many Disc Worker processes as you wish.
|
|
59
53
|
|
|
60
54
|
```bash
|
|
61
|
-
$ QUEUES=urgent,default disc
|
|
55
|
+
$ QUEUES=urgent,default disc -r ./disc_init.rb
|
|
62
56
|
```
|
|
63
57
|
|
|
64
58
|
## Settings
|
|
@@ -67,7 +61,6 @@ Disc takes its configuration from environment variables.
|
|
|
67
61
|
|
|
68
62
|
| ENV Variable | Default Value | Description
|
|
69
63
|
|:----------------:|:-----------------|:------------|
|
|
70
|
-
| DISC_REQUIRE | null | Ruby file that will be required by the worker processes, it should load all Disc::Job classes on your application and whatever else is needed to run them.
|
|
71
64
|
| QUEUES | 'default' | The list of queues that `Disc::Worker` will listen to, it can be a single queue name or a list of comma-separated queues |
|
|
72
65
|
| DISC_CONCURRENCY | '25' | Amount of threads to spawn when Celluloid is available. |
|
|
73
66
|
| DISQUE_NODES | 'localhost:7711' | This is the list of Disque servers to connect to, it can be a single node or a list of comma-separated nodes |
|
|
@@ -75,16 +68,67 @@ Disc takes its configuration from environment variables.
|
|
|
75
68
|
| DISQUE_TIMEOUT | '100' | Time in milliseconds that the client will wait for the Disque server to acknowledge and replicate a job |
|
|
76
69
|
| DISQUE_CYCLE | '1000' | The client keeps track of which nodes are providing more jobs, after the amount of operations specified in cycle it tries to connect to the preferred node. |
|
|
77
70
|
|
|
71
|
+
## Error handling
|
|
72
|
+
|
|
73
|
+
When a job raises an exception, `Disc.on_error` is invoked with the error and
|
|
74
|
+
the job data. By default, this method prints the error to standard error, but
|
|
75
|
+
you can override it to report the error to your favorite error aggregator.
|
|
76
|
+
|
|
77
|
+
``` ruby
|
|
78
|
+
# On disc_init.rb
|
|
79
|
+
def Disc.on_error(exception, job)
|
|
80
|
+
# ... report the error
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
Dir["./jobs/**/*.rb"].each { |job| require job }
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Lifecycle Callbacks
|
|
78
87
|
|
|
88
|
+
You can optionally define two methods on classes that include `Disc::Job` that
|
|
89
|
+
will act as callbacks:
|
|
79
90
|
|
|
91
|
+
``` ruby
|
|
92
|
+
def disc_start(job)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def disc_done(exception_or_nil)
|
|
96
|
+
end
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Before the `perform` method is invoked, Disc will call `disc_start` and pass the
|
|
100
|
+
job data as a Hash. After the job is finished, it will call `disc_done`.
|
|
101
|
+
|
|
102
|
+
You could use these callbacks for things like aggregating metrics about the jobs
|
|
103
|
+
(timing, number of times a certain job is run, etc), or for advanced logging,
|
|
104
|
+
for example.
|
|
105
|
+
|
|
106
|
+
## Job Definition
|
|
107
|
+
|
|
108
|
+
Both the error handler function and the `disc_start` callback get the data of
|
|
109
|
+
the current job as a Hash, that has the following schema.
|
|
110
|
+
|
|
111
|
+
| `'class'` | (String) The Job class. |
|
|
112
|
+
| `'arguments'` | (Array) The arguments passed to perform. |
|
|
113
|
+
| `'queue'` | (String) The queue from which this job was picked up. |
|
|
114
|
+
| `'id'` | (String) Disque's job ID. |
|
|
80
115
|
|
|
81
116
|
## PowerUps
|
|
82
117
|
|
|
83
|
-
Disc workers can run just fine on their own, but if you're using
|
|
118
|
+
Disc workers can run just fine on their own, but if you're using
|
|
119
|
+
[Celluloid](https://github.com/celluloid/celluloid) you migth want Disc to take
|
|
120
|
+
advantage of it and spawn multiple worker threads per process, doing this is
|
|
121
|
+
trivial! Just require Celluloid before your init file:
|
|
84
122
|
|
|
85
|
-
```
|
|
86
|
-
|
|
87
|
-
require 'celluloid'
|
|
123
|
+
```bash
|
|
124
|
+
$ QUEUES=urgent,default disc -r celluloid/current -r ./disc_init.rb
|
|
88
125
|
```
|
|
89
126
|
|
|
90
|
-
Whenever Disc detects that Celluloid is available it will use it to spawn a
|
|
127
|
+
Whenever Disc detects that Celluloid is available it will use it to spawn a
|
|
128
|
+
number of threads equal to the `DISC_CONCURRENCY` environment variable, or 25 by
|
|
129
|
+
default.
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
The code is released under an MIT license. See the [LICENSE](./LICENSE) file for
|
|
134
|
+
more information.
|
data/bin/disc
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
|
|
3
3
|
require 'disc'
|
|
4
|
-
require
|
|
4
|
+
require 'clap'
|
|
5
|
+
|
|
6
|
+
Clap.run ARGV,
|
|
7
|
+
"-r" => lambda { |file| require file }
|
|
5
8
|
|
|
6
9
|
if defined?(Celluloid)
|
|
7
10
|
concurrency = ENV.fetch('DISC_CONCURRENCY', '25').to_i
|
|
@@ -10,14 +13,24 @@ if defined?(Celluloid)
|
|
|
10
13
|
#{ concurrency }."
|
|
11
14
|
)
|
|
12
15
|
|
|
13
|
-
|
|
14
|
-
pool Disc::Worker,
|
|
15
|
-
size: ENV.fetch('DISC_CONCURRENCY', '25').to_i,
|
|
16
|
-
as: :worker_pool,
|
|
17
|
-
args: [{ run: true }]
|
|
18
|
-
end
|
|
16
|
+
Disc::Worker.send(:include, Celluloid)
|
|
19
17
|
|
|
20
|
-
|
|
18
|
+
if defined?(Celluloid::SupervisionGroup)
|
|
19
|
+
# Deprecated as of Celluloid 0.17, but still supported via "backported mode"
|
|
20
|
+
class Disc::WorkerGroup < Celluloid::SupervisionGroup
|
|
21
|
+
pool Disc::Worker,
|
|
22
|
+
size: ENV.fetch('DISC_CONCURRENCY', '25').to_i,
|
|
23
|
+
as: :worker_pool,
|
|
24
|
+
args: [{ run: true }]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
Disc::WorkerGroup.run
|
|
28
|
+
else
|
|
29
|
+
Disc::Worker.pool(
|
|
30
|
+
size: concurrency,
|
|
31
|
+
args: [{ run: true }]
|
|
32
|
+
)
|
|
33
|
+
end
|
|
21
34
|
else
|
|
22
35
|
STDOUT.puts(
|
|
23
36
|
"[Notice] Disc running in non-threaded mode, consider requiring celluloid\
|
data/disc.gemspec
CHANGED
|
@@ -1,15 +1,19 @@
|
|
|
1
|
+
require_relative "lib/disc/version"
|
|
2
|
+
|
|
1
3
|
Gem::Specification.new do |s|
|
|
2
|
-
s.name
|
|
3
|
-
s.version
|
|
4
|
-
s.summary
|
|
5
|
-
s.description
|
|
6
|
-
s.authors
|
|
7
|
-
s.email
|
|
8
|
-
s.homepage
|
|
9
|
-
s.license
|
|
10
|
-
s.files
|
|
4
|
+
s.name = 'disc'
|
|
5
|
+
s.version = Disc::VERSION
|
|
6
|
+
s.summary = 'A simple and powerful Disque job implementation'
|
|
7
|
+
s.description = 'Easily define and run background jobs using Disque'
|
|
8
|
+
s.authors = ['pote']
|
|
9
|
+
s.email = ['pote@tardis.com.uy']
|
|
10
|
+
s.homepage = 'https://github.com/pote/disc'
|
|
11
|
+
s.license = 'MIT'
|
|
12
|
+
s.files = `git ls-files`.split("\n")
|
|
11
13
|
|
|
12
14
|
s.executables.push('disc')
|
|
15
|
+
|
|
13
16
|
s.add_dependency('disque', '~> 0.0.6')
|
|
14
17
|
s.add_dependency('msgpack', '~> 0.6.1')
|
|
18
|
+
s.add_dependency('clap', '~> 1.0')
|
|
15
19
|
end
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Dir.glob("./**/*.rb") { |f| require f }
|
data/lib/disc.rb
CHANGED
|
@@ -2,6 +2,8 @@ require 'date'
|
|
|
2
2
|
require 'disque'
|
|
3
3
|
require 'msgpack'
|
|
4
4
|
|
|
5
|
+
require_relative 'disc/version'
|
|
6
|
+
|
|
5
7
|
class Disc
|
|
6
8
|
attr_reader :disque,
|
|
7
9
|
:disque_timeout
|
|
@@ -26,9 +28,11 @@ class Disc
|
|
|
26
28
|
@disque_timeout = timeout
|
|
27
29
|
end
|
|
28
30
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
+
def self.on_error(exception, job)
|
|
32
|
+
STDERR.puts exception
|
|
33
|
+
end
|
|
31
34
|
|
|
35
|
+
class Worker
|
|
32
36
|
attr_reader :disque,
|
|
33
37
|
:queues,
|
|
34
38
|
:timeout,
|
|
@@ -59,18 +63,28 @@ class Disc
|
|
|
59
63
|
self
|
|
60
64
|
end
|
|
61
65
|
|
|
62
|
-
|
|
63
66
|
def run
|
|
64
67
|
STDOUT.puts("Disc::Worker listening in #{queues}")
|
|
65
68
|
loop do
|
|
66
|
-
disque.fetch(
|
|
67
|
-
|
|
68
|
-
timeout: timeout,
|
|
69
|
-
count: count
|
|
70
|
-
) do |serialized_job, _|
|
|
69
|
+
jobs = disque.fetch(from: queues, timeout: timeout, count: count)
|
|
70
|
+
Array(jobs).each do |queue, msgid, serialized_job|
|
|
71
71
|
job = MessagePack.unpack(serialized_job)
|
|
72
|
-
|
|
73
|
-
|
|
72
|
+
job.update('id' => msgid, 'queue' => queue)
|
|
73
|
+
|
|
74
|
+
instance = Object.const_get(job['class']).new
|
|
75
|
+
begin
|
|
76
|
+
instance.disc_start(job)
|
|
77
|
+
instance.perform(*job['arguments'])
|
|
78
|
+
disque.call('ACKJOB', msgid)
|
|
79
|
+
rescue => err
|
|
80
|
+
Disc.on_error(err, job)
|
|
81
|
+
ensure
|
|
82
|
+
begin
|
|
83
|
+
instance.disc_done(err)
|
|
84
|
+
rescue => boom
|
|
85
|
+
Disc.on_error(boom, job)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
74
88
|
end
|
|
75
89
|
end
|
|
76
90
|
end
|
|
@@ -85,6 +99,12 @@ class Disc
|
|
|
85
99
|
base.extend(ClassMethods)
|
|
86
100
|
end
|
|
87
101
|
|
|
102
|
+
def disc_start(job)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def disc_done(error = nil)
|
|
106
|
+
end
|
|
107
|
+
|
|
88
108
|
module ClassMethods
|
|
89
109
|
def disque
|
|
90
110
|
defined?(@disque) ? @disque : Disc.disque
|
|
@@ -118,7 +138,7 @@ class Disc
|
|
|
118
138
|
queue,
|
|
119
139
|
args.to_msgpack,
|
|
120
140
|
Disc.disque_timeout,
|
|
121
|
-
delay:
|
|
141
|
+
delay: datetime && (datetime.to_time.to_i - DateTime.now.to_time.to_i)
|
|
122
142
|
)
|
|
123
143
|
end
|
|
124
144
|
|
data/lib/disc/version.rb
ADDED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: disc
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- pote
|
|
@@ -38,7 +38,21 @@ dependencies:
|
|
|
38
38
|
- - "~>"
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: 0.6.1
|
|
41
|
-
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: clap
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '1.0'
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '1.0'
|
|
55
|
+
description: Easily define and run background jobs using Disque
|
|
42
56
|
email:
|
|
43
57
|
- pote@tardis.com.uy
|
|
44
58
|
executables:
|
|
@@ -50,18 +64,19 @@ files:
|
|
|
50
64
|
- ".gems"
|
|
51
65
|
- ".gitignore"
|
|
52
66
|
- CONTRIBUTING.md
|
|
53
|
-
- LICENSE
|
|
67
|
+
- LICENSE
|
|
54
68
|
- Makefile
|
|
55
69
|
- README.md
|
|
56
70
|
- bin/disc
|
|
57
|
-
- disc-wars.jpg
|
|
58
71
|
- disc.gemspec
|
|
59
|
-
-
|
|
72
|
+
- examples/disc_init.rb
|
|
73
|
+
- examples/fail_job.rb
|
|
60
74
|
- examples/failer.rb
|
|
61
75
|
- examples/greeter.rb
|
|
62
76
|
- lib/disc.rb
|
|
77
|
+
- lib/disc/version.rb
|
|
63
78
|
- test/disc_test.rb
|
|
64
|
-
homepage: https://github.com/pote/
|
|
79
|
+
homepage: https://github.com/pote/disc
|
|
65
80
|
licenses:
|
|
66
81
|
- MIT
|
|
67
82
|
metadata: {}
|
data/disc-wars.jpg
DELETED
|
Binary file
|
data/disc_init_example.rb
DELETED