antelopes 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +5 -0
- data/CHANGELOG.md +15 -0
- data/Gemfile +4 -0
- data/README.md +30 -0
- data/Rakefile +12 -0
- data/antelopes.gemspec +23 -0
- data/lib/antelopes/job.rb +42 -0
- data/lib/antelopes/looper.rb +31 -0
- data/lib/antelopes/master.rb +33 -0
- data/lib/antelopes/puller.rb +34 -0
- data/lib/antelopes/pusher.rb +44 -0
- data/lib/antelopes/redis.rb +16 -0
- data/lib/antelopes/worker.rb +37 -0
- metadata +29 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7995a53cb70eeafa32ec80a194bf59ad2bf7ded9
|
4
|
+
data.tar.gz: 5927101da8a299a0c49a62001f0d6d8e6e504404
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d7e22f5ed0604eaa6bcbec7a3336512886006beab1936c1ec151649b93dd316d29e0a827a19b8a6db0bc7a6f1c1a6ab9bee4ebf9229cb71ebfc333590141beba
|
7
|
+
data.tar.gz: fe7732547d156e4e49e57516af1f06628db68d767dfe2f7f31f28635983d107a3339e41eea726ded1744e0bd1b1227077ebd6279f25943323fbc555a15cc5cf8
|
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Gemfile.lock
|
data/.rubocop.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# Antelopes
|
2
|
+
|
3
|
+
This project endgoal is to be a clean and framework-agnostic background jobs worker system,
|
4
|
+
primilary focused on batches.
|
5
|
+
|
6
|
+
This project is all new and has not yet reached a stable state.
|
7
|
+
|
8
|
+
## Contributing
|
9
|
+
|
10
|
+
* Fork it ( https://github.com/titeiko/antelopes/fork )
|
11
|
+
* Create your feature branch (git checkout -b my-new-feature)
|
12
|
+
* Commit your changes (git commit -am 'Add some feature')
|
13
|
+
* Push to the branch (git push origin my-new-feature)
|
14
|
+
* Create a new Pull Request
|
15
|
+
|
16
|
+
## Development Requirements
|
17
|
+
|
18
|
+
* Ruby 2.3+
|
19
|
+
* Bundler
|
20
|
+
* Redis 3+
|
21
|
+
|
22
|
+
## Testing
|
23
|
+
|
24
|
+
```
|
25
|
+
bundle exec rake
|
26
|
+
```
|
27
|
+
|
28
|
+
## Versioning
|
29
|
+
|
30
|
+
Antelopes uses Semantic Versioning 2.0.0
|
data/Rakefile
ADDED
data/antelopes.gemspec
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = 'antelopes'
|
5
|
+
s.version = '0.0.2'
|
6
|
+
s.summary = 'Nice and smart background jobs'
|
7
|
+
s.authors = ['Marion Duprey']
|
8
|
+
s.email = 'titeiko@gmail.com'
|
9
|
+
s.files = `git ls-files | grep -Ev '^(bin|test)'`.split("\n")
|
10
|
+
s.homepage = 'https://github.com/titeiko/antelopes'
|
11
|
+
s.license = 'MIT'
|
12
|
+
|
13
|
+
s.required_ruby_version = '>= 2.3'
|
14
|
+
|
15
|
+
s.add_runtime_dependency 'connection_pool', '~> 2'
|
16
|
+
s.add_runtime_dependency 'redis', '~> 3'
|
17
|
+
s.add_runtime_dependency 'serverengine', '~> 2'
|
18
|
+
|
19
|
+
s.add_development_dependency 'minitest', '~> 5'
|
20
|
+
s.add_development_dependency 'minitest-reporters', '~> 1'
|
21
|
+
s.add_development_dependency 'rake', '~> 12'
|
22
|
+
s.add_development_dependency 'pry', '~> 0'
|
23
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Job object representation
|
5
|
+
#
|
6
|
+
# @since 0.0.1
|
7
|
+
class Job
|
8
|
+
# @!attribute [r] jid
|
9
|
+
# @return [String] the uuid of the job
|
10
|
+
# @!attribute [r] job_class
|
11
|
+
# @return [String] the class name of the job to run
|
12
|
+
# @!attribute [r] job_method
|
13
|
+
# @return [Symbol] the instance method to execute. nil if job_class_method is set
|
14
|
+
# @!attribute [r] job_class_method
|
15
|
+
# @return [Symbol] the class method to execute. nil if job_method is set
|
16
|
+
# @!attribute [r] job_args
|
17
|
+
# @return [Hash] arguments for the job (class) method
|
18
|
+
attr_reader :jid, :job_class, :job_method, :job_class_method, :job_args
|
19
|
+
|
20
|
+
# Initialization from json hash
|
21
|
+
#
|
22
|
+
# @param json_payload [Hash] deserialized JSON hash
|
23
|
+
# @return [Job] a Job
|
24
|
+
def initialize(json_payload)
|
25
|
+
@jid = json_payload['jid']
|
26
|
+
setup_job_attrs(json_payload['job'])
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def setup_job_attrs(json_payload)
|
32
|
+
@job_class = json_payload['class']
|
33
|
+
@job_method = json_payload['method']&.to_sym
|
34
|
+
@job_class_method = json_payload['class_method']&.to_sym
|
35
|
+
setup_job_args(json_payload['args'])
|
36
|
+
end
|
37
|
+
|
38
|
+
def setup_job_args(args)
|
39
|
+
@job_args = Hash[args.map { |k, v| [k.to_sym, v] }]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# ServerEngine looper.
|
5
|
+
# It has two ways of working: either as a simple worker or
|
6
|
+
# as a manager.
|
7
|
+
# This class should not be used directly by Antelopes users.
|
8
|
+
#
|
9
|
+
# @since 0.0.1
|
10
|
+
# @private
|
11
|
+
module Looper
|
12
|
+
# Method called by {https://github.com/treasure-data/serverengine ServerEngine}
|
13
|
+
# that loops until stopped.
|
14
|
+
#
|
15
|
+
# @since 0.0.1
|
16
|
+
def run
|
17
|
+
logger.info 'Looper started'
|
18
|
+
@runner = Worker.new(logger: logger)
|
19
|
+
@runner.run until @stop
|
20
|
+
end
|
21
|
+
|
22
|
+
# Method called by {https://github.com/treasure-data/serverengine ServerEngine}
|
23
|
+
# to stop the worker when the service receives a signal to stop or restart.
|
24
|
+
#
|
25
|
+
# @since 0.0.1
|
26
|
+
def stop
|
27
|
+
logger.info 'Looper shutting down'
|
28
|
+
@stop = true
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# ServerEngine master.
|
5
|
+
# Its job is to create a connection pool towards Redis and
|
6
|
+
# share it with the loopers.
|
7
|
+
# This class should not be used directly by Antelopes users.
|
8
|
+
#
|
9
|
+
# @since 0.0.1
|
10
|
+
# @private
|
11
|
+
module Master
|
12
|
+
# @!attribute [r] redis
|
13
|
+
# @return [ConnectionPool] the a redis connection pool
|
14
|
+
attr_reader :redis
|
15
|
+
|
16
|
+
# Method called by ServerEngine before starting the workers.
|
17
|
+
# It initialize the redis connection pool used by the Loopers.
|
18
|
+
#
|
19
|
+
# @since 0.0.1
|
20
|
+
def before_run
|
21
|
+
logger.info 'Master starting'
|
22
|
+
@redis = ConnectionPool.new(size: 5, timeout: 3) { Antelopes::Redis.new.connection }
|
23
|
+
end
|
24
|
+
|
25
|
+
# Method called by ServerEngine before shutting down
|
26
|
+
#
|
27
|
+
# @since 0.0.1
|
28
|
+
def after_run
|
29
|
+
logger.info 'Master shutting down'
|
30
|
+
@redis.shutdown(&:quit)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Class used to pull jobs
|
5
|
+
# This should not be used directly by Antelopes users.
|
6
|
+
#
|
7
|
+
# @since 0.0.1
|
8
|
+
# @private
|
9
|
+
class Puller
|
10
|
+
# Initialization
|
11
|
+
#
|
12
|
+
# @param logger [ServerEngine::DaemonLogger] a logger
|
13
|
+
# @param redis [Redis] a {https://github.com/redis/redis-rb redis} connection
|
14
|
+
def initialize(logger: nil, redis: nil)
|
15
|
+
@logger = logger || ServerEngine::DaemonLogger.new($stdout)
|
16
|
+
@redis = redis || Antelopes::Redis.new.connection
|
17
|
+
end
|
18
|
+
|
19
|
+
# Method used by the workers to get a job to work on.
|
20
|
+
# When the job is started, it goes in the 'doing' list.
|
21
|
+
#
|
22
|
+
# @return [Hash] the job
|
23
|
+
def next_todo
|
24
|
+
jid = redis.brpoplpush('antelopes:todo', 'antelopes:doing', timeout: 1)
|
25
|
+
|
26
|
+
return if jid.nil?
|
27
|
+
Job.new(JSON.parse(redis.get("antelopes:job:#{jid}")))
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
attr_reader :redis, :logger
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Class used to push jobs in the todo queue
|
5
|
+
# This should not be used directly by Antelopes users.
|
6
|
+
#
|
7
|
+
# @since 0.0.1
|
8
|
+
# @private
|
9
|
+
class Pusher
|
10
|
+
# Initialization
|
11
|
+
#
|
12
|
+
# @param logger [ServerEngine::DaemonLogger] a logger
|
13
|
+
# @param redis [Redis] a {https://github.com/redis/redis-rb redis} connection
|
14
|
+
def initialize(logger: nil, redis: nil)
|
15
|
+
@logger = logger || ServerEngine::DaemonLogger.new($stdout)
|
16
|
+
@redis = redis || Antelopes::Redis.new.connection
|
17
|
+
end
|
18
|
+
|
19
|
+
# Mechod that actually adds the job to queue
|
20
|
+
#
|
21
|
+
# @example Enqueing a job
|
22
|
+
# result = Antelopes::Pusher.new.call(
|
23
|
+
# job: Hash[class: 'MyClass', method: 'call', args: Hash[foo: 'bar']]
|
24
|
+
# )
|
25
|
+
# result.jid
|
26
|
+
#
|
27
|
+
# @param job_params [Hash] parameters of the job
|
28
|
+
# @return [OpenStruct] response object
|
29
|
+
def call(job_params)
|
30
|
+
@result = OpenStruct.new(jid: SecureRandom.uuid)
|
31
|
+
|
32
|
+
redis.set("antelopes:job:#{@result.jid}", JSON.generate(job_params.merge(jid: @result.jid)))
|
33
|
+
redis.lpush('antelopes:todo', @result.jid)
|
34
|
+
|
35
|
+
logger.info "Pushed #{@result.jid} - #{job_params}"
|
36
|
+
|
37
|
+
@result
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
attr_reader :redis, :logger
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Class used to generate a Redis connection
|
5
|
+
#
|
6
|
+
# @since 0.0.1
|
7
|
+
class Redis
|
8
|
+
# Retrieve a Redis connection
|
9
|
+
#
|
10
|
+
# @todo Make it configurable
|
11
|
+
# @return [::Redis] a {https://github.com/redis/redis-rb Redis} connection
|
12
|
+
def connection
|
13
|
+
::Redis.new
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Basic worker that pulls a job, runs it, reports and repeats.
|
5
|
+
#
|
6
|
+
# @since 0.0.1
|
7
|
+
# @private
|
8
|
+
class Worker
|
9
|
+
# Worker initialization.
|
10
|
+
#
|
11
|
+
# @param logger [ServerEngine::DaemonLogger] a logger
|
12
|
+
# @param puller
|
13
|
+
def initialize(logger: ServerEngine::DaemonLogger.new($stdout), puller:)
|
14
|
+
@logger = logger
|
15
|
+
@puller = puller
|
16
|
+
end
|
17
|
+
|
18
|
+
# Method called by the looper at every loop.
|
19
|
+
#
|
20
|
+
# @since 0.0.1
|
21
|
+
def run
|
22
|
+
job = puller.pull
|
23
|
+
return if job.nil?
|
24
|
+
|
25
|
+
klass = Object.const_get(job.job_class)
|
26
|
+
if job.job_method.nil?
|
27
|
+
klass.public_send(job.job_class_method, **job.job_args)
|
28
|
+
else
|
29
|
+
klass.new.public_send(job.job_method, **job.job_args)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
attr_reader :logger, :puller
|
36
|
+
end
|
37
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: antelopes
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marion Duprey
|
@@ -56,56 +56,56 @@ dependencies:
|
|
56
56
|
name: minitest
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '5'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '5'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: minitest-reporters
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - "
|
73
|
+
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
75
|
+
version: '1'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - "
|
80
|
+
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
82
|
+
version: '1'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: rake
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - "
|
87
|
+
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '12'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - "
|
94
|
+
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '12'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: pry
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- - "
|
101
|
+
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '0'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- - "
|
108
|
+
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
111
|
description:
|
@@ -114,7 +114,21 @@ executables: []
|
|
114
114
|
extensions: []
|
115
115
|
extra_rdoc_files: []
|
116
116
|
files:
|
117
|
+
- ".gitignore"
|
118
|
+
- ".rubocop.yml"
|
119
|
+
- CHANGELOG.md
|
120
|
+
- Gemfile
|
121
|
+
- README.md
|
122
|
+
- Rakefile
|
123
|
+
- antelopes.gemspec
|
117
124
|
- lib/antelopes.rb
|
125
|
+
- lib/antelopes/job.rb
|
126
|
+
- lib/antelopes/looper.rb
|
127
|
+
- lib/antelopes/master.rb
|
128
|
+
- lib/antelopes/puller.rb
|
129
|
+
- lib/antelopes/pusher.rb
|
130
|
+
- lib/antelopes/redis.rb
|
131
|
+
- lib/antelopes/worker.rb
|
118
132
|
homepage: https://github.com/titeiko/antelopes
|
119
133
|
licenses:
|
120
134
|
- MIT
|