antelopes 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +5 -0
- data/CHANGELOG.md +15 -0
- data/Gemfile +4 -0
- data/README.md +30 -0
- data/Rakefile +12 -0
- data/antelopes.gemspec +23 -0
- data/lib/antelopes/job.rb +42 -0
- data/lib/antelopes/looper.rb +31 -0
- data/lib/antelopes/master.rb +33 -0
- data/lib/antelopes/puller.rb +34 -0
- data/lib/antelopes/pusher.rb +44 -0
- data/lib/antelopes/redis.rb +16 -0
- data/lib/antelopes/worker.rb +37 -0
- metadata +29 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7995a53cb70eeafa32ec80a194bf59ad2bf7ded9
|
4
|
+
data.tar.gz: 5927101da8a299a0c49a62001f0d6d8e6e504404
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d7e22f5ed0604eaa6bcbec7a3336512886006beab1936c1ec151649b93dd316d29e0a827a19b8a6db0bc7a6f1c1a6ab9bee4ebf9229cb71ebfc333590141beba
|
7
|
+
data.tar.gz: fe7732547d156e4e49e57516af1f06628db68d767dfe2f7f31f28635983d107a3339e41eea726ded1744e0bd1b1227077ebd6279f25943323fbc555a15cc5cf8
|
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
Gemfile.lock
|
data/.rubocop.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# Antelopes
|
2
|
+
|
3
|
+
This project endgoal is to be a clean and framework-agnostic background jobs worker system,
|
4
|
+
primilary focused on batches.
|
5
|
+
|
6
|
+
This project is all new and has not yet reached a stable state.
|
7
|
+
|
8
|
+
## Contributing
|
9
|
+
|
10
|
+
* Fork it ( https://github.com/titeiko/antelopes/fork )
|
11
|
+
* Create your feature branch (git checkout -b my-new-feature)
|
12
|
+
* Commit your changes (git commit -am 'Add some feature')
|
13
|
+
* Push to the branch (git push origin my-new-feature)
|
14
|
+
* Create a new Pull Request
|
15
|
+
|
16
|
+
## Development Requirements
|
17
|
+
|
18
|
+
* Ruby 2.3+
|
19
|
+
* Bundler
|
20
|
+
* Redis 3+
|
21
|
+
|
22
|
+
## Testing
|
23
|
+
|
24
|
+
```
|
25
|
+
bundle exec rake
|
26
|
+
```
|
27
|
+
|
28
|
+
## Versioning
|
29
|
+
|
30
|
+
Antelopes uses Semantic Versioning 2.0.0
|
data/Rakefile
ADDED
data/antelopes.gemspec
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = 'antelopes'
|
5
|
+
s.version = '0.0.2'
|
6
|
+
s.summary = 'Nice and smart background jobs'
|
7
|
+
s.authors = ['Marion Duprey']
|
8
|
+
s.email = 'titeiko@gmail.com'
|
9
|
+
s.files = `git ls-files | grep -Ev '^(bin|test)'`.split("\n")
|
10
|
+
s.homepage = 'https://github.com/titeiko/antelopes'
|
11
|
+
s.license = 'MIT'
|
12
|
+
|
13
|
+
s.required_ruby_version = '>= 2.3'
|
14
|
+
|
15
|
+
s.add_runtime_dependency 'connection_pool', '~> 2'
|
16
|
+
s.add_runtime_dependency 'redis', '~> 3'
|
17
|
+
s.add_runtime_dependency 'serverengine', '~> 2'
|
18
|
+
|
19
|
+
s.add_development_dependency 'minitest', '~> 5'
|
20
|
+
s.add_development_dependency 'minitest-reporters', '~> 1'
|
21
|
+
s.add_development_dependency 'rake', '~> 12'
|
22
|
+
s.add_development_dependency 'pry', '~> 0'
|
23
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Job object representation
|
5
|
+
#
|
6
|
+
# @since 0.0.1
|
7
|
+
class Job
|
8
|
+
# @!attribute [r] jid
|
9
|
+
# @return [String] the uuid of the job
|
10
|
+
# @!attribute [r] job_class
|
11
|
+
# @return [String] the class name of the job to run
|
12
|
+
# @!attribute [r] job_method
|
13
|
+
# @return [Symbol] the instance method to execute. nil if job_class_method is set
|
14
|
+
# @!attribute [r] job_class_method
|
15
|
+
# @return [Symbol] the class method to execute. nil if job_method is set
|
16
|
+
# @!attribute [r] job_args
|
17
|
+
# @return [Hash] arguments for the job (class) method
|
18
|
+
attr_reader :jid, :job_class, :job_method, :job_class_method, :job_args
|
19
|
+
|
20
|
+
# Initialization from json hash
|
21
|
+
#
|
22
|
+
# @param json_payload [Hash] deserialized JSON hash
|
23
|
+
# @return [Job] a Job
|
24
|
+
def initialize(json_payload)
|
25
|
+
@jid = json_payload['jid']
|
26
|
+
setup_job_attrs(json_payload['job'])
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def setup_job_attrs(json_payload)
|
32
|
+
@job_class = json_payload['class']
|
33
|
+
@job_method = json_payload['method']&.to_sym
|
34
|
+
@job_class_method = json_payload['class_method']&.to_sym
|
35
|
+
setup_job_args(json_payload['args'])
|
36
|
+
end
|
37
|
+
|
38
|
+
def setup_job_args(args)
|
39
|
+
@job_args = Hash[args.map { |k, v| [k.to_sym, v] }]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# ServerEngine looper.
|
5
|
+
# It has two ways of working: either as a simple worker or
|
6
|
+
# as a manager.
|
7
|
+
# This class should not be used directly by Antelopes users.
|
8
|
+
#
|
9
|
+
# @since 0.0.1
|
10
|
+
# @private
|
11
|
+
module Looper
|
12
|
+
# Method called by {https://github.com/treasure-data/serverengine ServerEngine}
|
13
|
+
# that loops until stopped.
|
14
|
+
#
|
15
|
+
# @since 0.0.1
|
16
|
+
def run
|
17
|
+
logger.info 'Looper started'
|
18
|
+
@runner = Worker.new(logger: logger)
|
19
|
+
@runner.run until @stop
|
20
|
+
end
|
21
|
+
|
22
|
+
# Method called by {https://github.com/treasure-data/serverengine ServerEngine}
|
23
|
+
# to stop the worker when the service receives a signal to stop or restart.
|
24
|
+
#
|
25
|
+
# @since 0.0.1
|
26
|
+
def stop
|
27
|
+
logger.info 'Looper shutting down'
|
28
|
+
@stop = true
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# ServerEngine master.
|
5
|
+
# Its job is to create a connection pool towards Redis and
|
6
|
+
# share it with the loopers.
|
7
|
+
# This class should not be used directly by Antelopes users.
|
8
|
+
#
|
9
|
+
# @since 0.0.1
|
10
|
+
# @private
|
11
|
+
module Master
|
12
|
+
# @!attribute [r] redis
|
13
|
+
# @return [ConnectionPool] the a redis connection pool
|
14
|
+
attr_reader :redis
|
15
|
+
|
16
|
+
# Method called by ServerEngine before starting the workers.
|
17
|
+
# It initialize the redis connection pool used by the Loopers.
|
18
|
+
#
|
19
|
+
# @since 0.0.1
|
20
|
+
def before_run
|
21
|
+
logger.info 'Master starting'
|
22
|
+
@redis = ConnectionPool.new(size: 5, timeout: 3) { Antelopes::Redis.new.connection }
|
23
|
+
end
|
24
|
+
|
25
|
+
# Method called by ServerEngine before shutting down
|
26
|
+
#
|
27
|
+
# @since 0.0.1
|
28
|
+
def after_run
|
29
|
+
logger.info 'Master shutting down'
|
30
|
+
@redis.shutdown(&:quit)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Class used to pull jobs
|
5
|
+
# This should not be used directly by Antelopes users.
|
6
|
+
#
|
7
|
+
# @since 0.0.1
|
8
|
+
# @private
|
9
|
+
class Puller
|
10
|
+
# Initialization
|
11
|
+
#
|
12
|
+
# @param logger [ServerEngine::DaemonLogger] a logger
|
13
|
+
# @param redis [Redis] a {https://github.com/redis/redis-rb redis} connection
|
14
|
+
def initialize(logger: nil, redis: nil)
|
15
|
+
@logger = logger || ServerEngine::DaemonLogger.new($stdout)
|
16
|
+
@redis = redis || Antelopes::Redis.new.connection
|
17
|
+
end
|
18
|
+
|
19
|
+
# Method used by the workers to get a job to work on.
|
20
|
+
# When the job is started, it goes in the 'doing' list.
|
21
|
+
#
|
22
|
+
# @return [Hash] the job
|
23
|
+
def next_todo
|
24
|
+
jid = redis.brpoplpush('antelopes:todo', 'antelopes:doing', timeout: 1)
|
25
|
+
|
26
|
+
return if jid.nil?
|
27
|
+
Job.new(JSON.parse(redis.get("antelopes:job:#{jid}")))
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
attr_reader :redis, :logger
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Class used to push jobs in the todo queue
|
5
|
+
# This should not be used directly by Antelopes users.
|
6
|
+
#
|
7
|
+
# @since 0.0.1
|
8
|
+
# @private
|
9
|
+
class Pusher
|
10
|
+
# Initialization
|
11
|
+
#
|
12
|
+
# @param logger [ServerEngine::DaemonLogger] a logger
|
13
|
+
# @param redis [Redis] a {https://github.com/redis/redis-rb redis} connection
|
14
|
+
def initialize(logger: nil, redis: nil)
|
15
|
+
@logger = logger || ServerEngine::DaemonLogger.new($stdout)
|
16
|
+
@redis = redis || Antelopes::Redis.new.connection
|
17
|
+
end
|
18
|
+
|
19
|
+
# Mechod that actually adds the job to queue
|
20
|
+
#
|
21
|
+
# @example Enqueing a job
|
22
|
+
# result = Antelopes::Pusher.new.call(
|
23
|
+
# job: Hash[class: 'MyClass', method: 'call', args: Hash[foo: 'bar']]
|
24
|
+
# )
|
25
|
+
# result.jid
|
26
|
+
#
|
27
|
+
# @param job_params [Hash] parameters of the job
|
28
|
+
# @return [OpenStruct] response object
|
29
|
+
def call(job_params)
|
30
|
+
@result = OpenStruct.new(jid: SecureRandom.uuid)
|
31
|
+
|
32
|
+
redis.set("antelopes:job:#{@result.jid}", JSON.generate(job_params.merge(jid: @result.jid)))
|
33
|
+
redis.lpush('antelopes:todo', @result.jid)
|
34
|
+
|
35
|
+
logger.info "Pushed #{@result.jid} - #{job_params}"
|
36
|
+
|
37
|
+
@result
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
attr_reader :redis, :logger
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Class used to generate a Redis connection
|
5
|
+
#
|
6
|
+
# @since 0.0.1
|
7
|
+
class Redis
|
8
|
+
# Retrieve a Redis connection
|
9
|
+
#
|
10
|
+
# @todo Make it configurable
|
11
|
+
# @return [::Redis] a {https://github.com/redis/redis-rb Redis} connection
|
12
|
+
def connection
|
13
|
+
::Redis.new
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Antelopes
|
4
|
+
# Basic worker that pulls a job, runs it, reports and repeats.
|
5
|
+
#
|
6
|
+
# @since 0.0.1
|
7
|
+
# @private
|
8
|
+
class Worker
|
9
|
+
# Worker initialization.
|
10
|
+
#
|
11
|
+
# @param logger [ServerEngine::DaemonLogger] a logger
|
12
|
+
# @param puller
|
13
|
+
def initialize(logger: ServerEngine::DaemonLogger.new($stdout), puller:)
|
14
|
+
@logger = logger
|
15
|
+
@puller = puller
|
16
|
+
end
|
17
|
+
|
18
|
+
# Method called by the looper at every loop.
|
19
|
+
#
|
20
|
+
# @since 0.0.1
|
21
|
+
def run
|
22
|
+
job = puller.pull
|
23
|
+
return if job.nil?
|
24
|
+
|
25
|
+
klass = Object.const_get(job.job_class)
|
26
|
+
if job.job_method.nil?
|
27
|
+
klass.public_send(job.job_class_method, **job.job_args)
|
28
|
+
else
|
29
|
+
klass.new.public_send(job.job_method, **job.job_args)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
attr_reader :logger, :puller
|
36
|
+
end
|
37
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: antelopes
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marion Duprey
|
@@ -56,56 +56,56 @@ dependencies:
|
|
56
56
|
name: minitest
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - "
|
59
|
+
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '5'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - "
|
66
|
+
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
68
|
+
version: '5'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: minitest-reporters
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- - "
|
73
|
+
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '
|
75
|
+
version: '1'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- - "
|
80
|
+
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '
|
82
|
+
version: '1'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: rake
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - "
|
87
|
+
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
89
|
+
version: '12'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - "
|
94
|
+
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
96
|
+
version: '12'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: pry
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- - "
|
101
|
+
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '0'
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- - "
|
108
|
+
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
111
|
description:
|
@@ -114,7 +114,21 @@ executables: []
|
|
114
114
|
extensions: []
|
115
115
|
extra_rdoc_files: []
|
116
116
|
files:
|
117
|
+
- ".gitignore"
|
118
|
+
- ".rubocop.yml"
|
119
|
+
- CHANGELOG.md
|
120
|
+
- Gemfile
|
121
|
+
- README.md
|
122
|
+
- Rakefile
|
123
|
+
- antelopes.gemspec
|
117
124
|
- lib/antelopes.rb
|
125
|
+
- lib/antelopes/job.rb
|
126
|
+
- lib/antelopes/looper.rb
|
127
|
+
- lib/antelopes/master.rb
|
128
|
+
- lib/antelopes/puller.rb
|
129
|
+
- lib/antelopes/pusher.rb
|
130
|
+
- lib/antelopes/redis.rb
|
131
|
+
- lib/antelopes/worker.rb
|
118
132
|
homepage: https://github.com/titeiko/antelopes
|
119
133
|
licenses:
|
120
134
|
- MIT
|