cloudtasker 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.rspec +3 -0
- data/.rubocop.yml +27 -0
- data/.travis.yml +7 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +247 -0
- data/LICENSE.txt +21 -0
- data/README.md +43 -0
- data/Rakefile +8 -0
- data/app/controllers/cloudtasker/application_controller.rb +6 -0
- data/app/controllers/cloudtasker/worker_controller.rb +38 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/cloudtasker.gemspec +48 -0
- data/config/routes.rb +5 -0
- data/lib/cloudtasker.rb +31 -0
- data/lib/cloudtasker/authentication_error.rb +6 -0
- data/lib/cloudtasker/authenticator.rb +55 -0
- data/lib/cloudtasker/batch.rb +5 -0
- data/lib/cloudtasker/batch/batch_progress.rb +97 -0
- data/lib/cloudtasker/batch/config.rb +11 -0
- data/lib/cloudtasker/batch/extension/worker.rb +13 -0
- data/lib/cloudtasker/batch/job.rb +320 -0
- data/lib/cloudtasker/batch/middleware.rb +24 -0
- data/lib/cloudtasker/batch/middleware/server.rb +14 -0
- data/lib/cloudtasker/config.rb +122 -0
- data/lib/cloudtasker/cron.rb +5 -0
- data/lib/cloudtasker/cron/config.rb +11 -0
- data/lib/cloudtasker/cron/job.rb +207 -0
- data/lib/cloudtasker/cron/middleware.rb +21 -0
- data/lib/cloudtasker/cron/middleware/server.rb +14 -0
- data/lib/cloudtasker/cron/schedule.rb +227 -0
- data/lib/cloudtasker/engine.rb +20 -0
- data/lib/cloudtasker/invalid_worker_error.rb +6 -0
- data/lib/cloudtasker/meta_store.rb +86 -0
- data/lib/cloudtasker/middleware/chain.rb +250 -0
- data/lib/cloudtasker/redis_client.rb +115 -0
- data/lib/cloudtasker/task.rb +175 -0
- data/lib/cloudtasker/unique_job.rb +5 -0
- data/lib/cloudtasker/unique_job/config.rb +10 -0
- data/lib/cloudtasker/unique_job/conflict_strategy/base_strategy.rb +37 -0
- data/lib/cloudtasker/unique_job/conflict_strategy/raise.rb +28 -0
- data/lib/cloudtasker/unique_job/conflict_strategy/reject.rb +11 -0
- data/lib/cloudtasker/unique_job/conflict_strategy/reschedule.rb +30 -0
- data/lib/cloudtasker/unique_job/job.rb +136 -0
- data/lib/cloudtasker/unique_job/lock/base_lock.rb +70 -0
- data/lib/cloudtasker/unique_job/lock/no_op.rb +11 -0
- data/lib/cloudtasker/unique_job/lock/until_executed.rb +34 -0
- data/lib/cloudtasker/unique_job/lock/until_executing.rb +30 -0
- data/lib/cloudtasker/unique_job/lock/while_executing.rb +23 -0
- data/lib/cloudtasker/unique_job/lock_error.rb +8 -0
- data/lib/cloudtasker/unique_job/middleware.rb +36 -0
- data/lib/cloudtasker/unique_job/middleware/client.rb +14 -0
- data/lib/cloudtasker/unique_job/middleware/server.rb +14 -0
- data/lib/cloudtasker/version.rb +5 -0
- data/lib/cloudtasker/worker.rb +211 -0
- metadata +286 -0
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Cloudtasker
|
4
|
+
# Cloudtasker Rails engine
|
5
|
+
class Engine < ::Rails::Engine
|
6
|
+
isolate_namespace Cloudtasker
|
7
|
+
|
8
|
+
initializer 'cloudtasker', before: :load_config_initializers do
|
9
|
+
Rails.application.routes.append do
|
10
|
+
mount Cloudtasker::Engine, at: '/cloudtasker'
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
config.generators do |g|
|
15
|
+
g.test_framework :rspec, fixture: false
|
16
|
+
g.assets false
|
17
|
+
g.helper false
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Cloudtasker
|
4
|
+
# Manage meta information on workers. This meta stored is intended
|
5
|
+
# to be used by middlewares needing to store extra information on the
|
6
|
+
# job.
|
7
|
+
# The objective of this class is to provide a shared store to middleware
|
8
|
+
# while controlling access to its keys by preveenting access the hash directly
|
9
|
+
# (e.g. avoid wild merge or replace operations).
|
10
|
+
class MetaStore
|
11
|
+
#
|
12
|
+
# Build a new instance of the class.
|
13
|
+
#
|
14
|
+
# @param [<Type>] hash The worker meta hash
|
15
|
+
#
|
16
|
+
def initialize(hash = {})
|
17
|
+
@meta = JSON.parse((hash || {}).to_json, symbolize_names: true)
|
18
|
+
end
|
19
|
+
|
20
|
+
#
|
21
|
+
# Retrieve meta entry.
|
22
|
+
#
|
23
|
+
# @param [String, Symbol] key The key of the meta entry.
|
24
|
+
#
|
25
|
+
# @return [Any] The value of the meta entry.
|
26
|
+
#
|
27
|
+
def get(key)
|
28
|
+
@meta[key.to_sym] if key
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# Set meta entry
|
33
|
+
#
|
34
|
+
# @param [String, Symbol] key The key of the meta entry.
|
35
|
+
# @param [Any] val The value of the meta entry.
|
36
|
+
#
|
37
|
+
# @return [Any] The value set
|
38
|
+
#
|
39
|
+
def set(key, val)
|
40
|
+
@meta[key.to_sym] = val if key
|
41
|
+
end
|
42
|
+
|
43
|
+
#
|
44
|
+
# Remove a meta information.
|
45
|
+
#
|
46
|
+
# @param [String, Symbol] key The key of the entry to delete.
|
47
|
+
#
|
48
|
+
# @return [Any] The value of the deleted key
|
49
|
+
#
|
50
|
+
def del(key)
|
51
|
+
@meta.delete(key.to_sym) if key
|
52
|
+
end
|
53
|
+
|
54
|
+
#
|
55
|
+
# Return the meta store as Hash.
|
56
|
+
#
|
57
|
+
# @return [Hash] The meta store as Hash.
|
58
|
+
#
|
59
|
+
def to_h
|
60
|
+
# Deep dup
|
61
|
+
JSON.parse(@meta.to_json, symbolize_names: true)
|
62
|
+
end
|
63
|
+
|
64
|
+
#
|
65
|
+
# Return the meta store as json.
|
66
|
+
#
|
67
|
+
# @param [Array<any>] *arg The to_json args.
|
68
|
+
#
|
69
|
+
# @return [String] The meta store as json.
|
70
|
+
#
|
71
|
+
def to_json(*arg)
|
72
|
+
@meta.to_json(*arg)
|
73
|
+
end
|
74
|
+
|
75
|
+
#
|
76
|
+
# Equality operator.
|
77
|
+
#
|
78
|
+
# @param [Any] other The object being compared.
|
79
|
+
#
|
80
|
+
# @return [Boolean] True if the object is equal.
|
81
|
+
#
|
82
|
+
def ==(other)
|
83
|
+
to_json == other.try(:to_json)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,250 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Cloudtasker
|
4
|
+
module Middleware
|
5
|
+
# The class below was originally taken from Sidekiq.
|
6
|
+
# See: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/middleware/chain.rb
|
7
|
+
#
|
8
|
+
# Middleware are callables configured to run before/after a message is processed.
|
9
|
+
# Middlewares can be configured to run on the client side (when jobs are pushed
|
10
|
+
# to Cloud Tasks) as well as on the server side (when jobs are processed by
|
11
|
+
# your application)
|
12
|
+
#
|
13
|
+
# To add middleware for the client:
|
14
|
+
#
|
15
|
+
# Cloudtasker.configure do |config|
|
16
|
+
# config.client_middleware do |chain|
|
17
|
+
# chain.add MyClientHook
|
18
|
+
# end
|
19
|
+
# end
|
20
|
+
#
|
21
|
+
# To modify middleware for the server, just call
|
22
|
+
# with another block:
|
23
|
+
#
|
24
|
+
# Cloudtasker.configure do |config|
|
25
|
+
# config.server_middleware do |chain|
|
26
|
+
# chain.add MyServerHook
|
27
|
+
# chain.remove ActiveRecord
|
28
|
+
# end
|
29
|
+
# end
|
30
|
+
#
|
31
|
+
# To insert immediately preceding another entry:
|
32
|
+
#
|
33
|
+
# Cloudtasker.configure do |config|
|
34
|
+
# config.client_middleware do |chain|
|
35
|
+
# chain.insert_before ActiveRecord, MyClientHook
|
36
|
+
# end
|
37
|
+
# end
|
38
|
+
#
|
39
|
+
# To insert immediately after another entry:
|
40
|
+
#
|
41
|
+
# Cloudtasker.configure do |config|
|
42
|
+
# config.client_middleware do |chain|
|
43
|
+
# chain.insert_after ActiveRecord, MyClientHook
|
44
|
+
# end
|
45
|
+
# end
|
46
|
+
#
|
47
|
+
# This is an example of a minimal server middleware:
|
48
|
+
#
|
49
|
+
# class MyServerHook
|
50
|
+
# def call(worker_instance, msg, queue)
|
51
|
+
# puts "Before work"
|
52
|
+
# yield
|
53
|
+
# puts "After work"
|
54
|
+
# end
|
55
|
+
# end
|
56
|
+
#
|
57
|
+
# This is an example of a minimal client middleware, note
|
58
|
+
# the method must return the result or the job will not push
|
59
|
+
# to Redis:
|
60
|
+
#
|
61
|
+
# class MyClientHook
|
62
|
+
# def call(worker_class, msg, queue, redis_pool)
|
63
|
+
# puts "Before push"
|
64
|
+
# result = yield
|
65
|
+
# puts "After push"
|
66
|
+
# result
|
67
|
+
# end
|
68
|
+
# end
|
69
|
+
#
|
70
|
+
class Chain
|
71
|
+
include Enumerable
|
72
|
+
|
73
|
+
#
|
74
|
+
# Build a new middleware chain.
|
75
|
+
#
|
76
|
+
def initialize
|
77
|
+
@entries = nil
|
78
|
+
yield self if block_given?
|
79
|
+
end
|
80
|
+
|
81
|
+
#
|
82
|
+
# Iterate over the list middlewares and execute the block on each item.
|
83
|
+
#
|
84
|
+
# @param [Proc] &block The block to execute on each item.
|
85
|
+
#
|
86
|
+
def each(&block)
|
87
|
+
entries.each(&block)
|
88
|
+
end
|
89
|
+
|
90
|
+
#
|
91
|
+
# Return the list of middlewares.
|
92
|
+
#
|
93
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The list of middlewares
|
94
|
+
#
|
95
|
+
def entries
|
96
|
+
@entries ||= []
|
97
|
+
end
|
98
|
+
|
99
|
+
#
|
100
|
+
# Remove a middleware from the list.
|
101
|
+
#
|
102
|
+
# @param [Class] klass The middleware class to remove.
|
103
|
+
#
|
104
|
+
def remove(klass)
|
105
|
+
entries.delete_if { |entry| entry.klass == klass }
|
106
|
+
end
|
107
|
+
|
108
|
+
#
|
109
|
+
# Add a middleware at the end of the list.
|
110
|
+
#
|
111
|
+
# @param [Class] klass The middleware class to add.
|
112
|
+
# @param [Arry<any>] *args The list of arguments to the middleware.
|
113
|
+
#
|
114
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
115
|
+
#
|
116
|
+
def add(klass, *args)
|
117
|
+
remove(klass) if exists?(klass)
|
118
|
+
entries << Entry.new(klass, *args)
|
119
|
+
end
|
120
|
+
|
121
|
+
#
|
122
|
+
# Add a middleware at the beginning of the list.
|
123
|
+
#
|
124
|
+
# @param [Class] klass The middleware class to add.
|
125
|
+
# @param [Arry<any>] *args The list of arguments to the middleware.
|
126
|
+
#
|
127
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
128
|
+
#
|
129
|
+
def prepend(klass, *args)
|
130
|
+
remove(klass) if exists?(klass)
|
131
|
+
entries.insert(0, Entry.new(klass, *args))
|
132
|
+
end
|
133
|
+
|
134
|
+
#
|
135
|
+
# Add a middleware before another middleware.
|
136
|
+
#
|
137
|
+
# @param [Class] oldklass The middleware class before which the new middleware should be inserted.
|
138
|
+
# @param [Class] newklass The middleware class to insert.
|
139
|
+
# @param [Arry<any>] *args The list of arguments for the inserted middleware.
|
140
|
+
#
|
141
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
142
|
+
#
|
143
|
+
def insert_before(oldklass, newklass, *args)
|
144
|
+
i = entries.index { |entry| entry.klass == newklass }
|
145
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
146
|
+
i = entries.index { |entry| entry.klass == oldklass } || 0
|
147
|
+
entries.insert(i, new_entry)
|
148
|
+
end
|
149
|
+
|
150
|
+
#
|
151
|
+
# Add a middleware after another middleware.
|
152
|
+
#
|
153
|
+
# @param [Class] oldklass The middleware class after which the new middleware should be inserted.
|
154
|
+
# @param [Class] newklass The middleware class to insert.
|
155
|
+
# @param [Arry<any>] *args The list of arguments for the inserted middleware.
|
156
|
+
#
|
157
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
158
|
+
#
|
159
|
+
def insert_after(oldklass, newklass, *args)
|
160
|
+
i = entries.index { |entry| entry.klass == newklass }
|
161
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
162
|
+
i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
|
163
|
+
entries.insert(i + 1, new_entry)
|
164
|
+
end
|
165
|
+
|
166
|
+
#
|
167
|
+
# Checks if middleware has been added to the list.
|
168
|
+
#
|
169
|
+
# @param [Class] klass The middleware class to check.
|
170
|
+
#
|
171
|
+
# @return [Boolean] Return true if the middleware is in the list.
|
172
|
+
#
|
173
|
+
def exists?(klass)
|
174
|
+
any? { |entry| entry.klass == klass }
|
175
|
+
end
|
176
|
+
|
177
|
+
#
|
178
|
+
# Checks if the middlware list is empty
|
179
|
+
#
|
180
|
+
# @return [Boolean] Return true if the middleware list is empty.
|
181
|
+
#
|
182
|
+
def empty?
|
183
|
+
@entries.nil? || @entries.empty?
|
184
|
+
end
|
185
|
+
|
186
|
+
#
|
187
|
+
# Return a list of instantiated middlewares. Each middleware gets
|
188
|
+
# initialize with the args originally passed to `add`, `insert_before` etc.
|
189
|
+
#
|
190
|
+
# @return [Array<any>] The list of instantiated middlewares.
|
191
|
+
#
|
192
|
+
def retrieve
|
193
|
+
map(&:make_new)
|
194
|
+
end
|
195
|
+
|
196
|
+
#
|
197
|
+
# Empty the list of middlewares.
|
198
|
+
#
|
199
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
200
|
+
#
|
201
|
+
def clear
|
202
|
+
entries.clear
|
203
|
+
end
|
204
|
+
|
205
|
+
#
|
206
|
+
# Invoke the chain of middlewares.
|
207
|
+
#
|
208
|
+
# @param [Array<any>] *args The args to pass to each middleware.
|
209
|
+
#
|
210
|
+
def invoke(*args)
|
211
|
+
return yield if empty?
|
212
|
+
|
213
|
+
chain = retrieve.dup
|
214
|
+
traverse_chain = lambda do
|
215
|
+
if chain.empty?
|
216
|
+
yield
|
217
|
+
else
|
218
|
+
chain.shift.call(*args, &traverse_chain)
|
219
|
+
end
|
220
|
+
end
|
221
|
+
traverse_chain.call
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
# Middleware list item.
|
226
|
+
class Entry
|
227
|
+
attr_reader :klass, :args
|
228
|
+
|
229
|
+
#
|
230
|
+
# Build a new entry.
|
231
|
+
#
|
232
|
+
# @param [Class] klass The middleware class.
|
233
|
+
# @param [Array<any>] *args The list of arguments for the middleware.
|
234
|
+
#
|
235
|
+
def initialize(klass, *args)
|
236
|
+
@klass = klass
|
237
|
+
@args = args
|
238
|
+
end
|
239
|
+
|
240
|
+
#
|
241
|
+
# Return an instantiated middleware.
|
242
|
+
#
|
243
|
+
# @return [Any] The instantiated middleware.
|
244
|
+
#
|
245
|
+
def make_new
|
246
|
+
@klass.new(*@args)
|
247
|
+
end
|
248
|
+
end
|
249
|
+
end
|
250
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis'
|
4
|
+
|
5
|
+
module Cloudtasker
|
6
|
+
# A wrapper with helper methods for redis
|
7
|
+
module RedisClient
|
8
|
+
module_function
|
9
|
+
|
10
|
+
# Suffix added to cache keys when locking them
|
11
|
+
LOCK_KEY_SUFFIX = 'lock'
|
12
|
+
|
13
|
+
#
|
14
|
+
# Return the underlying redis client.
|
15
|
+
#
|
16
|
+
# @return [Redis] The redis client.
|
17
|
+
#
|
18
|
+
def client
|
19
|
+
@client ||= Redis.new(Cloudtasker.config.redis || {})
|
20
|
+
end
|
21
|
+
|
22
|
+
#
|
23
|
+
# Get a cache entry and parse it as JSON.
|
24
|
+
#
|
25
|
+
# @param [String, Symbol] key The cache key to fetch.
|
26
|
+
#
|
27
|
+
# @return [Hash, Array] The content of the cache key, parsed as JSON.
|
28
|
+
#
|
29
|
+
def fetch(key)
|
30
|
+
return nil unless (val = client.get(key.to_s))
|
31
|
+
|
32
|
+
JSON.parse(val, symbolize_names: true)
|
33
|
+
rescue JSON::ParserError
|
34
|
+
nil
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Write a cache entry as JSON.
|
39
|
+
#
|
40
|
+
# @param [String, Symbol] key The cache key to write.
|
41
|
+
# @param [Hash, Array] content The content to write.
|
42
|
+
#
|
43
|
+
# @return [String] Redis response code.
|
44
|
+
#
|
45
|
+
def write(key, content)
|
46
|
+
client.set(key.to_s, content.to_json)
|
47
|
+
end
|
48
|
+
|
49
|
+
#
|
50
|
+
# Acquire a lock on a cache entry.
|
51
|
+
#
|
52
|
+
# @example
|
53
|
+
# RedisClient.with_lock('foo')
|
54
|
+
# content = RedisClient.fetch('foo')
|
55
|
+
# RedisClient.set(content.merge(bar: 'bar).to_json)
|
56
|
+
# end
|
57
|
+
#
|
58
|
+
# @param [String] cache_key The cache key to access.
|
59
|
+
#
|
60
|
+
def with_lock(cache_key)
|
61
|
+
return nil unless cache_key
|
62
|
+
|
63
|
+
# Wait to acquire lock
|
64
|
+
lock_key = [cache_key, LOCK_KEY_SUFFIX].join('/')
|
65
|
+
true until client.setnx(lock_key, true)
|
66
|
+
|
67
|
+
# yield content
|
68
|
+
yield
|
69
|
+
ensure
|
70
|
+
client.del(lock_key)
|
71
|
+
end
|
72
|
+
|
73
|
+
#
|
74
|
+
# Clear all redis keys
|
75
|
+
#
|
76
|
+
# @return [Integer] The number of keys deleted
|
77
|
+
#
|
78
|
+
def clear
|
79
|
+
all_keys = keys
|
80
|
+
return 0 if all_keys.empty?
|
81
|
+
|
82
|
+
# Delete all keys
|
83
|
+
del(*all_keys)
|
84
|
+
end
|
85
|
+
|
86
|
+
#
|
87
|
+
# Delegate all methods to the redis client.
|
88
|
+
#
|
89
|
+
# @param [String, Symbol] name The method to delegate.
|
90
|
+
# @param [Array<any>] *args The list of method arguments.
|
91
|
+
# @param [Proc] &block Block passed to the method.
|
92
|
+
#
|
93
|
+
# @return [Any] The method return value
|
94
|
+
#
|
95
|
+
def method_missing(name, *args, &block)
|
96
|
+
if client.respond_to?(name)
|
97
|
+
client.send(name, *args, &block)
|
98
|
+
else
|
99
|
+
super
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
#
|
104
|
+
# Check if the class respond to a certain method.
|
105
|
+
#
|
106
|
+
# @param [String, Symbol] name The name of the method.
|
107
|
+
# @param [Boolean] include_private Whether to check private methods or not. Default to false.
|
108
|
+
#
|
109
|
+
# @return [Boolean] Return true if the class respond to this method.
|
110
|
+
#
|
111
|
+
def respond_to_missing?(name, include_private = false)
|
112
|
+
client.respond_to?(name) || super
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|