cloudtasker 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.rspec +3 -0
- data/.rubocop.yml +27 -0
- data/.travis.yml +7 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +247 -0
- data/LICENSE.txt +21 -0
- data/README.md +43 -0
- data/Rakefile +8 -0
- data/app/controllers/cloudtasker/application_controller.rb +6 -0
- data/app/controllers/cloudtasker/worker_controller.rb +38 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/cloudtasker.gemspec +48 -0
- data/config/routes.rb +5 -0
- data/lib/cloudtasker.rb +31 -0
- data/lib/cloudtasker/authentication_error.rb +6 -0
- data/lib/cloudtasker/authenticator.rb +55 -0
- data/lib/cloudtasker/batch.rb +5 -0
- data/lib/cloudtasker/batch/batch_progress.rb +97 -0
- data/lib/cloudtasker/batch/config.rb +11 -0
- data/lib/cloudtasker/batch/extension/worker.rb +13 -0
- data/lib/cloudtasker/batch/job.rb +320 -0
- data/lib/cloudtasker/batch/middleware.rb +24 -0
- data/lib/cloudtasker/batch/middleware/server.rb +14 -0
- data/lib/cloudtasker/config.rb +122 -0
- data/lib/cloudtasker/cron.rb +5 -0
- data/lib/cloudtasker/cron/config.rb +11 -0
- data/lib/cloudtasker/cron/job.rb +207 -0
- data/lib/cloudtasker/cron/middleware.rb +21 -0
- data/lib/cloudtasker/cron/middleware/server.rb +14 -0
- data/lib/cloudtasker/cron/schedule.rb +227 -0
- data/lib/cloudtasker/engine.rb +20 -0
- data/lib/cloudtasker/invalid_worker_error.rb +6 -0
- data/lib/cloudtasker/meta_store.rb +86 -0
- data/lib/cloudtasker/middleware/chain.rb +250 -0
- data/lib/cloudtasker/redis_client.rb +115 -0
- data/lib/cloudtasker/task.rb +175 -0
- data/lib/cloudtasker/unique_job.rb +5 -0
- data/lib/cloudtasker/unique_job/config.rb +10 -0
- data/lib/cloudtasker/unique_job/conflict_strategy/base_strategy.rb +37 -0
- data/lib/cloudtasker/unique_job/conflict_strategy/raise.rb +28 -0
- data/lib/cloudtasker/unique_job/conflict_strategy/reject.rb +11 -0
- data/lib/cloudtasker/unique_job/conflict_strategy/reschedule.rb +30 -0
- data/lib/cloudtasker/unique_job/job.rb +136 -0
- data/lib/cloudtasker/unique_job/lock/base_lock.rb +70 -0
- data/lib/cloudtasker/unique_job/lock/no_op.rb +11 -0
- data/lib/cloudtasker/unique_job/lock/until_executed.rb +34 -0
- data/lib/cloudtasker/unique_job/lock/until_executing.rb +30 -0
- data/lib/cloudtasker/unique_job/lock/while_executing.rb +23 -0
- data/lib/cloudtasker/unique_job/lock_error.rb +8 -0
- data/lib/cloudtasker/unique_job/middleware.rb +36 -0
- data/lib/cloudtasker/unique_job/middleware/client.rb +14 -0
- data/lib/cloudtasker/unique_job/middleware/server.rb +14 -0
- data/lib/cloudtasker/version.rb +5 -0
- data/lib/cloudtasker/worker.rb +211 -0
- metadata +286 -0
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Cloudtasker
|
4
|
+
# Cloudtasker Rails engine
|
5
|
+
class Engine < ::Rails::Engine
|
6
|
+
isolate_namespace Cloudtasker
|
7
|
+
|
8
|
+
initializer 'cloudtasker', before: :load_config_initializers do
|
9
|
+
Rails.application.routes.append do
|
10
|
+
mount Cloudtasker::Engine, at: '/cloudtasker'
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
config.generators do |g|
|
15
|
+
g.test_framework :rspec, fixture: false
|
16
|
+
g.assets false
|
17
|
+
g.helper false
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Cloudtasker
|
4
|
+
# Manage meta information on workers. This meta stored is intended
|
5
|
+
# to be used by middlewares needing to store extra information on the
|
6
|
+
# job.
|
7
|
+
# The objective of this class is to provide a shared store to middleware
|
8
|
+
# while controlling access to its keys by preveenting access the hash directly
|
9
|
+
# (e.g. avoid wild merge or replace operations).
|
10
|
+
class MetaStore
|
11
|
+
#
|
12
|
+
# Build a new instance of the class.
|
13
|
+
#
|
14
|
+
# @param [<Type>] hash The worker meta hash
|
15
|
+
#
|
16
|
+
def initialize(hash = {})
|
17
|
+
@meta = JSON.parse((hash || {}).to_json, symbolize_names: true)
|
18
|
+
end
|
19
|
+
|
20
|
+
#
|
21
|
+
# Retrieve meta entry.
|
22
|
+
#
|
23
|
+
# @param [String, Symbol] key The key of the meta entry.
|
24
|
+
#
|
25
|
+
# @return [Any] The value of the meta entry.
|
26
|
+
#
|
27
|
+
def get(key)
|
28
|
+
@meta[key.to_sym] if key
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# Set meta entry
|
33
|
+
#
|
34
|
+
# @param [String, Symbol] key The key of the meta entry.
|
35
|
+
# @param [Any] val The value of the meta entry.
|
36
|
+
#
|
37
|
+
# @return [Any] The value set
|
38
|
+
#
|
39
|
+
def set(key, val)
|
40
|
+
@meta[key.to_sym] = val if key
|
41
|
+
end
|
42
|
+
|
43
|
+
#
|
44
|
+
# Remove a meta information.
|
45
|
+
#
|
46
|
+
# @param [String, Symbol] key The key of the entry to delete.
|
47
|
+
#
|
48
|
+
# @return [Any] The value of the deleted key
|
49
|
+
#
|
50
|
+
def del(key)
|
51
|
+
@meta.delete(key.to_sym) if key
|
52
|
+
end
|
53
|
+
|
54
|
+
#
|
55
|
+
# Return the meta store as Hash.
|
56
|
+
#
|
57
|
+
# @return [Hash] The meta store as Hash.
|
58
|
+
#
|
59
|
+
def to_h
|
60
|
+
# Deep dup
|
61
|
+
JSON.parse(@meta.to_json, symbolize_names: true)
|
62
|
+
end
|
63
|
+
|
64
|
+
#
|
65
|
+
# Return the meta store as json.
|
66
|
+
#
|
67
|
+
# @param [Array<any>] *arg The to_json args.
|
68
|
+
#
|
69
|
+
# @return [String] The meta store as json.
|
70
|
+
#
|
71
|
+
def to_json(*arg)
|
72
|
+
@meta.to_json(*arg)
|
73
|
+
end
|
74
|
+
|
75
|
+
#
|
76
|
+
# Equality operator.
|
77
|
+
#
|
78
|
+
# @param [Any] other The object being compared.
|
79
|
+
#
|
80
|
+
# @return [Boolean] True if the object is equal.
|
81
|
+
#
|
82
|
+
def ==(other)
|
83
|
+
to_json == other.try(:to_json)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,250 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Cloudtasker
|
4
|
+
module Middleware
|
5
|
+
# The class below was originally taken from Sidekiq.
|
6
|
+
# See: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/middleware/chain.rb
|
7
|
+
#
|
8
|
+
# Middleware are callables configured to run before/after a message is processed.
|
9
|
+
# Middlewares can be configured to run on the client side (when jobs are pushed
|
10
|
+
# to Cloud Tasks) as well as on the server side (when jobs are processed by
|
11
|
+
# your application)
|
12
|
+
#
|
13
|
+
# To add middleware for the client:
|
14
|
+
#
|
15
|
+
# Cloudtasker.configure do |config|
|
16
|
+
# config.client_middleware do |chain|
|
17
|
+
# chain.add MyClientHook
|
18
|
+
# end
|
19
|
+
# end
|
20
|
+
#
|
21
|
+
# To modify middleware for the server, just call
|
22
|
+
# with another block:
|
23
|
+
#
|
24
|
+
# Cloudtasker.configure do |config|
|
25
|
+
# config.server_middleware do |chain|
|
26
|
+
# chain.add MyServerHook
|
27
|
+
# chain.remove ActiveRecord
|
28
|
+
# end
|
29
|
+
# end
|
30
|
+
#
|
31
|
+
# To insert immediately preceding another entry:
|
32
|
+
#
|
33
|
+
# Cloudtasker.configure do |config|
|
34
|
+
# config.client_middleware do |chain|
|
35
|
+
# chain.insert_before ActiveRecord, MyClientHook
|
36
|
+
# end
|
37
|
+
# end
|
38
|
+
#
|
39
|
+
# To insert immediately after another entry:
|
40
|
+
#
|
41
|
+
# Cloudtasker.configure do |config|
|
42
|
+
# config.client_middleware do |chain|
|
43
|
+
# chain.insert_after ActiveRecord, MyClientHook
|
44
|
+
# end
|
45
|
+
# end
|
46
|
+
#
|
47
|
+
# This is an example of a minimal server middleware:
|
48
|
+
#
|
49
|
+
# class MyServerHook
|
50
|
+
# def call(worker_instance, msg, queue)
|
51
|
+
# puts "Before work"
|
52
|
+
# yield
|
53
|
+
# puts "After work"
|
54
|
+
# end
|
55
|
+
# end
|
56
|
+
#
|
57
|
+
# This is an example of a minimal client middleware, note
|
58
|
+
# the method must return the result or the job will not push
|
59
|
+
# to Redis:
|
60
|
+
#
|
61
|
+
# class MyClientHook
|
62
|
+
# def call(worker_class, msg, queue, redis_pool)
|
63
|
+
# puts "Before push"
|
64
|
+
# result = yield
|
65
|
+
# puts "After push"
|
66
|
+
# result
|
67
|
+
# end
|
68
|
+
# end
|
69
|
+
#
|
70
|
+
class Chain
|
71
|
+
include Enumerable
|
72
|
+
|
73
|
+
#
|
74
|
+
# Build a new middleware chain.
|
75
|
+
#
|
76
|
+
def initialize
|
77
|
+
@entries = nil
|
78
|
+
yield self if block_given?
|
79
|
+
end
|
80
|
+
|
81
|
+
#
|
82
|
+
# Iterate over the list middlewares and execute the block on each item.
|
83
|
+
#
|
84
|
+
# @param [Proc] &block The block to execute on each item.
|
85
|
+
#
|
86
|
+
def each(&block)
|
87
|
+
entries.each(&block)
|
88
|
+
end
|
89
|
+
|
90
|
+
#
|
91
|
+
# Return the list of middlewares.
|
92
|
+
#
|
93
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The list of middlewares
|
94
|
+
#
|
95
|
+
def entries
|
96
|
+
@entries ||= []
|
97
|
+
end
|
98
|
+
|
99
|
+
#
|
100
|
+
# Remove a middleware from the list.
|
101
|
+
#
|
102
|
+
# @param [Class] klass The middleware class to remove.
|
103
|
+
#
|
104
|
+
def remove(klass)
|
105
|
+
entries.delete_if { |entry| entry.klass == klass }
|
106
|
+
end
|
107
|
+
|
108
|
+
#
|
109
|
+
# Add a middleware at the end of the list.
|
110
|
+
#
|
111
|
+
# @param [Class] klass The middleware class to add.
|
112
|
+
# @param [Arry<any>] *args The list of arguments to the middleware.
|
113
|
+
#
|
114
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
115
|
+
#
|
116
|
+
def add(klass, *args)
|
117
|
+
remove(klass) if exists?(klass)
|
118
|
+
entries << Entry.new(klass, *args)
|
119
|
+
end
|
120
|
+
|
121
|
+
#
|
122
|
+
# Add a middleware at the beginning of the list.
|
123
|
+
#
|
124
|
+
# @param [Class] klass The middleware class to add.
|
125
|
+
# @param [Arry<any>] *args The list of arguments to the middleware.
|
126
|
+
#
|
127
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
128
|
+
#
|
129
|
+
def prepend(klass, *args)
|
130
|
+
remove(klass) if exists?(klass)
|
131
|
+
entries.insert(0, Entry.new(klass, *args))
|
132
|
+
end
|
133
|
+
|
134
|
+
#
|
135
|
+
# Add a middleware before another middleware.
|
136
|
+
#
|
137
|
+
# @param [Class] oldklass The middleware class before which the new middleware should be inserted.
|
138
|
+
# @param [Class] newklass The middleware class to insert.
|
139
|
+
# @param [Arry<any>] *args The list of arguments for the inserted middleware.
|
140
|
+
#
|
141
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
142
|
+
#
|
143
|
+
def insert_before(oldklass, newklass, *args)
|
144
|
+
i = entries.index { |entry| entry.klass == newklass }
|
145
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
146
|
+
i = entries.index { |entry| entry.klass == oldklass } || 0
|
147
|
+
entries.insert(i, new_entry)
|
148
|
+
end
|
149
|
+
|
150
|
+
#
|
151
|
+
# Add a middleware after another middleware.
|
152
|
+
#
|
153
|
+
# @param [Class] oldklass The middleware class after which the new middleware should be inserted.
|
154
|
+
# @param [Class] newklass The middleware class to insert.
|
155
|
+
# @param [Arry<any>] *args The list of arguments for the inserted middleware.
|
156
|
+
#
|
157
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
158
|
+
#
|
159
|
+
def insert_after(oldklass, newklass, *args)
|
160
|
+
i = entries.index { |entry| entry.klass == newklass }
|
161
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
162
|
+
i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
|
163
|
+
entries.insert(i + 1, new_entry)
|
164
|
+
end
|
165
|
+
|
166
|
+
#
|
167
|
+
# Checks if middleware has been added to the list.
|
168
|
+
#
|
169
|
+
# @param [Class] klass The middleware class to check.
|
170
|
+
#
|
171
|
+
# @return [Boolean] Return true if the middleware is in the list.
|
172
|
+
#
|
173
|
+
def exists?(klass)
|
174
|
+
any? { |entry| entry.klass == klass }
|
175
|
+
end
|
176
|
+
|
177
|
+
#
|
178
|
+
# Checks if the middlware list is empty
|
179
|
+
#
|
180
|
+
# @return [Boolean] Return true if the middleware list is empty.
|
181
|
+
#
|
182
|
+
def empty?
|
183
|
+
@entries.nil? || @entries.empty?
|
184
|
+
end
|
185
|
+
|
186
|
+
#
|
187
|
+
# Return a list of instantiated middlewares. Each middleware gets
|
188
|
+
# initialize with the args originally passed to `add`, `insert_before` etc.
|
189
|
+
#
|
190
|
+
# @return [Array<any>] The list of instantiated middlewares.
|
191
|
+
#
|
192
|
+
def retrieve
|
193
|
+
map(&:make_new)
|
194
|
+
end
|
195
|
+
|
196
|
+
#
|
197
|
+
# Empty the list of middlewares.
|
198
|
+
#
|
199
|
+
# @return [Array<Cloudtasker::Middleware::Chain::Entry>] The updated list of middlewares
|
200
|
+
#
|
201
|
+
def clear
|
202
|
+
entries.clear
|
203
|
+
end
|
204
|
+
|
205
|
+
#
|
206
|
+
# Invoke the chain of middlewares.
|
207
|
+
#
|
208
|
+
# @param [Array<any>] *args The args to pass to each middleware.
|
209
|
+
#
|
210
|
+
def invoke(*args)
|
211
|
+
return yield if empty?
|
212
|
+
|
213
|
+
chain = retrieve.dup
|
214
|
+
traverse_chain = lambda do
|
215
|
+
if chain.empty?
|
216
|
+
yield
|
217
|
+
else
|
218
|
+
chain.shift.call(*args, &traverse_chain)
|
219
|
+
end
|
220
|
+
end
|
221
|
+
traverse_chain.call
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
# Middleware list item.
|
226
|
+
class Entry
|
227
|
+
attr_reader :klass, :args
|
228
|
+
|
229
|
+
#
|
230
|
+
# Build a new entry.
|
231
|
+
#
|
232
|
+
# @param [Class] klass The middleware class.
|
233
|
+
# @param [Array<any>] *args The list of arguments for the middleware.
|
234
|
+
#
|
235
|
+
def initialize(klass, *args)
|
236
|
+
@klass = klass
|
237
|
+
@args = args
|
238
|
+
end
|
239
|
+
|
240
|
+
#
|
241
|
+
# Return an instantiated middleware.
|
242
|
+
#
|
243
|
+
# @return [Any] The instantiated middleware.
|
244
|
+
#
|
245
|
+
def make_new
|
246
|
+
@klass.new(*@args)
|
247
|
+
end
|
248
|
+
end
|
249
|
+
end
|
250
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'redis'
|
4
|
+
|
5
|
+
module Cloudtasker
|
6
|
+
# A wrapper with helper methods for redis
|
7
|
+
module RedisClient
|
8
|
+
module_function
|
9
|
+
|
10
|
+
# Suffix added to cache keys when locking them
|
11
|
+
LOCK_KEY_SUFFIX = 'lock'
|
12
|
+
|
13
|
+
#
|
14
|
+
# Return the underlying redis client.
|
15
|
+
#
|
16
|
+
# @return [Redis] The redis client.
|
17
|
+
#
|
18
|
+
def client
|
19
|
+
@client ||= Redis.new(Cloudtasker.config.redis || {})
|
20
|
+
end
|
21
|
+
|
22
|
+
#
|
23
|
+
# Get a cache entry and parse it as JSON.
|
24
|
+
#
|
25
|
+
# @param [String, Symbol] key The cache key to fetch.
|
26
|
+
#
|
27
|
+
# @return [Hash, Array] The content of the cache key, parsed as JSON.
|
28
|
+
#
|
29
|
+
def fetch(key)
|
30
|
+
return nil unless (val = client.get(key.to_s))
|
31
|
+
|
32
|
+
JSON.parse(val, symbolize_names: true)
|
33
|
+
rescue JSON::ParserError
|
34
|
+
nil
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Write a cache entry as JSON.
|
39
|
+
#
|
40
|
+
# @param [String, Symbol] key The cache key to write.
|
41
|
+
# @param [Hash, Array] content The content to write.
|
42
|
+
#
|
43
|
+
# @return [String] Redis response code.
|
44
|
+
#
|
45
|
+
def write(key, content)
|
46
|
+
client.set(key.to_s, content.to_json)
|
47
|
+
end
|
48
|
+
|
49
|
+
#
|
50
|
+
# Acquire a lock on a cache entry.
|
51
|
+
#
|
52
|
+
# @example
|
53
|
+
# RedisClient.with_lock('foo')
|
54
|
+
# content = RedisClient.fetch('foo')
|
55
|
+
# RedisClient.set(content.merge(bar: 'bar).to_json)
|
56
|
+
# end
|
57
|
+
#
|
58
|
+
# @param [String] cache_key The cache key to access.
|
59
|
+
#
|
60
|
+
def with_lock(cache_key)
|
61
|
+
return nil unless cache_key
|
62
|
+
|
63
|
+
# Wait to acquire lock
|
64
|
+
lock_key = [cache_key, LOCK_KEY_SUFFIX].join('/')
|
65
|
+
true until client.setnx(lock_key, true)
|
66
|
+
|
67
|
+
# yield content
|
68
|
+
yield
|
69
|
+
ensure
|
70
|
+
client.del(lock_key)
|
71
|
+
end
|
72
|
+
|
73
|
+
#
|
74
|
+
# Clear all redis keys
|
75
|
+
#
|
76
|
+
# @return [Integer] The number of keys deleted
|
77
|
+
#
|
78
|
+
def clear
|
79
|
+
all_keys = keys
|
80
|
+
return 0 if all_keys.empty?
|
81
|
+
|
82
|
+
# Delete all keys
|
83
|
+
del(*all_keys)
|
84
|
+
end
|
85
|
+
|
86
|
+
#
|
87
|
+
# Delegate all methods to the redis client.
|
88
|
+
#
|
89
|
+
# @param [String, Symbol] name The method to delegate.
|
90
|
+
# @param [Array<any>] *args The list of method arguments.
|
91
|
+
# @param [Proc] &block Block passed to the method.
|
92
|
+
#
|
93
|
+
# @return [Any] The method return value
|
94
|
+
#
|
95
|
+
def method_missing(name, *args, &block)
|
96
|
+
if client.respond_to?(name)
|
97
|
+
client.send(name, *args, &block)
|
98
|
+
else
|
99
|
+
super
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
#
|
104
|
+
# Check if the class respond to a certain method.
|
105
|
+
#
|
106
|
+
# @param [String, Symbol] name The name of the method.
|
107
|
+
# @param [Boolean] include_private Whether to check private methods or not. Default to false.
|
108
|
+
#
|
109
|
+
# @return [Boolean] Return true if the class respond to this method.
|
110
|
+
#
|
111
|
+
def respond_to_missing?(name, include_private = false)
|
112
|
+
client.respond_to?(name) || super
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|