cloudtasker 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/Gemfile.lock +2 -2
- data/README.md +128 -12
- data/app/controllers/cloudtasker/worker_controller.rb +1 -1
- data/cloudtasker.gemspec +2 -2
- data/docs/BATCH_JOBS.md +26 -1
- data/docs/CRON_JOBS.md +5 -1
- data/exe/cloudtasker +13 -1
- data/lib/cloudtasker.rb +0 -1
- data/lib/cloudtasker/backend/google_cloud_task.rb +41 -8
- data/lib/cloudtasker/backend/memory_task.rb +5 -3
- data/lib/cloudtasker/backend/redis_task.rb +17 -9
- data/lib/cloudtasker/batch/batch_progress.rb +11 -2
- data/lib/cloudtasker/batch/job.rb +18 -4
- data/lib/cloudtasker/cli.rb +6 -5
- data/lib/cloudtasker/cloud_task.rb +4 -2
- data/lib/cloudtasker/config.rb +14 -8
- data/lib/cloudtasker/cron/job.rb +2 -2
- data/lib/cloudtasker/cron/schedule.rb +26 -14
- data/lib/cloudtasker/local_server.rb +44 -22
- data/lib/cloudtasker/redis_client.rb +5 -6
- data/lib/cloudtasker/unique_job/job.rb +2 -2
- data/lib/cloudtasker/version.rb +1 -1
- data/lib/cloudtasker/worker.rb +44 -9
- data/lib/cloudtasker/worker_handler.rb +5 -3
- data/lib/cloudtasker/worker_logger.rb +1 -1
- data/lib/cloudtasker/worker_wrapper.rb +52 -0
- data/lib/tasks/setup_queue.rake +12 -2
- metadata +5 -5
- data/lib/cloudtasker/railtie.rb +0 -10
@@ -4,9 +4,7 @@ require 'redis'
|
|
4
4
|
|
5
5
|
module Cloudtasker
|
6
6
|
# A wrapper with helper methods for redis
|
7
|
-
|
8
|
-
module_function
|
9
|
-
|
7
|
+
class RedisClient
|
10
8
|
# Suffix added to cache keys when locking them
|
11
9
|
LOCK_KEY_PREFIX = 'cloudtasker/lock'
|
12
10
|
|
@@ -50,9 +48,10 @@ module Cloudtasker
|
|
50
48
|
# Acquire a lock on a cache entry.
|
51
49
|
#
|
52
50
|
# @example
|
53
|
-
# RedisClient.
|
54
|
-
#
|
55
|
-
#
|
51
|
+
# redis = RedisClient.new
|
52
|
+
# redis.with_lock('foo')
|
53
|
+
# content = redis.fetch('foo')
|
54
|
+
# redis.set(content.merge(bar: 'bar).to_json)
|
56
55
|
# end
|
57
56
|
#
|
58
57
|
# @param [String] cache_key The cache key to access.
|
@@ -100,10 +100,10 @@ module Cloudtasker
|
|
100
100
|
#
|
101
101
|
# Return the Cloudtasker redis client.
|
102
102
|
#
|
103
|
-
# @return [
|
103
|
+
# @return [Cloudtasker::RedisClient] The cloudtasker redis client.
|
104
104
|
#
|
105
105
|
def redis
|
106
|
-
Cloudtasker::RedisClient
|
106
|
+
@redis ||= Cloudtasker::RedisClient.new
|
107
107
|
end
|
108
108
|
|
109
109
|
#
|
data/lib/cloudtasker/version.rb
CHANGED
data/lib/cloudtasker/worker.rb
CHANGED
@@ -6,6 +6,7 @@ module Cloudtasker
|
|
6
6
|
# Add class method to including class
|
7
7
|
def self.included(base)
|
8
8
|
base.extend(ClassMethods)
|
9
|
+
base.attr_writer :job_queue
|
9
10
|
base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries
|
10
11
|
end
|
11
12
|
|
@@ -45,7 +46,7 @@ module Cloudtasker
|
|
45
46
|
return nil unless worker_klass.include?(self)
|
46
47
|
|
47
48
|
# Return instantiated worker
|
48
|
-
worker_klass.new(payload.slice(:job_args, :job_id, :job_meta, :job_retries))
|
49
|
+
worker_klass.new(payload.slice(:job_queue, :job_args, :job_id, :job_meta, :job_retries))
|
49
50
|
rescue NameError
|
50
51
|
nil
|
51
52
|
end
|
@@ -81,7 +82,7 @@ module Cloudtasker
|
|
81
82
|
# @return [Cloudtasker::CloudTask] The Google Task response
|
82
83
|
#
|
83
84
|
def perform_async(*args)
|
84
|
-
|
85
|
+
schedule(args: args)
|
85
86
|
end
|
86
87
|
|
87
88
|
#
|
@@ -93,7 +94,7 @@ module Cloudtasker
|
|
93
94
|
# @return [Cloudtasker::CloudTask] The Google Task response
|
94
95
|
#
|
95
96
|
def perform_in(interval, *args)
|
96
|
-
|
97
|
+
schedule(args: args, time_in: interval)
|
97
98
|
end
|
98
99
|
|
99
100
|
#
|
@@ -105,7 +106,21 @@ module Cloudtasker
|
|
105
106
|
# @return [Cloudtasker::CloudTask] The Google Task response
|
106
107
|
#
|
107
108
|
def perform_at(time_at, *args)
|
108
|
-
|
109
|
+
schedule(args: args, time_at: time_at)
|
110
|
+
end
|
111
|
+
|
112
|
+
#
|
113
|
+
# Enqueue a worker with explicity options.
|
114
|
+
#
|
115
|
+
# @param [Array<any>] args The job arguments.
|
116
|
+
# @param [Time, Integer] time_in The delay in seconds.
|
117
|
+
# @param [Time, Integer] time_at The time at which the job should run.
|
118
|
+
# @param [String, Symbol] queue The queue on which the worker should run.
|
119
|
+
#
|
120
|
+
# @return [Cloudtasker::CloudTask] The Google Task response
|
121
|
+
#
|
122
|
+
def schedule(args: nil, time_in: nil, time_at: nil, queue: nil)
|
123
|
+
new(job_args: args, job_queue: queue).schedule({ interval: time_in, time_at: time_at }.compact)
|
109
124
|
end
|
110
125
|
|
111
126
|
#
|
@@ -124,11 +139,30 @@ module Cloudtasker
|
|
124
139
|
# @param [Array<any>] job_args The list of perform args.
|
125
140
|
# @param [String] job_id A unique ID identifying this job.
|
126
141
|
#
|
127
|
-
def initialize(job_args:
|
128
|
-
@job_args = job_args
|
142
|
+
def initialize(job_queue: nil, job_args: nil, job_id: nil, job_meta: {}, job_retries: 0)
|
143
|
+
@job_args = job_args || []
|
129
144
|
@job_id = job_id || SecureRandom.uuid
|
130
145
|
@job_meta = MetaStore.new(job_meta)
|
131
146
|
@job_retries = job_retries || 0
|
147
|
+
@job_queue = job_queue
|
148
|
+
end
|
149
|
+
|
150
|
+
#
|
151
|
+
# Return the class name of the worker.
|
152
|
+
#
|
153
|
+
# @return [String] The class name.
|
154
|
+
#
|
155
|
+
def job_class_name
|
156
|
+
self.class.to_s
|
157
|
+
end
|
158
|
+
|
159
|
+
#
|
160
|
+
# Return the queue to use for this worker.
|
161
|
+
#
|
162
|
+
# @return [String] The name of queue.
|
163
|
+
#
|
164
|
+
def job_queue
|
165
|
+
(@job_queue ||= self.class.cloudtasker_options_hash[:queue] || Config::DEFAULT_JOB_QUEUE).to_s
|
132
166
|
end
|
133
167
|
|
134
168
|
#
|
@@ -199,10 +233,10 @@ module Cloudtasker
|
|
199
233
|
# Return a new instance of the worker with the same args and metadata
|
200
234
|
# but with a different id.
|
201
235
|
#
|
202
|
-
# @return [
|
236
|
+
# @return [Cloudtasker::Worker] <description>
|
203
237
|
#
|
204
238
|
def new_instance
|
205
|
-
self.class.new(job_args: job_args, job_meta: job_meta)
|
239
|
+
self.class.new(job_queue: job_queue, job_args: job_args, job_meta: job_meta)
|
206
240
|
end
|
207
241
|
|
208
242
|
#
|
@@ -216,7 +250,8 @@ module Cloudtasker
|
|
216
250
|
job_id: job_id,
|
217
251
|
job_args: job_args,
|
218
252
|
job_meta: job_meta.to_h,
|
219
|
-
job_retries: job_retries
|
253
|
+
job_retries: job_retries,
|
254
|
+
job_queue: job_queue
|
220
255
|
}
|
221
256
|
end
|
222
257
|
|
@@ -5,7 +5,7 @@ require 'google/cloud/tasks'
|
|
5
5
|
module Cloudtasker
|
6
6
|
# Build, serialize and schedule tasks on the processing backend.
|
7
7
|
class WorkerHandler
|
8
|
-
attr_reader :worker
|
8
|
+
attr_reader :worker
|
9
9
|
|
10
10
|
# Alrogith used to sign the verification token
|
11
11
|
JWT_ALG = 'HS256'
|
@@ -46,7 +46,8 @@ module Cloudtasker
|
|
46
46
|
'Authorization' => "Bearer #{Authenticator.verification_token}"
|
47
47
|
},
|
48
48
|
body: worker_payload.to_json
|
49
|
-
}
|
49
|
+
},
|
50
|
+
queue: worker.job_queue
|
50
51
|
}
|
51
52
|
end
|
52
53
|
|
@@ -64,7 +65,8 @@ module Cloudtasker
|
|
64
65
|
#
|
65
66
|
def worker_payload
|
66
67
|
@worker_payload ||= {
|
67
|
-
worker: worker.
|
68
|
+
worker: worker.job_class_name,
|
69
|
+
job_queue: worker.job_queue,
|
68
70
|
job_id: worker.job_id,
|
69
71
|
job_args: worker.job_args,
|
70
72
|
job_meta: worker.job_meta.to_h
|
@@ -11,7 +11,7 @@ module Cloudtasker
|
|
11
11
|
end
|
12
12
|
|
13
13
|
# Only log the job meta information by default (exclude arguments)
|
14
|
-
DEFAULT_CONTEXT_PROCESSOR = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta) }
|
14
|
+
DEFAULT_CONTEXT_PROCESSOR = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta, :job_queue) }
|
15
15
|
|
16
16
|
#
|
17
17
|
# Build a new instance of the class.
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'cloudtasker/worker'
|
4
|
+
|
5
|
+
module Cloudtasker
|
6
|
+
# A worker class used to schedule jobs without actually
|
7
|
+
# instantiating the worker class. This is useful for middlewares
|
8
|
+
# needing to enqueue jobs in a Rails initializer. Rails 6 complains
|
9
|
+
# about instantiating workers in an iniitializer because of autoloading
|
10
|
+
# in zeitwerk mode.
|
11
|
+
#
|
12
|
+
# Downside of this wrapper: any cloudtasker_options specified on on the
|
13
|
+
# worker_class will be ignored.
|
14
|
+
#
|
15
|
+
# See: https://github.com/rails/rails/issues/36363
|
16
|
+
#
|
17
|
+
class WorkerWrapper
|
18
|
+
include Worker
|
19
|
+
|
20
|
+
attr_accessor :worker_name
|
21
|
+
|
22
|
+
#
|
23
|
+
# Build a new instance of the class.
|
24
|
+
#
|
25
|
+
# @param [String] worker_class The name of the worker class.
|
26
|
+
# @param [Hash] **opts The worker arguments.
|
27
|
+
#
|
28
|
+
def initialize(worker_name:, **opts)
|
29
|
+
@worker_name = worker_name
|
30
|
+
super(opts)
|
31
|
+
end
|
32
|
+
|
33
|
+
#
|
34
|
+
# Override parent. Return the underlying worker class name.
|
35
|
+
#
|
36
|
+
# @return [String] The worker class.
|
37
|
+
#
|
38
|
+
def job_class_name
|
39
|
+
worker_name
|
40
|
+
end
|
41
|
+
|
42
|
+
#
|
43
|
+
# Return a new instance of the worker with the same args and metadata
|
44
|
+
# but with a different id.
|
45
|
+
#
|
46
|
+
# @return [Cloudtasker::WorkerWrapper] <description>
|
47
|
+
#
|
48
|
+
def new_instance
|
49
|
+
self.class.new(worker_name: worker_name, job_queue: job_queue, job_args: job_args, job_meta: job_meta)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
data/lib/tasks/setup_queue.rake
CHANGED
@@ -1,10 +1,20 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'cloudtasker/backend/google_cloud_task'
|
4
|
+
require 'cloudtasker/config'
|
5
|
+
|
6
|
+
ENV['GOOGLE_AUTH_SUPPRESS_CREDENTIALS_WARNINGS'] ||= 'true'
|
4
7
|
|
5
8
|
namespace :cloudtasker do
|
6
|
-
desc 'Setup
|
9
|
+
desc 'Setup a Cloud Task queue. (default options: ' \
|
10
|
+
"name=#{Cloudtasker::Config::DEFAULT_JOB_QUEUE}, " \
|
11
|
+
"concurrency=#{Cloudtasker::Config::DEFAULT_QUEUE_CONCURRENCY}, " \
|
12
|
+
"retries=#{Cloudtasker::Config::DEFAULT_QUEUE_RETRIES})"
|
7
13
|
task setup_queue: :environment do
|
8
|
-
Cloudtasker::Backend::GoogleCloudTask.setup_queue
|
14
|
+
puts Cloudtasker::Backend::GoogleCloudTask.setup_queue(
|
15
|
+
name: ENV['name'],
|
16
|
+
concurrency: ENV['concurrency'],
|
17
|
+
retries: ENV['retries']
|
18
|
+
)
|
9
19
|
end
|
10
20
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloudtasker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arnaud Lachaume
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-11-
|
11
|
+
date: 2019-11-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -248,7 +248,7 @@ dependencies:
|
|
248
248
|
- - ">="
|
249
249
|
- !ruby/object:Gem::Version
|
250
250
|
version: '0'
|
251
|
-
description: Background jobs for Ruby using Google Cloud Tasks (
|
251
|
+
description: Background jobs for Ruby using Google Cloud Tasks (beta)
|
252
252
|
email:
|
253
253
|
- arnaud.lachaume@keypup.io
|
254
254
|
executables:
|
@@ -322,7 +322,6 @@ files:
|
|
322
322
|
- lib/cloudtasker/local_server.rb
|
323
323
|
- lib/cloudtasker/meta_store.rb
|
324
324
|
- lib/cloudtasker/middleware/chain.rb
|
325
|
-
- lib/cloudtasker/railtie.rb
|
326
325
|
- lib/cloudtasker/redis_client.rb
|
327
326
|
- lib/cloudtasker/testing.rb
|
328
327
|
- lib/cloudtasker/unique_job.rb
|
@@ -344,6 +343,7 @@ files:
|
|
344
343
|
- lib/cloudtasker/worker.rb
|
345
344
|
- lib/cloudtasker/worker_handler.rb
|
346
345
|
- lib/cloudtasker/worker_logger.rb
|
346
|
+
- lib/cloudtasker/worker_wrapper.rb
|
347
347
|
- lib/tasks/setup_queue.rake
|
348
348
|
homepage: https://github.com/keypup-io/cloudtasker
|
349
349
|
licenses:
|
@@ -371,5 +371,5 @@ rubyforge_project:
|
|
371
371
|
rubygems_version: 2.7.9
|
372
372
|
signing_key:
|
373
373
|
specification_version: 4
|
374
|
-
summary: Background jobs for Ruby using Google Cloud Tasks (
|
374
|
+
summary: Background jobs for Ruby using Google Cloud Tasks (beta)
|
375
375
|
test_files: []
|